From noreply at buildbot.pypy.org  Sat Jun  1 01:05:29 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 01:05:29 +0200 (CEST)
Subject: [pypy-commit] cffi default: One more test
Message-ID: <20130531230529.D836F1C153B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1262:720cdf84b1e3
Date: 2013-06-01 01:03 +0200
http://bitbucket.org/cffi/cffi/changeset/720cdf84b1e3/

Log:	One more test

diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py
--- a/testing/test_ffi_backend.py
+++ b/testing/test_ffi_backend.py
@@ -138,6 +138,7 @@
         self.check("char x; long long :0; char y;", L, 1, L + 1)
         self.check("short x, y; int :0; int :0;", 2, 2, 4)
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
     def test_error_cases(self):
         ffi = FFI()

From noreply at buildbot.pypy.org  Sat Jun  1 01:05:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 01:05:30 +0200 (CEST)
Subject: [pypy-commit] cffi default: Always run the C compiler,
 even if it's MSVC.  Found out that the
Message-ID: <20130531230530.F256C1C3007@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1263:cc890bf469a7
Date: 2013-06-01 01:05 +0200
http://bitbucket.org/cffi/cffi/changeset/cc890bf469a7/

Log:	Always run the C compiler, even if it's MSVC. Found out that the
	logic to pack bitfields is, of course, very different.

diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py
--- a/testing/test_ffi_backend.py
+++ b/testing/test_ffi_backend.py
@@ -44,36 +44,31 @@
         ffi.cdef("struct s1 { %s };" % source)
         ctype = ffi.typeof("struct s1")
         # verify the information with gcc
-        if sys.platform != "win32":
-            ffi1 = FFI()
-            ffi1.cdef("""
-                static const int Gofs_y, Galign, Gsize;
-                struct s1 *try_with_value(int fieldnum, long long value);
-            """)
-            fnames = [name for name, cfield in ctype.fields
-                           if name and cfield.bitsize > 0]
-            setters = ['case %d: s.%s = value; break;' % iname
-                       for iname in enumerate(fnames)]
-            lib = ffi1.verify("""
-                struct s1 { %s };
-                struct sa { char a; struct s1 b; };
-                #define Gofs_y  offsetof(struct s1, y)
-                #define Galign  offsetof(struct sa, b)
-                #define Gsize   sizeof(struct s1)
-                struct s1 *try_with_value(int fieldnum, long long value)
-                {
-                    static struct s1 s;
-                    memset(&s, 0, sizeof(s));
-                    switch (fieldnum) { %s }
-                    return &s;
-                }
-            """ % (source, ' '.join(setters)))
-            assert lib.Gofs_y == expected_ofs_y
-            assert lib.Galign == expected_align
-            assert lib.Gsize  == expected_size
-        else:
-            lib = None
-            fnames = None
+        ffi1 = FFI()
+        ffi1.cdef("""
+            static const int Gofs_y, Galign, Gsize;
+            struct s1 *try_with_value(int fieldnum, long long value);
+        """)
+        fnames = [name for name, cfield in ctype.fields
+                       if name and cfield.bitsize > 0]
+        setters = ['case %d: s.%s = value; break;' % iname
+                   for iname in enumerate(fnames)]
+        lib = ffi1.verify("""
+            struct s1 { %s };
+            struct sa { char a; struct s1 b; };
+            #define Gofs_y  offsetof(struct s1, y)
+            #define Galign  offsetof(struct sa, b)
+            #define Gsize   sizeof(struct s1)
+            struct s1 *try_with_value(int fieldnum, long long value)
+            {
+                static struct s1 s;
+                memset(&s, 0, sizeof(s));
+                switch (fieldnum) { %s }
+                return &s;
+            }
+        """ % (source, ' '.join(setters)))
+        assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (
+            expected_ofs_y, expected_align, expected_size)
         # the real test follows
         assert ffi.offsetof("struct s1", "y") == expected_ofs_y
         assert ffi.alignof("struct s1") == expected_align
@@ -98,10 +93,9 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
-        if lib is not None:
-            t = lib.try_with_value(fnames.index(name), value)
-            raw2 = ffi.buffer(t, len(raw1))[:]
-            assert raw1 == raw2
+        t = lib.try_with_value(fnames.index(name), value)
+        raw2 = ffi.buffer(t, len(raw1))[:]
+        assert raw1 == raw2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)

From noreply at buildbot.pypy.org  Sat Jun  1 11:48:17 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Sat,  1 Jun 2013 11:48:17 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: (tfel,
 lwassermann): first iteration of working, exposed functions
Message-ID: <20130601094817.391951C0196@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r430:554141f002ff
Date: 2013-05-31 17:48 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/554141f002ff/

Log:	(tfel, lwassermann): first iteration of working, exposed functions

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
new file mode 100644
--- /dev/null
+++ b/spyvm/interpreter_proxy.py
@@ -0,0 +1,54 @@
+# struct VirtualMachine* sqGetInterpreterProxy(void);
+
+# typedef struct VirtualMachine {
+# 	sqInt (*minorVersion)(void);
+# } VirtualMachine;
+
+# Loading a Plugin:
+# 	plugin setInterpreter: proxy.
+# 	(plugin respondsTo: #initialiseModule) ifTrue:[plugin initialiseModule].
+# 	plugin perform: primitiveName asSymbol.
+from spyvm import error
+
+from rpython.rlib.entrypoint import entrypoint
+from rpython.rtyper.annlowlevel import llhelper
+from rpython.rlib.exports import export_struct
+from rpython.rtyper.lltypesystem.lltype import FuncType, Struct, Ptr
+from rpython.rtyper.lltypesystem import lltype
+
+sqInt = lltype.Signed
+sqLong = lltype.SignedLongLong
+
+minorVFTP = Ptr(FuncType([], sqInt))
+
+VirtualMachine = Struct("VirtualMachine",
+					("minorVersion", minorVFTP)
+				)
+VMPtr = Ptr(VirtualMachine)
+# export_struct("VirtualMachine", VirtualMachine)
+
+ at entrypoint('main', [], c_name='sqGetInterpreterProxy')
+def sqGetInterpreterProxy():
+	if not InterpreterProxy.vm_initialized:
+		vm_proxy = lltype.malloc(VirtualMachine, flavor='raw')
+		vm_proxy.minorVersion = llhelper(minorVFTP, minorVersion)
+		InterpreterProxy.vm_proxy = vm_proxy
+		InterpreterProxy.vm_initialized = True
+	return InterpreterProxy.vm_proxy
+
+def minorVersion():
+	return 1
+
+
+class _InterpreterProxy(object):
+	_immutable_fields_ = ['vm_initialized?']
+
+	def __init__(self):
+		self.vm_proxy = lltype.nullptr(VMPtr.TO)
+		self.vm_initialized = False
+
+	def call(self, signature, interp, s_frame, argcount, s_method):
+		print "Hello World..."
+		raise error.Exit("External Call")
+
+InterpreterProxy = _InterpreterProxy()

From noreply at buildbot.pypy.org  Sat Jun  1 11:48:18 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Sat,  1 Jun 2013 11:48:18 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added decorator for
	interpreterProxy functions
Message-ID: <20130601094818.68E881C0690@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r431:63aa3c3de920
Date: 2013-05-31 19:30 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/63aa3c3de920/

Log:	added decorator for interpreterProxy functions

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -1,54 +1,74 @@
 # struct VirtualMachine* sqGetInterpreterProxy(void);
 
 # typedef struct VirtualMachine {
-# 	sqInt (*minorVersion)(void);
+#   sqInt (*minorVersion)(void);
 # } VirtualMachine;
 
 # Loading a Plugin:
-# 	plugin setInterpreter: proxy.
-# 	(plugin respondsTo: #initialiseModule) ifTrue:[plugin initialiseModule].
-# 	plugin perform: primitiveName asSymbol.
+#   plugin setInterpreter: proxy.
+#   (plugin respondsTo: #initialiseModule) ifTrue:[plugin initialiseModule].
+#   plugin perform: primitiveName asSymbol.
 from spyvm import error
 
 from rpython.rlib.entrypoint import entrypoint
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.exports import export_struct
-from rpython.rtyper.lltypesystem.lltype import FuncType, Struct, Ptr
+from rpython.rtyper.lltypesystem.lltype import FuncType, Ptr
 from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.unroll import unrolling_iterable
 
 sqInt = lltype.Signed
 sqLong = lltype.SignedLongLong
 
-minorVFTP = Ptr(FuncType([], sqInt))
+major = minor = 0
+functions = []
 
-VirtualMachine = Struct("VirtualMachine",
-					("minorVersion", minorVFTP)
-				)
+def expose_on_virtual_machine_proxy(signature, minor=0, major=1):
+    f_ptr = Ptr(signature)
+    if minor < minor:
+        minor = minor
+    if major < major:
+        major = major
+    def decorator(func):
+        functions.append((func.func_name, f_ptr, func))
+        return func
+    return decorator
+
+ at expose_on_virtual_machine_proxy(FuncType([], sqInt))
+def minorVersion():
+    return minor
+
+ at expose_on_virtual_machine_proxy(FuncType([], sqInt))
+def majorVersion():
+    return major
+
+VirtualMachine = lltype.Struct("VirtualMachine",
+        *map(lambda x: (x[0], x[1]), functions))
 VMPtr = Ptr(VirtualMachine)
-# export_struct("VirtualMachine", VirtualMachine)
+
+proxy_functions = unrolling_iterable(functions)
 
 @entrypoint('main', [], c_name='sqGetInterpreterProxy')
 def sqGetInterpreterProxy():
-	if not InterpreterProxy.vm_initialized:
-		vm_proxy = lltype.malloc(VirtualMachine, flavor='raw')
-		vm_proxy.minorVersion = llhelper(minorVFTP, minorVersion)
-		InterpreterProxy.vm_proxy = vm_proxy
-		InterpreterProxy.vm_initialized = True
-	return InterpreterProxy.vm_proxy
+    if not InterpreterProxy.vm_initialized:
+        vm_proxy = lltype.malloc(VirtualMachine, flavor='raw')
+        for func_name, signature, func in proxy_functions:
+            setattr(vm_proxy, func_name, llhelper(signature, func))
+        InterpreterProxy.vm_proxy = vm_proxy
+        InterpreterProxy.vm_initialized = True
+    return InterpreterProxy.vm_proxy
 
-def minorVersion():
-	return 1
-
+# export_struct("VirtualMachine", VirtualMachine)
 
 class _InterpreterProxy(object):
-	_immutable_fields_ = ['vm_initialized?']
+    _immutable_fields_ = ['vm_initialized?']
 
-	def __init__(self):
-		self.vm_proxy = lltype.nullptr(VMPtr.TO)
-		self.vm_initialized = False
+    def __init__(self):
+        self.vm_proxy = lltype.nullptr(VMPtr.TO)
+        self.vm_initialized = False
 
-	def call(self, signature, interp, s_frame, argcount, s_method):
-		print "Hello World..."
-		raise error.Exit("External Call")
+    def call(self, signature, interp, s_frame, argcount, s_method):
+        print "Hello World..."
+        raise error.Exit("External Call")
 
 InterpreterProxy = _InterpreterProxy()
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -850,6 +850,9 @@
     elif signature[0] == "VMDebugging":
         from spyvm.plugins.vmdebugging import DebuggingPlugin
         return DebuggingPlugin.call(signature[1], interp, s_frame, argcount, s_method)
+    else:
+        from spyvm.interpreter_proxy import InterpreterProxy
+        return InterpreterProxy.call(signature, interp, s_frame, argcount, s_method)
     raise PrimitiveFailedError
 
 # ___________________________________________________________________________
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -7,6 +7,7 @@
 from spyvm import model, interpreter, squeakimage, objspace, wrapper,\
     error, shadow
 from spyvm.tool.analyseimage import create_image
+from spyvm.interpreter_proxy import VirtualMachine
 
 
 def _run_benchmark(interp, number, benchmark):

From noreply at buildbot.pypy.org  Sat Jun  1 11:48:19 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Sat,  1 Jun 2013 11:48:19 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added hint to the
 VirtualMachine, to have it's field names preserved
Message-ID: <20130601094819.8DA871C0196@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r432:19f938127015
Date: 2013-05-31 20:59 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/19f938127015/

Log:	added hint to the VirtualMachine, to have it's field names preserved

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -14,11 +14,11 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.exports import export_struct
 from rpython.rtyper.lltypesystem.lltype import FuncType, Ptr
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.unroll import unrolling_iterable
 
-sqInt = lltype.Signed
-sqLong = lltype.SignedLongLong
+sqInt = rffi.INT
+sqLong = rffi.LONG
 
 major = minor = 0
 functions = []
@@ -30,7 +30,7 @@
     if major < major:
         major = major
     def decorator(func):
-        functions.append((func.func_name, f_ptr, func))
+        functions.append(("c_" + func.func_name, f_ptr, func))
         return func
     return decorator
 
@@ -43,7 +43,8 @@
     return major
 
 VirtualMachine = lltype.Struct("VirtualMachine",
-        *map(lambda x: (x[0], x[1]), functions))
+        *map(lambda x: (x[0], x[1]), functions),
+        hints={'c_name': 'VirtualMachine'})
 VMPtr = Ptr(VirtualMachine)
 
 proxy_functions = unrolling_iterable(functions)
@@ -58,7 +59,7 @@
         InterpreterProxy.vm_initialized = True
     return InterpreterProxy.vm_proxy
 
-# export_struct("VirtualMachine", VirtualMachine)
+# rffi.llexternal is supposed to represent c-functions.
 
 class _InterpreterProxy(object):
     _immutable_fields_ = ['vm_initialized?']

From noreply at buildbot.pypy.org  Sat Jun  1 11:48:20 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Sat,  1 Jun 2013 11:48:20 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: renamed
 interpreter_proxy.InterpreterProxy to IProxy
Message-ID: <20130601094820.A0FC61C0196@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r433:03dc3c9bbf83
Date: 2013-05-31 22:40 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/03dc3c9bbf83/

Log:	renamed interpreter_proxy.InterpreterProxy to IProxy added 13
	interpreterProxy functions, oop & Array-result handling is only
	stubed

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -8,8 +8,6 @@
 #   plugin setInterpreter: proxy.
 #   (plugin respondsTo: #initialiseModule) ifTrue:[plugin initialiseModule].
 #   plugin perform: primitiveName asSymbol.
-from spyvm import error
-
 from rpython.rlib.entrypoint import entrypoint
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.exports import export_struct
@@ -17,8 +15,11 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.unroll import unrolling_iterable
 
+from spyvm import error, model
+
 sqInt = rffi.INT
 sqLong = rffi.LONG
+sqDouble = rffi.DOUBLE
 
 major = minor = 0
 functions = []
@@ -42,6 +43,109 @@
 def majorVersion():
     return major
 
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def pop(nItems):
+    IProxy.s_frame.pop_n(nItems)
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt, sqInt], sqInt))
+def popthenPush(nItems, oop):
+    s_frame = IProxy.s_frame
+    s_frame.pop_n(nItems)
+    s_frame.push(IProxy.oop_to_object(oop))
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def push(oop):
+    s_frame = IProxy.s_frame
+    s_frame.push(IProxy.oop_to_object(oop))
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def pushBool(trueOrFalse):
+    s_frame = IProxy.s_frame
+    if trueOrFalse:
+        s_frame.push(IProxy.interp.space.w_true)
+    else:
+        s_frame.push(IProxy.interp.space.w_false)
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqDouble], sqInt))
+def pushFloat(f):
+    s_frame = IProxy.s_frame
+    s_frame.push(IProxy.space.wrap_float(f))
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def pushInteger(n):
+    s_frame = IProxy.s_frame
+    s_frame.push(IProxy.space.wrap_int(n))
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqDouble))
+def stackFloatValue(offset):
+    s_frame = IProxy.s_frame
+    f = s_frame.peek(offset)
+    if isinstance(f, model.W_Float):
+        return f.value
+    else:
+        IProxy.successFlag = False
+        return 0.0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def stackIntegerValue(offset):
+    s_frame = IProxy.s_frame
+    n = s_frame.peek(offset)
+    try:
+        return IProxy.space.unwrap_int(n)
+    except error.PrimitiveFailedError:
+        IProxy.successFlag = False
+        return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def stackObjectValue(offset):
+    s_frame = IProxy.s_frame
+    w_object = s_frame.peek(offset)
+    if not isinstance(w_object, model.W_SmallInteger):
+        return IProxy.object_to_oop(w_object)
+    IProxy.successFlag = False
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def stackValue(offset):
+    s_frame = IProxy.s_frame
+    return IProxy.object_to_oop(s_frame.peek(offset))
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def argumentCountOf(methodOOP):
+    w_method = IProxy.oop_to_object(methodOOP)
+    if isinstance(w_method, model.W_CompiledMethod):
+        return w_method.argsize
+    IProxy.successFlag = False
+    return 0
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], Ptr(lltype.Array(sqInt))))
+def arrayValueOf(oop):
+    w_array = IProxy.oop_to_object(oop)
+    if isinstance(w_array, model.W_WordsObject) or isinstance(w_array, model.W_BytesObject):
+        raise NotImplementedError
+    IProxy.successFlag = False
+    return []
+
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+def byteSizeOf(oop):
+    w_object = IProxy.oop_to_object(oop)
+    s_class = w_object.shadow_of_my_class(IProxy.space)
+    size = s_class.instsize()
+    if s_class.isvariable():
+        size += w_object.primsize(IProxy.space)
+    if isinstance(w_object, model.W_BytesObject):
+        size *= size * 4
+    return IProxy.space.wrap_int(size)
+
+
+# ##############################################################################
+
 VirtualMachine = lltype.Struct("VirtualMachine",
         *map(lambda x: (x[0], x[1]), functions),
         hints={'c_name': 'VirtualMachine'})
@@ -51,13 +155,13 @@
 
 @entrypoint('main', [], c_name='sqGetInterpreterProxy')
 def sqGetInterpreterProxy():
-    if not InterpreterProxy.vm_initialized:
+    if not IProxy.vm_initialized:
         vm_proxy = lltype.malloc(VirtualMachine, flavor='raw')
         for func_name, signature, func in proxy_functions:
             setattr(vm_proxy, func_name, llhelper(signature, func))
-        InterpreterProxy.vm_proxy = vm_proxy
-        InterpreterProxy.vm_initialized = True
-    return InterpreterProxy.vm_proxy
+        IProxy.vm_proxy = vm_proxy
+        IProxy.vm_initialized = True
+    return IProxy.vm_proxy
 
 # rffi.llexternal is supposed to represent c-functions.
 
@@ -67,9 +171,31 @@
     def __init__(self):
         self.vm_proxy = lltype.nullptr(VMPtr.TO)
         self.vm_initialized = False
+        self.reset()
+
+    def reset(self):
+        self.interp = None
+        self.s_frame = None
+        self.argcount = 0
+        self.s_method = None
+        self.successFlag = True
 
     def call(self, signature, interp, s_frame, argcount, s_method):
-        print "Hello World..."
-        raise error.Exit("External Call")
+        self.interp = interp
+        self.s_frame = s_frame
+        self.argcount = argcount
+        self.s_method = s_method
+        self.space = interp.space
+        try:
+            print "Hello World..."
+            raise error.Exit("External Call")
+        finally:
+            self.reset()
 
-InterpreterProxy = _InterpreterProxy()
+    def oop_to_object(self, oop):
+        return self.interp.space.w_nil
+
+    def object_to_oop(self, oop):
+        return 0
+
+IProxy = _InterpreterProxy()
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -851,8 +851,8 @@
         from spyvm.plugins.vmdebugging import DebuggingPlugin
         return DebuggingPlugin.call(signature[1], interp, s_frame, argcount, s_method)
     else:
-        from spyvm.interpreter_proxy import InterpreterProxy
-        return InterpreterProxy.call(signature, interp, s_frame, argcount, s_method)
+        from spyvm.interpreter_proxy import IProxy
+        return IProxy.call(signature, interp, s_frame, argcount, s_method)
     raise PrimitiveFailedError
 
 # ___________________________________________________________________________

From noreply at buildbot.pypy.org  Sat Jun  1 11:48:21 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Sat,  1 Jun 2013 11:48:21 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed two translation caveats
Message-ID: <20130601094821.AC5201C0196@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r434:85ca24ecf5fb
Date: 2013-06-01 11:48 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/85ca24ecf5fb/

Log:	fixed two translation caveats

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -20,6 +20,7 @@
 sqInt = rffi.INT
 sqLong = rffi.LONG
 sqDouble = rffi.DOUBLE
+sqIntArray = rffi.CArray(sqInt)
 
 major = minor = 0
 functions = []
@@ -124,13 +125,13 @@
     IProxy.successFlag = False
     return 0
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], Ptr(lltype.Array(sqInt))))
+ at expose_on_virtual_machine_proxy(FuncType([sqInt], Ptr(sqIntArray)))
 def arrayValueOf(oop):
     w_array = IProxy.oop_to_object(oop)
     if isinstance(w_array, model.W_WordsObject) or isinstance(w_array, model.W_BytesObject):
         raise NotImplementedError
     IProxy.successFlag = False
-    return []
+    return rffi.cast(Ptr(sqIntArray), 0)
 
 @expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
 def byteSizeOf(oop):
@@ -140,8 +141,8 @@
     if s_class.isvariable():
         size += w_object.primsize(IProxy.space)
     if isinstance(w_object, model.W_BytesObject):
-        size *= size * 4
-    return IProxy.space.wrap_int(size)
+        size *= 4
+    return size
 
 
 # ##############################################################################

From noreply at buildbot.pypy.org  Sat Jun  1 11:51:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 11:51:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Attempt to fix issue #1506: change back
 the config var 'SO' to be the regular
Message-ID: <20130601095132.C4D1E1C0196@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64698:722471a15693
Date: 2013-06-01 11:50 +0200
http://bitbucket.org/pypy/pypy/changeset/722471a15693/

Log:	Attempt to fix issue #1506: change back the config var 'SO' to be
	the regular so extension for that platforms (e.g. ".so"). Instead,
	use the extended naming ".pypy-20.so" only in build_ext.

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -58,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -80,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars

From noreply at buildbot.pypy.org  Sat Jun  1 12:10:53 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 12:10:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Yay,
 found the source of one occasional crash we get on buildbot.
Message-ID: <20130601101053.C7C0D1C0196@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64699:fe0866846c4b
Date: 2013-06-01 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/fe0866846c4b/

Log:	Yay, found the source of one occasional crash we get on buildbot.

diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py
--- a/rpython/translator/c/test/test_genc.py
+++ b/rpython/translator/c/test/test_genc.py
@@ -37,7 +37,7 @@
     if isinstance(v, float):
         from rpython.rlib.rfloat import formatd, DTSF_ADD_DOT_0
         return formatd(v, 'r', 0, DTSF_ADD_DOT_0)
-    return v
+    return str(v)   # always return a string, to get consistent types
 
 def parse_longlong(a):
     p0, p1 = a.split(":")
@@ -205,6 +205,28 @@
     py.test.raises(Exception, f1, "world")  # check that it's really typed
 
 
+def test_int_becomes_float():
+    # used to crash "very often": the long chain of mangle() calls end
+    # up converting the return value of f() from an int to a float, but
+    # if blocks are followed in random order by the annotator, it will
+    # very likely first follow the call to llrepr_out() done after the
+    # call to f(), getting an int first (and a float only later).
+    @specialize.arg(1)
+    def mangle(x, chain):
+        if chain:
+            return mangle(x, chain[1:])
+        return x - 0.5
+    def f(x):
+        if x > 10:
+            x = mangle(x, (1,1,1,1,1,1,1,1,1,1))
+        return x + 1
+
+    f1 = compile(f, [int])
+
+    assert f1(5) == 6
+    assert f1(12) == 12.5
+
+
 def test_string_arg():
     def f(s):
         total = 0

From noreply at buildbot.pypy.org  Sat Jun  1 12:13:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 12:13:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: ah,
 it doesn't fail on Linux64.  that's why it was happily merged in
Message-ID: <20130601101300.B50F01C0196@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64700:aae73212c3d2
Date: 2013-06-01 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/aae73212c3d2/

Log:	ah, it doesn't fail on Linux64. that's why it was happily merged in

diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -24,4 +24,4 @@
 
     def setup_class(cls):
         import py
-        py.test.xfail("FIXME: dtype('int32') == dtype('int32') fails")
+        py.test.xfail("FIXME: dtype('int32') == dtype('int32') fails (but only on 32-bit?)")

From noreply at buildbot.pypy.org  Sat Jun  1 13:07:50 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 13:07:50 +0200 (CEST)
Subject: [pypy-commit] cffi default: MSVC-style bitfields, first attempt
Message-ID: <20130601110750.A092D1C1128@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1264:4e06afeabe1c
Date: 2013-06-01 13:04 +0200
http://bitbucket.org/cffi/cffi/changeset/4e06afeabe1c/

Log:	MSVC-style bitfields, first attempt

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3424,6 +3424,8 @@
     return cf;   /* borrowed reference */
 }
 
+#define SF_MSVC_BITFIELDS 1
+
 static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args)
 {
     CTypeDescrObject *ct;
@@ -3433,11 +3435,17 @@
     Py_ssize_t totalsize = -1;
     int totalalignment = -1;
     CFieldObject **previous;
-
-    if (!PyArg_ParseTuple(args, "O!O!|Oni:complete_struct_or_union",
+    int prev_bitfield_size, prev_bitfield_free;
+#ifdef MS_WIN32
+    int sflags = SF_MSVC_BITFIELDS;
+#else
+    int sflags = 0;
+#endif
+
+    if (!PyArg_ParseTuple(args, "O!O!|Onii:complete_struct_or_union",
                           &CTypeDescr_Type, &ct,
                           &PyList_Type, &fields,
-                          &ignored, &totalsize, &totalalignment))
+                          &ignored, &totalsize, &totalalignment, &sflags))
         return NULL;
 
     if ((ct->ct_flags & (CT_STRUCT|CT_IS_OPAQUE)) ==
@@ -3457,6 +3465,8 @@
     alignment = 1;
     boffset = 0;         /* this number is in *bits*, not bytes! */
     boffsetmax = 0;      /* the maximum value of boffset, in bits too */
+    prev_bitfield_size = 0;
+    prev_bitfield_free = 0;
     nb_fields = PyList_GET_SIZE(fields);
     interned_fields = PyDict_New();
     if (interned_fields == NULL)
@@ -3543,6 +3553,7 @@
                 previous = &(*previous)->cf_next;
             }
             boffset += ftype->ct_size * 8;
+            prev_bitfield_size = 0;
         }
         else {
             /* this is the case of a bitfield */
@@ -3592,29 +3603,59 @@
                     assert(boffset < field_offset_bytes * 8);
                 }
                 boffset = field_offset_bytes * 8;   /* the only effect */
+                prev_bitfield_size = 0;
             }
             else {
-                /* Can the field start at the offset given by 'boffset'?  It
-                   can if it would entirely fit into an aligned ftype field. */
-                bits_already_occupied = boffset - (field_offset_bytes * 8);
-
-                if (bits_already_occupied + fbitsize > 8 * ftype->ct_size) {
-                    /* it would not fit, we need to start at the next
-                       allowed position */
-                    field_offset_bytes += falign;
-                    assert(boffset < field_offset_bytes * 8);
-                    boffset = field_offset_bytes * 8;
-                    bitshift = 0;
+                if (!(sflags & SF_MSVC_BITFIELDS)) {
+                    /* GCC's algorithm */
+
+                    /* Can the field start at the offset given by 'boffset'?  It
+                       can if it would entirely fit into an aligned ftype field. */
+                    bits_already_occupied = boffset - (field_offset_bytes * 8);
+
+                    if (bits_already_occupied + fbitsize > 8 * ftype->ct_size) {
+                        /* it would not fit, we need to start at the next
+                           allowed position */
+                        field_offset_bytes += falign;
+                        assert(boffset < field_offset_bytes * 8);
+                        boffset = field_offset_bytes * 8;
+                        bitshift = 0;
+                    }
+                    else {
+                        bitshift = bits_already_occupied;
+                        assert(bitshift >= 0);
+                    }
+                    boffset += fbitsize;
                 }
-                else
-                    bitshift = bits_already_occupied;
+                else {
+                    /* MSVC's algorithm */
+
+                    /* A bitfield is considered as taking the full width
+                       of their declared type.  It can share some bits
+                       with the previous field only if it was also a
+                       bitfield and used a type of the same size. */
+                    if (prev_bitfield_size == ftype->ct_size &&
+                        prev_bitfield_free >= fbitsize) {
+                        /* yes: reuse */
+                        bitshift = 8 * prev_bitfield_size - prev_bitfield_free;
+                    }
+                    else {
+                        /* no: start a new full field */
+                        boffset = (boffset + falign*8-1) & ~(falign*8-1); /*align*/
+                        boffset += ftype->ct_size * 8;
+                        bitshift = 0;
+                        prev_bitfield_size = ftype->ct_size;
+                        prev_bitfield_free = 8 * prev_bitfield_size;
+                    }
+                    prev_bitfield_free -= fbitsize;
+                    field_offset_bytes = boffset / 8 - ftype->ct_size;
+                }
 
                 *previous = _add_field(interned_fields, fname, ftype,
                                        field_offset_bytes, bitshift, fbitsize);
                 if (*previous == NULL)
                     goto error;
                 previous = &(*previous)->cf_next;
-                boffset += fbitsize;
             }
         }
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2768,22 +2768,28 @@
     assert wr() is None
     py.test.raises(RuntimeError, from_handle, cast(BCharP, 0))
 
-def test_bitfield_as_gcc():
+def _test_bitfield_details(flag):
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
+    BUInt = new_primitive_type("unsigned int")
     BStruct = new_struct_type("foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
-                                       ('b', BInt, 9),
-                                       ('c', BChar, -1)])
-    assert typeoffsetof(BStruct, 'c') == (BChar, 3)
-    assert sizeof(BStruct) == 4
+                                       ('b1', BInt, 9),
+                                       ('b2', BUInt, 7),
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 3)
+        assert sizeof(BStruct) == 4
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 8)
+        assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
-                                       ('c', BChar, -1)])
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
     assert typeoffsetof(BStruct, 'c') == (BChar, 4)
     assert sizeof(BStruct) == 5
     assert alignof(BStruct) == 1
@@ -2792,12 +2798,19 @@
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
-                                       ('c', BChar, -1)])
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
     assert typeoffsetof(BStruct, 'c') == (BChar, 4)
     assert sizeof(BStruct) == 5
     assert alignof(BStruct) == 1
 
 
+def test_bitfield_as_gcc():
+    _test_bitfield_details(flag=0)
+
+def test_bitfield_as_msvc():
+    _test_bitfield_details(flag=1)
+
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "0.7"

From noreply at buildbot.pypy.org  Sat Jun  1 13:07:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 13:07:51 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fix this test for MSVC
Message-ID: <20130601110751.B70161C1128@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1265:110b6b1931df
Date: 2013-06-01 13:07 +0200
http://bitbucket.org/cffi/cffi/changeset/110b6b1931df/

Log:	Fix this test for MSVC

diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py
--- a/testing/test_ffi_backend.py
+++ b/testing/test_ffi_backend.py
@@ -40,6 +40,9 @@
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
+        # NOTE: 'expected_*' is the numbers expected from GCC.
+        # The numbers expected from MSVC are not explicitly written
+        # in this file, and will just be taken from the compiler.
         ffi = FFI()
         ffi.cdef("struct s1 { %s };" % source)
         ctype = ffi.typeof("struct s1")
@@ -67,8 +70,13 @@
                 return &s;
             }
         """ % (source, ' '.join(setters)))
-        assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (
-            expected_ofs_y, expected_align, expected_size)
+        if sys.platform == 'win32':
+            expected_ofs_y = lib.Gofs_y
+            expected_align = lib.Galign
+            expected_size  = lib.Gsize
+        else:
+            assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (
+                expected_ofs_y, expected_align, expected_size)
         # the real test follows
         assert ffi.offsetof("struct s1", "y") == expected_ofs_y
         assert ffi.alignof("struct s1") == expected_align

From noreply at buildbot.pypy.org  Sat Jun  1 13:34:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 13:34:45 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fix MSVC bitfields in all tested cases.
Message-ID: <20130601113445.4EBCB1C0690@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1266:263caa88878e
Date: 2013-06-01 13:34 +0200
http://bitbucket.org/cffi/cffi/changeset/263caa88878e/

Log:	Fix MSVC bitfields in all tested cases.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3477,7 +3477,7 @@
     for (i=0; i<nb_fields; i++) {
         PyObject *fname;
         CTypeDescrObject *ftype;
-        int fbitsize = -1, falign, foffset = -1;
+        int fbitsize = -1, falign, do_align, foffset = -1;
 
         if (!PyArg_ParseTuple(PyList_GET_ITEM(fields, i), "O!O!|ii:list item",
                               &PyText_Type, &fname,
@@ -3501,7 +3501,19 @@
         falign = get_alignment(ftype);
         if (falign < 0)
             goto error;
-        if (alignment < falign && (fbitsize < 0 || PyText_GetSize(fname) > 0))
+
+        do_align = 1;
+        if (fbitsize >= 0) {
+            if (!(sflags & SF_MSVC_BITFIELDS)) {
+                /* GCC: anonymous bitfields (of any size) don't cause alignment */
+                do_align = PyText_GetSize(fname) > 0;
+            }
+            else {
+                /* MSVC: zero-sized bitfields don't cause alignment */
+                do_align = fbitsize > 0;
+            }
+        }
+        if (alignment < falign && do_align)
             alignment = falign;
 
         if (fbitsize < 0) {
@@ -3598,11 +3610,23 @@
                                  "field '%s.%s' is declared with :0",
                                  ct->ct_name, PyText_AS_UTF8(fname));
                 }
-                if (boffset > field_offset_bytes * 8) {
-                    field_offset_bytes += falign;
-                    assert(boffset < field_offset_bytes * 8);
+                if (!(sflags & SF_MSVC_BITFIELDS)) {
+                    /* GCC's notion of "ftype :0;" */
+
+                    /* pad boffset to a value aligned for "ftype" */
+                    if (boffset > field_offset_bytes * 8) {
+                        field_offset_bytes += falign;
+                        assert(boffset < field_offset_bytes * 8);
+                    }
+                    boffset = field_offset_bytes * 8;
                 }
-                boffset = field_offset_bytes * 8;   /* the only effect */
+                else {
+                    /* MSVC's notion of "ftype :0;" */
+
+                    /* Mostly ignored.  It seems they only serve as
+                       separator between other bitfields, to force them
+                       into separate words. */
+                }
                 prev_bitfield_size = 0;
             }
             else {
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2791,16 +2791,24 @@
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
     assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
-    assert alignof(BStruct) == 1
+    if flag == 0:   # gcc
+        assert sizeof(BStruct) == 5
+        assert alignof(BStruct) == 1
+    else:           # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 5
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 1)
+        assert sizeof(BStruct) == 2
     assert alignof(BStruct) == 1
 
 
diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py
--- a/testing/test_ffi_backend.py
+++ b/testing/test_ffi_backend.py
@@ -137,6 +137,7 @@
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
         self.check("char x; int :0; char y;", 4, 1, 5)
+        self.check("char x; int :0; int :0; char y;", 4, 1, 5)
         self.check("char x; long long :0; char y;", L, 1, L + 1)
         self.check("short x, y; int :0; int :0;", 2, 2, 4)
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)

From noreply at buildbot.pypy.org  Sat Jun  1 13:59:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 13:59:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update to cffi/263caa88878e
Message-ID: <20130601115904.1FF901C0690@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64701:7b2e8cc68c61
Date: 2013-06-01 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/7b2e8cc68c61/

Log:	Update to cffi/263caa88878e

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -1,3 +1,4 @@
+import sys
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 
@@ -113,6 +114,14 @@
 
 # ____________________________________________________________
 
+SF_MSVC_BITFIELDS = 1
+
+if sys.platform == 'win32':
+    DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+else:
+    DEFAULT_SFLAGS = 0
+
+
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -121,9 +130,11 @@
 def new_union_type(space, name):
     return ctypestruct.W_CTypeUnion(space, name)
 
- at unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int)
+ at unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
+             sflags=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1):
+                             totalsize=-1, totalalignment=-1,
+                             sflags=DEFAULT_SFLAGS):
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise OperationError(space.w_TypeError,
@@ -134,6 +145,8 @@
     alignment = 1
     boffset = 0         # this number is in *bits*, not bytes!
     boffsetmax = 0      # the maximum value of boffset, in bits too
+    prev_bitfield_size = 0
+    prev_bitfield_free = 0
     fields_w = space.listview(w_fields)
     fields_list = []
     fields_dict = {}
@@ -166,7 +179,15 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield
         falign = ftype.alignof()
-        if alignment < falign and (fbitsize < 0 or fname != ''):
+        do_align = True
+        if fbitsize >= 0:
+            if (sflags & SF_MSVC_BITFIELDS) == 0:
+                # GCC: anonymous bitfields (of any size) don't cause alignment
+                do_align = (fname != '')
+            else:
+                # MSVC: zero-sized bitfields don't cause alignment
+                do_align = (fbitsize > 0)
+        if alignment < falign and do_align:
             alignment = falign
         #
         if fbitsize < 0:
@@ -208,6 +229,7 @@
                 fields_dict[fname] = fld
 
             boffset += ftype.size * 8
+            prev_bitfield_size = 0
 
         else:
             # this is the case of a bitfield
@@ -243,31 +265,67 @@
                     raise operationerrfmt(space.w_TypeError,
                                           "field '%s.%s' is declared with :0",
                                           w_ctype.name, fname)
-                if boffset > field_offset_bytes * 8:
-                    field_offset_bytes += falign
-                    assert boffset < field_offset_bytes * 8
-                boffset = field_offset_bytes * 8
+                if (sflags & SF_MSVC_BITFIELDS) == 0:
+                    # GCC's notion of "ftype :0;"
+                    # pad boffset to a value aligned for "ftype"
+                    if boffset > field_offset_bytes * 8:
+                        field_offset_bytes += falign
+                        assert boffset < field_offset_bytes * 8
+                    boffset = field_offset_bytes * 8
+                else:
+                    # MSVC's notion of "ftype :0;
+                    # Mostly ignored.  It seems they only serve as
+                    # separator between other bitfields, to force them
+                    # into separate words.
+                    pass
+                prev_bitfield_size = 0
+
             else:
-                # Can the field start at the offset given by 'boffset'?  It
-                # can if it would entirely fit into an aligned ftype field.
-                bits_already_occupied = boffset - (field_offset_bytes * 8)
+                if (sflags & SF_MSVC_BITFIELDS) == 0:
+                    # GCC's algorithm
 
-                if bits_already_occupied + fbitsize > 8 * ftype.size:
-                    # it would not fit, we need to start at the next
-                    # allowed position
-                    field_offset_bytes += falign
-                    assert boffset < field_offset_bytes * 8
-                    boffset = field_offset_bytes * 8
-                    bitshift = 0
+                    # Can the field start at the offset given by 'boffset'?  It
+                    # can if it would entirely fit into an aligned ftype field.
+                    bits_already_occupied = boffset - (field_offset_bytes * 8)
+
+                    if bits_already_occupied + fbitsize > 8 * ftype.size:
+                        # it would not fit, we need to start at the next
+                        # allowed position
+                        field_offset_bytes += falign
+                        assert boffset < field_offset_bytes * 8
+                        boffset = field_offset_bytes * 8
+                        bitshift = 0
+                    else:
+                        bitshift = bits_already_occupied
+                        assert bitshift >= 0
+                    boffset += fbitsize
+
                 else:
-                    bitshift = bits_already_occupied
+                    # MSVC's algorithm
+
+                    # A bitfield is considered as taking the full width
+                    # of their declared type.  It can share some bits
+                    # with the previous field only if it was also a
+                    # bitfield and used a type of the same size.
+                    if (prev_bitfield_size == ftype.size and
+                        prev_bitfield_free >= fbitsize):
+                        # yes: reuse
+                        bitshift = 8 * prev_bitfield_size - prev_bitfield_free
+                    else:
+                        # no: start a new full field
+                        boffset = (boffset + falign*8-1) & ~(falign*8-1)
+                        boffset += ftype.size * 8
+                        bitshift = 0
+                        prev_bitfield_size = ftype.size
+                        prev_bitfield_free = 8 * prev_bitfield_size
+                    #
+                    prev_bitfield_free -= fbitsize
+                    field_offset_bytes = boffset / 8 - ftype.size
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
-            
-                boffset += fbitsize
 
         if boffset > boffsetmax:
             boffsetmax = boffset
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2757,36 +2757,57 @@
     assert wr() is None
     py.test.raises(RuntimeError, from_handle, cast(BCharP, 0))
 
-def test_bitfield_as_gcc():
+def _test_bitfield_details(flag):
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
+    BUInt = new_primitive_type("unsigned int")
     BStruct = new_struct_type("foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
-                                       ('b', BInt, 9),
-                                       ('c', BChar, -1)])
-    assert typeoffsetof(BStruct, 'c') == (BChar, 3)
-    assert sizeof(BStruct) == 4
+                                       ('b1', BInt, 9),
+                                       ('b2', BUInt, 7),
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 3)
+        assert sizeof(BStruct) == 4
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 8)
+        assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
-                                       ('c', BChar, -1)])
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
     assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
-    assert alignof(BStruct) == 1
+    if flag == 0:   # gcc
+        assert sizeof(BStruct) == 5
+        assert alignof(BStruct) == 1
+    else:           # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
-                                       ('c', BChar, -1)])
-    assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 5
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 1)
+        assert sizeof(BStruct) == 2
     assert alignof(BStruct) == 1
 
 
+def test_bitfield_as_gcc():
+    _test_bitfield_details(flag=0)
+
+def test_bitfield_as_msvc():
+    _test_bitfield_details(flag=1)
+
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "0.7"

From noreply at buildbot.pypy.org  Sat Jun  1 17:51:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 17:51:19 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Continue writing and finding new
 ideas. Kill the previous text for now.
Message-ID: <20130601155119.17D781C0196@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54:4a7fb6ee72c1
Date: 2013-06-01 17:50 +0200
http://bitbucket.org/pypy/stmgc/changeset/4a7fb6ee72c1/

Log:	Continue writing and finding new ideas. Kill the previous text for
	now.

diff --git a/c3/doc-stmgc.txt b/c3/doc-stmgc.txt
--- a/c3/doc-stmgc.txt
+++ b/c3/doc-stmgc.txt
@@ -4,13 +4,13 @@
 
 In this document we say "high-level object" to mean an object from the
 point of the user of the library, as opposed to an "object copy", which
-occupies the space of one allocated piece of memory.  One high-level
-object can exist in several copies simultaneously.  This concept of
-"copy" should not be confused with a "revision", which stands for a
-globally consistent copy of all objects.  One revision is the result of
-one transaction.  A program usually has one revision per thread in
-progress, plus any number of older committed revisions.  The committed
-revisions are globally ordered.
+is what occupies the space of one allocated piece of memory.  One
+high-level object can exist in several copies simultaneously.  This
+concept of "copy" should not be confused with a "revision", which stands
+for a globally consistent copy of all objects.  One revision is the
+result of one transaction.  A program usually has one revision per
+thread in progress, plus any number of older committed revisions.  The
+committed revisions are globally ordered.
 
 The object copies exist in one of three main states: they can be
 "private", "protected" or "public".  A copy is private when it belongs
@@ -37,14 +37,13 @@
 revision; and the other, private, is the current copy.  If the
 transaction aborts we can forget the private copy and reuse the previous
 protected copy.  If the transaction commits we forget the previous
-protected copy instead; then at this point all private objects become
-protected.  If the object is modified again in the near future, we reuse
-the memory that was occupied by the previous copy to store the next
-private copy.  As a result, each of these two spaces in memory can be
-young or old.  When the GC runs, if any one of these two copies is
-young, only the other copy is kept.  Similarly during major collections,
-only one copy is kept.  So objects no longer modified will eventually
-consume only one space.
+protected copy instead; then the private copy becomes protected, like
+*all* private copies at this point.  If the object is modified again in
+the near future, we reuse the memory that was occupied by the previous
+copy to store the next private copy.  As a result, each of these two
+spaces in memory can be young or old.  If an object is no longer
+modified for long enough, the next (minor or major) GC will free one of
+the two spaces it uses.
 
 The way to share data between threads goes via prebuilt objects, which
 are always public: it is their existence that gives the starting point
@@ -60,19 +59,21 @@
 copy.  Any access from a different thread will trigger "stealing", as
 explained next.
 
-2. A thread tries to access a public object but finds that another
-thread has committed changes to it (hereafter called the "foreign
-thread").  Then we "steal" the object.  It is a read-only operation
-performed by peeking on the foreign thread's data.  The operation
-involves making a duplicate of the original copy if it was in the
-foreign thread's nursery (so that no thread ever reads another thread's
-nursery, outside of "stealing").  The stolen copy, or the original
-protected copy if it was not in the nursery, is then marked as public.
-From now on nobody is allowed to change the content of this copy, and it
-becomes the current public copy.  These public copies accumulate: every
-time the same object is stolen by a different thread, a new public copy
-is made (so that unrelated threads don't have to worry about existing
-public copies being written to).
+2. When we are running a thread, it may try to access a public object
+but find that another thread (the "foreign thread") has committed
+changes to it .  Then we "steal" the object.  It is a read-only
+operation performed by peeking on the foreign thread's data.  The
+operation involves making a duplicate of the original copy, if it was in
+the foreign thread's nursery (so that no thread ever reads another
+thread's nursery, outside of "stealing").  The stolen copy, or the
+original protected copy if it was not in the nursery, is then marked as
+public.  From now on nobody is allowed to change (or free) the content
+of this copy, and it becomes the current public copy.  These public
+copies accumulate: every time the same object is stolen by a different
+thread, a new public copy is made (so that unrelated threads don't have
+to worry about existing public copies being updated).  (This chain of
+objects is freed at the next major GC, which is a stop-the-world
+operation.)
 
 3. A subtle but important point about making a public copy is about all
 the references stored in the object: if they point to other protected
@@ -83,16 +84,16 @@
 "older" public copy of a protected object (although it is not actually
 older of course).  If "we", the thread that just stole the object, then
 try to follow one of the references, we will access one of these stubs,
-and go back to point 1: we will need to steal the target object's
-protected copy.
+and go back to point 2: stealing the target object's protected copy.
 
 
-Implementation
---------------
+Read/Write Barriers
+-------------------
 
 This design is made to optimize the hopefully common case: objects we
-handle are mostly private or protected.  We can design in consequence
-the following three points:
+handle are mostly private or protected, or if they are public, they are
+mostly read-only.  We can design in consequence the following three
+points:
 
 1. the extra data stored in the objects (GC flags, and one extra word
 called `h_revision`).
@@ -102,433 +103,94 @@
 3. the read/write barriers.
 
 Point 3 is essential for performance: we want most importantly a read
-barrier that doesn't trigger for the case of reading protected and
-private objects, which is supposed to be the most common case.
-Moreover, it should also not trigger in the basic case of reading a
-never-modified public object.  There are basically three cases:
+barrier that doesn't trigger for the cases described above.  There are
+basically three cases:
 
 1. read_barrier(P) = P   [if P is directly a non-modified copy]
 
-2. read_barrier(P) = P->h_revision   [a protected copy with a private one]
+2. read_barrier(P) = P->h_revision   [protected copy -> private copy]
 
 3. all other more complex cases, handled by a call.
 
 It is possible to compress the first two cases into C code that GCC
-compiles into two or three assembler instructions, using a conditional
-move.  (Still, it is unclear so far if case 2 is worth inlining at every
-read barrier site, or should be left to the call.)
+compiles into a total of two or three assembler instructions, using a
+conditional move.  (Still, it is unclear so far if case 2 is worth
+inlining at every read barrier site, or should be left to the call.)
 
-The case of the write barrier is similar, but differs in the exact
-checks: basically case 1 is only for private objects.  This could be
+The case of the write barrier is similar, but differs in the check for
+case 1: this case is reduced to only private objects.  This could be
 done simply by checking a different GC flags.
 
+Note that this design relies on the following property: in a given copy
+of an object which was committed at revision N, all pointers points to
+copies of objects which were committed at or before revision N.  This
+property is true by construction, but we must be careful not to break it
+by "optimizing" the content of a copy.  In particular the GC, during
+both minor and major collections, has to preserve this property.  
 
 
-
-
-
-------------
-
-
-
-
-
-
-Independently, each object can be private or non-private (we used to say
-local or global).  The private objects are the ones belonging to the
-transaction currently in progress.  The non-private objects belong to
-already-committed transactions.
-
-The concepts of age (old or young) and privacy are kept separated, and
-all four combinations can exist: it allows good performance both in case
-of a lot of very short transactions and in the case of one very long
-transaction.  In the first case, we leave most young objects behind in
-the nursery, as non-private objects, without moving them (they can still
-be freed later by the next minor collection).  In the other case, over
-the course of several minor collections, we have the same behavior as a
-regular two-generational GC system which copies the surviving nursery
-objects into old ones; but these objects remain private as long as the
-transaction isn't finished.
-
-We'll also divide the non-private objects in two subcategories: "public"
-for the old non-private objects, and "protected" for the young
-non-private objects.  Protected objects, although not private, are still
-protected from direct access from other thread as long as they are in
-the nursery.
-
-
-                          non-private | private
-           +------------------------------------------------------------
-           |
-       old |         public objects   |   old private objects
-  ---------|
-           |
-     young |     [ protected objects  |  private objects  (--> grows) ]
-  (nursery)|
-
-
-Because in principle no object can be modified in-place once it has been
-committed, this limits what pointers it can contain: they must directly
-reference other objects that are not more recent than the container.
-(But an object may contain a reference to some outdated version of an
-object; then we use the normal h_revision link to find the latest
-version of the latter.)  So there are no regular pointers from a
-non-private object to a private object (which are always the most recent
-ones from the point of view of their thread).  We add another
-constraint: we don't want any public object to contain a direct pointer
-to a protected object.
-
-The only way to have a link in the "less-public" direction is via the
-h_revision field.  The multiple versions of the same higher level object
-look like this:
-
-
-    [=the static object, if it was prebuilt=]
-      |
-      | h_revision
-      |
-      `------> [=public1=]
-                 |
-                 | h_revision
-                 |
-                 `------> [=public2=]
-                            |
-                            | h_revision with bit 2 set
-                            |
-                            `------> [=protected3=]
-                                       |
-                                       | h_revision
-                                       |
-                                       `------> [=protected4=]
-
-
-The h_revision is normally a regular pointer to the next version.  But
-one of these links may go from public to protected, which is marked by
-having the bit 2 set in h_revision.  And the latest object in the chain
-has h_revision set to an odd value (bit 1 set), which gives the revision
-in which that object was committed.
-
-In addition to the diagram above, a thread may also have one private
-version (originally based on the head of the chain, but it might get
-out-of-date).  This link may be recorded "on-line" [O1] or "off-line"
-[O2], see below.  When the transaction commits, no object is copied in
-memory, but any private object becomes non-private (i.e. protected or
-public).  If the object has any previous revision, then what is so far
-the head of the chain sees its h_revision replaced with a pointer to the
-new object.  If this creates a public-to-protected link, then its bit 2
-is set.
-
-Minor collections are mostly regular first-generation collections from
-the point of view of the GC: they move all surviving nursery objects
-outside.  A minor collection always turns all protected objects into
-public ones.  (In particular, if a transaction started after the most
-recent minor collection, then there are no old private objects;
-conversely, if it started before the most recent minor collection, then
-there are no protected objects.)
-
-However another process can force a public copy of a protected object:
-"stealing".  Stealing is triggered when thread A attempts to follow the
-h_revision, finds its bit 2 set, and finds that it points outside thread
-A's own nursery --- and so it is a protected object from a foreign
-thread B.  In that case, thread A will "steal" the target object's
-latest version, making it public.  (This is done instead of simply
-waiting for thread B's next minor collection, because it can occur in an
-arbitrary amount of time: for all we know thread B may be blocked in a
-`sleep(100)`.)
-
-The diagram above becomes like this, after thread A starts with a
-pointer to "public1" and attempts to find its later version:
-
-
-      [=public1=]
-        |
-        | h_revision
-        |
-        `------> [=public2=]
-                   |
-                   | h_revision
-                   |
-                   |
-                   |        [=protected3=]
-                   |          |
-                   |          | h_revision
-                   |          |
-                   |          `------> [=protected4=]
-                   |                     .
-                   |                     . h_revision
-                   |                     .
-                   `----------------------------> [=public5=]
-
-
-with a new copy "public5" of the object being created outside the
-nursery.  Afterwards, the h_revision of "public2" points directly to
-"public5", and so any future access to the object from "public1" or
-"public2" will work directly.  (The dotted h_revision link from
-"protected4" is not actually written by the stealing thread; it is only
-recorded for the original thread to do later [O5].)
-
-The nursery objects are not written at all by the stealing process.  No
-write to nursery objects ever occurs from other threads, and the only
-reads occur during stealing.  But say we have protected objects in a
-thread A, and want to steal one of them from a thread B.  This operation
-still requires care: mostly, we have to be sure that the objects won't
-be freed concurrently by thread A while we read them in thread B.  This
-is done by acquiring thread A's collection lock from thread B.  This
-lock is also acquired by thread A itself during its own minor
-collections.  So stealing cannot occur in parallel with thread A running
-a minor collection (or thread C stealing from the same thread A), but it
-can occur in parallel with thread A's normal execution.  This compromize
-should be fine: if thread A is currently doing a minor collection, then
-thread B can as well wait a bit until it is finished and then try again:
-afterwards, all of thread A's protected objects will have become public
-anyway.
-
-                                    -+-
-
-No public object may contain a reference to a protected object.  This
-adds a problem that we ignored so far.  Fresh public objects are created
-by three processes: 1. when we commit; 2. when stealing; 3. when we do
-a minor collection.  The 3rd case is ignored here because there are no
-protected objects after a minor collection.
-
-1. When we commit, all our private objects become non-private.  In particular,
-if there are old private objects, they become public.  Of course this
-problem doesn't exist if there are no old private objects, which is the
-common case if we're doing a lot of small transactions: no object can be
-old and private if the current transaction started after the most recent
-minor collection.  The case that pauses problem here is the opposite: a
-minor collection occurred during the present transaction.  In this case
-there are no protected object so far: the nursery contains only private
-objects.
-
-So committing creates newly protected and public objects; any of these
-can contain references to any other, including "newly public -> newly
-protected" --- i.e. old -> young.  Fortunately, we keep track of such
-references anyway [O3] for the purpose of the generational garbage
-collection (by having our STM-specific write barrier work in this case
-like a traditional generational write barrier).
-
-The 2nd case is stealing: it produces a new, public revision (this is
-"public5" in the example above).  If we simply copy its content from
-"protected4" then it will likely contain references to other protected
-objects.
-
-In both cases we know exactly which objects must be fixed.  There are
-two different behaviors that can be implemented (or maybe some
-combination of both is best).  The first option is that for any
-reference that still goes to a protected object, we allocate a new
-public revision of that object, and make the reference go to that.  We
-end up with repeating the same process recursively --- this makes a new
-public copy of the protected object, which may contain more references
-to protected objects, so we make public copies of these ones too, and so
-on.  The second option is to stop this recursion: instead of allocating
-a full public revision, we allocate a public stub that contains only an
-h_revision pointing to the protected version, with the bit 2 set.  This
-works because no pointer to the stub ever escapes the STM subsystem.
-
-                                    -+-
-
-The age of an object is implicit.  Each thread has enough data to know
-if an object is young for this thread or not: whether its address falls
-inside our nursery or not.  (In practice we also need to check a small
-set of extra young objects, the ones that have been allocated outside
-the nursery.)
-
-Private objects are fully read-write but only visible from one thread.
-Public objects are visible by any thread, and are theoretically
-read-only.  Protected objects are intermediate.  In more details:
-
-~~ Private objects ~~
-
-Private objects cannot be accessed from other threads.  From their own
-thread, they are distinguished by having a particular value in
-h_revision.  We use a negative odd number.  When the transaction
-commits, we change the number that we use.  All previous private
-objects' h_revision field is no longer equal to this number, so they
-automatically become non-private (without needing to enumerate them
-all).
-
-The number is negative, which is already correct at commit time for all
-objects that are new during this transaction: a negative odd number is
-"infinitely old", i.e. older than any real revision number (a positive
-odd number).  This is fine for the first revision of any object.  For
-the objects that are not first revision (i.e. that have not been created
-during this transaction but merely marked as modified during this
-transaction), we have to replace h_revision with the real revision
-number obtained during commit.
-
-~~ Public objects ~~
-
-In public objects, h_revision is initially set as described above, to
-the revision in which the object was created (an odd number).  When one
-thread is about to commit a new revision of this public object, it
-changes atomically the h_revision field of the public object to a
-thread-specific "locked" value.  After checks to ensure that the whole
-commit is consistent, h_revision is again replaced with a pointer to the
-new revision.  If this new revision is protected, then we set the bit 2
-of the pointer, as described above.
-
-The GC flags in the header of a public object are never modified, expect
-as follows: the flag GCFLAG_PUBLIC_TO_PRIVATE is added when one thread
-makes a private copy of this public object.  As it is the only change
-that can occur --- writing a value equal to the old flags combination
-plus this particular flag --- it doesn't need special protection.  It is
-used as an optimization in the read barrier: if we have an object
-without this flag, then we don't have to look in off-line dictionaries
-[O2] to find if we made a private copy.
-
-~~ Protected objects ~~
-
-In protected objects, access from other thread is more restricted.  This
-means that we can directly use h_revision to point to a private copy as
-soon as there is one.  We need to keep somewhere else (see [O1]) the
-overwritten value of h_revision, which gives the revision number at
-which the protected object was created; it needs to be written back in
-case of a transaction abort.
-
-The GC flags in the header of a protected object may only be modified by
-the thread that protects it (not by a stealing thread).  (In fact it's
-not clear so far that they ever need to be modified at all.)
-
-                                    -+-
-
-Stealing: whenever we encounter an h_revision with the bit 2 set, we
-have to check the target object *without reading it*.  This is essential
-because the target object may belong to a different thread, and anything
-can occur concurrently (e.g. the nursery where it lives may be cleared).
-This is done by checking in which nursery the address falls: first we
-check our own nursery, and then if not, each other thread's nursery.
-
-In the latter case we enter stealing mode by acquiring the collection
-lock of the target thread.  With this lock we can then read some of the
-off-line data of the target thread.  This requires the target thread to
-guarantee that it does not change this data concurrently, but only
-when it has itself acquired the collection lock.
-
-                                    -+-
-
-Details of the data structures
+The extendable timestamp model
 ------------------------------
 
-Each thread has its own `struct tx_descriptor` structure to store its
-STM- and GC-related thread-local metadata off-line.  In particular, it
-stores all data about the STM status of public objects (e.g. whether
-they currently have a private copy).
+A public object copy is either up-to-date (no more recent committed
+copy) or outdated.  The public copies need to have globally consistent
+revision numbers (produced at commit time).  If there are several copies
+of the same object, we only need to store the revision number of the
+most recent one.  The previous copies are simply outdated and need
+instead to store a pointer to a more recent copy.
 
-The object themselves have two words each: a GC header, and the
-h_revision number/pointer.  The GC header stores the type id of the
-object, and additional GC flags.
+The important property is that committed transactions must be
+"linearized": when we look at them a posteriori, it must be as if they
+ran serially in some order.  This includes the reads done during the
+transaction: they must return data from the most recently committed copy
+of the objects (in the same order).  This is done with a shared global
+variable, the "global time", that gives the most recently committed
+revision number.  Each transaction in progress stores a temporary
+"starting time".  It is initially set to the current global time.  If,
+at the end of the transaction, all objects read during the transaction
+have a revision not greater than this starting time, then we have no
+"read-write" conflict (i.e. reads of an object that another thread has
+modified; there are also "write-write" conflicts).  An improvement over
+this basic model is that if, during the transaction, we are about to
+read a new object and detect a read-write conflict, we can try to
+"extend" the starting time to the value that is now stored in the global
+time variable.  If none of the objects that we have read previously have
+been modified in the interval, then the transaction would have given the
+same results if it had started at the new time.
 
-The rest is data structures in the `struct tx_descriptor`:
+The model described above is known in the literature as the "extendable
+timestamp" model.  We apply it for public object.  It can however be
+tweaked for protected objects.
 
 
-- [O1] protected_with_private_copy: this is a list of all protected
-  objects with a private copy.  In addition, we eagerly replace the
-  h_revision of the protected object with a pointer to the private
-  object.  It speeds up the read barrier in the common case where the
-  chain of objects ends in a protected object.  But we need to store the
-  original value of this h_revision field somewhere else; this is needed
-  to be able to restore it in case of abort, as well as when stealing,
-  when we need to know what revision number the protected object had.
-  To store this, private copies of protected objects are allocated with
-  an extra word after them.  (As the majority of the private objects
-  are expected to be fresh, rather than the copy of some older version
-  object, we only need to allocate this extra word in the minority of
-  cases.)
+Pointers and revision numbers on protected objects
+--------------------------------------------------
 
-  (Note that the private copies are necessarily young: it's not possible
-  that protected objects exist at the same time as old private objects.)
+In traditional transactional systems, we have a special case to speed up
+transactions that don't do any write; but that seems too restricted to
+be useful in PyPy.  Instead, we can have a special case for transactions
+that don't write to any *public* object.  Our assumption is that
+transactions can be anywhere from very small to very large; the small
+ones are unlikely to change any object that has been seen by another
+thread.  Moreover the previous transaction in the same thread is
+unlikely to have got one of its objects stolen.
 
-  Access pattern: [O1] is only ever accessed by the local thread.  The
-  extra word after the private copy is written once, _before_ we change
-  h_revision in the protected object; it may subsequently be read freely
-  by that thread _and_ when stealing is in progress.
+To cover this case more efficiently, we assign in theory to each
+committed transaction a pair of numbers.  The first one is the regular
+global time at which the transaction was committed.  The second one is a
+thread-local number (never actually made explicit in the code).  The
+global order of the committed transactions is given by the ordering of
+these 2-tuples.
 
+A commit with writes to public objects works as described above; it
+gives the transaction the number `(global_time, 0)`, and atomically
+increments `global_time`.
 
-- [O2] public_to_private: a dictionary that maps public objects to their
-  private version.  Public objects that may be keys in such a dictionary
-  have GCFLAG_PUBLIC_TO_PRIVATE.  This dict [O2] has the same purpose as
-  the list [O1], but it differs in the representation: the public
-  objects are seen by all threads, and so we cannot store anything
-  globally on them.
-
-  Access pattern: [O2] is only ever accessed by the local thread.
-
-
-- [O3] private_old_pointing_to_young: a list of old private objects that
-  may contain pointers to young private objects.  Used for minor
-  collections.
-
-  Access pattern: [O3] is only ever accessed by the local thread.
-
-
-- [O4] public_to_young: a list of all public objects with a
-  corresponding young object.  This list starts with the public objects
-  whose h_revision references a protected object (with bit 2 set), and
-  continues with the public objects that have a corresponding _young_
-  private object (a subset of the keys of `public_to_private`).  This is
-  useful information when doing a minor collection.  The point of
-  storing both kinds of objects in the same list is that after each
-  commit, all objects of the 2nd kind become objects of the 1st kind ---
-  which is implemented simply by moving the boundary to the end of the
-  list.
-
-  Access pattern: [O4] is only ever accessed by the local thread.
-
- 
-- [O5] stolen_objects: a list of protected objects that have been
-  stolen, together with the new public copy.  This list is written by
-  the stealing thread, and it's up to the local thread to notice that it
-  is not empty and to fix the situation.  It should be checked at the
-  latest at commit time, but also when building a private copy of an
-  existing object.  These two cases require acquiring the local
-  collection lock.
-
-  (Additionally, there is the risk of not noticing a stolen object in
-  that list, and continuing to run a transaction for a long time after
-  some other thread committed a new revision of the public object.  To
-  avoid this, we start the read/write barriers by checking if this list
-  is non-empty.  This can be done without acquiring the lock, as it is
-  not needed for correctness; so it is extremely cheap.)
-
-  Access pattern: only threads that have acquired the local collection
-  lock can access [O5] (apart from the check for non-emptiness described
-  in the previous paragraph).
-
-
-Major and minor collections
----------------------------
-
-Major collections occur to free the memory used by old objects.  For now
-we assume that major collections are rare enough, and simply synchronize
-all threads.  The thread that initiates a major collection first checks
-that each thread ran a minor collection just before (and does it on its
-behalf if not, e.g. if it is blocked in a system call).  Afterwards,
-major collection only has to deal with old objects (public or
-private-to-some-thread).
-
-As a first approximation major collection is a regular, non-concurrent,
-non-parallel GC.  (It could be made parallel relatively easily, by using
-the other waiting threads to help, rather than just have them wait.
-More subtly, in a few cases we could interrupt other threads' system
-calls to let them help, too.)
-
-The unusual characteristic of our GC is that in our case we can compress
-the h_revision chains, thus freeing more objects.  This is true for both
-minor and major collections.  During major collections, we need to keep
-alive the latest public version of a surviving object, as well as the
-private versions, if any.  The older versions can be freed.  We need a
-bit of care to adjust pointers everywhere.  Additionally, this process
-might occasionally figure out that a transaction in progress is actually
-going to abort in the future (because e.g. it has got an object in its
-read set that has a more recent committed revision).  Such transactions
-can be aborted now.
-
-Similarly, during minor collections we only need to keep the most recent
-protected revision of an object, as well as the young private version,
-if any.  An important point about minor collections is that they only
-look at their own thread's nursery, and so can occur without any
-cross-thread synchronization.  (If it is useful, we could also figure
-out a case where the current transaction is going to abort in the
-future: when objects listed as keys in `public_to_private` have become
-supersceded by a more recent commit.)
+A commit with no write to any public object produces the number
+`(start_time, N+1)`, provided that we didn't have any object stolen
+since `start_time`.  This condition is enough to guarantee that it is ok
+to linearize the transaction at `(start_time, N+1)` even though other
+threads might already have produced transactions linearized at some
+greater time.  Indeed, the fact that no object of ours was stolen
+means that no other thread's transaction depends on any object we're
+about to commit.

From noreply at buildbot.pypy.org  Sat Jun  1 20:24:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Jun 2013 20:24:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add an explicit error message when
 trying to build on Win64.
Message-ID: <20130601182441.4F6441C30D5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64702:b0201e19a481
Date: 2013-06-01 20:24 +0200
http://bitbucket.org/pypy/pypy/changeset/b0201e19a481/

Log:	Add an explicit error message when trying to build on Win64.

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -28,6 +28,8 @@
     return _get_compiler_type(cc, False)
 
 def Windows_x64(cc=None):
+    raise Exception("Win64 is not supported.  You must either build for Win32"
+                    " or contribute the missing support in PyPy.")
     return _get_compiler_type(cc, True)
     
 def _get_msvc_env(vsver, x64flag):

From noreply at buildbot.pypy.org  Sat Jun  1 21:11:02 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  1 Jun 2013 21:11:02 +0200 (CEST)
Subject: [pypy-commit] pypy win32-fixes3: merge default into branch
Message-ID: <20130601191102.C75761C1120@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: win32-fixes3
Changeset: r64703:416013f3e0de
Date: 2013-06-01 22:10 +0300
http://bitbucket.org/pypy/pypy/changeset/416013f3e0de/

Log:	merge default into branch

diff too long, truncating to 2000 out of 7875 lines

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -58,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -80,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,20 +134,22 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
+}
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
 }
 
 def getLevelName(level):
@@ -164,7 +166,7 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+    return _levelToName.get(level, ("Level %s" % level))
 
 def addLevelName(level, levelName):
     """
@@ -174,8 +176,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -183,9 +185,9 @@
     if isinstance(level, int):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
@@ -277,7 +279,7 @@
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - long(ct)) * 1000
+        self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads and thread:
             self.thread = thread.get_ident()
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -156,7 +156,7 @@
         h = klass(*args)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -187,7 +187,7 @@
     opts = cp.options(sectname)
     if "level" in opts:
         level = cp.get(sectname, "level")
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = cp.get(sectname, "handlers")
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,16 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    try:
-                        self._sock._decref_socketios()
-                    except AttributeError:
-                        pass  # bah, someone built a _fileobject manually
-                              # with some unexpected replacement of the
-                              # _socketobject class
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -65,7 +65,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
         finally:
             logging._releaseLock()
 
@@ -97,8 +98,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -156,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -701,7 +704,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
     # ----------
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -74,6 +74,9 @@
         class FFILibrary(types.ModuleType):
             _cffi_generic_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir
         library = FFILibrary("")
         #
         # finally, call the loaded_gen_xxx() functions.  This will set
@@ -168,21 +171,22 @@
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
-            if any(isinstance(type, model.StructOrUnion) for type in tp.args):
+            if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
-                for i, type in enumerate(tp.args):
-                    if isinstance(type, model.StructOrUnion):
-                        type = model.PointerType(type)
-                        indirections.append((i, type))
-                    indirect_args.append(type)
+                for i, typ in enumerate(tp.args):
+                    if isinstance(typ, model.StructOrUnion):
+                        typ = model.PointerType(typ)
+                        indirections.append((i, typ))
+                    indirect_args.append(typ)
                 tp = model.FunctionPtrType(tuple(indirect_args),
                                            tp.result, tp.ellipsis)
             BFunc = self.ffi._get_cached_btype(tp)
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
-            for i, type in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, type)
+            for i, typ in indirections:
+                newfunction = self._make_struct_wrapper(newfunction, i, typ)
         setattr(library, name, newfunction)
+        type(library)._cffi_dir.append(name)
 
     def _make_struct_wrapper(self, oldfunc, i, tp):
         backend = self.ffi._backend
@@ -390,6 +394,7 @@
         is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
         value = self._load_constant(is_int, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # enums
@@ -437,6 +442,7 @@
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
             setattr(library, enumerator, enumvalue)
+            type(library)._cffi_dir.append(enumerator)
 
     # ----------
     # macros: for now only for integers
@@ -450,6 +456,7 @@
     def _loaded_gen_macro(self, tp, name, module, library):
         value = self._load_constant(True, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # global variables
@@ -475,6 +482,7 @@
                 BArray = self.ffi._get_cached_btype(tp)
                 value = self.ffi.cast(BArray, value)
             setattr(library, name, value)
+            type(library)._cffi_dir.append(name)
             return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
@@ -486,7 +494,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
 cffimod_header = r'''
 #include <stdio.h>
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
new file mode 100644
--- /dev/null
+++ b/lib_pypy/greenlet.egg-info
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: greenlet
+Version: 0.4.0
+Summary: Lightweight in-process concurrent programming
+Home-page: https://github.com/python-greenlet/greenlet
+Author: Ralf Schmitt (for CPython), PyPy team
+Author-email: pypy-dev at python.org
+License: MIT License
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -32,11 +32,10 @@
      "rctime" , "select", "zipimport", "_lsprof",
      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
-     "thread", "itertools", "pyexpat", "_ssl", "array",
+     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
      "binascii", "_multiprocessing", '_warnings',
      "_collections", "_multibytecodec", "micronumpy", "_ffi",
-     "_continuation", "_cffi_backend", "_csv"] # "cpyext", "cppyy"]
-# disabled until problems are fixed
+     "_continuation", "_cffi_backend", "_csv", "cppyy"]
 ))
 
 translation_modules = default_modules.copy()
diff --git a/pypy/doc/rffi.rst b/pypy/doc/rffi.rst
--- a/pypy/doc/rffi.rst
+++ b/pypy/doc/rffi.rst
@@ -5,7 +5,7 @@
 Purpose
 -------
 
-This document describes an FFI for RPython language, concentrating
+This document describes an FFI for the RPython language, concentrating
 on low-level backends like C. It describes
 how to declare and call low-level (C) functions from RPython level.
 
@@ -50,7 +50,7 @@
 ------
 
 In rffi_ there are various declared types for C-structures, like CCHARP
-(char*), SIZE_T (size_t) and others. refer to file for details. 
+(char*), SIZE_T (size_t) and others. Refer to file for details. 
 Instances of non-primitive types must be alloced by hand, with call 
 to lltype.malloc, and freed by lltype.free both with keyword argument 
 flavor='raw'. There are several helpers like string -> char*
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,15 @@
 
 .. branch: remove-iter-smm
 Remove multi-methods on iterators
+
+.. branch: emit-call-x86
+.. branch: emit-call-arm
+
+.. branch: on-abort-resops
+Added list of resops to the pypyjit on_abort hook.
+
+.. branch: logging-perf
+Speeds up the stdlib logging module
+
+.. branch: operrfmt-NT
+Adds a couple convenient format specifiers to operationerrfmt
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -86,12 +86,9 @@
             args_w = space.fixedview(w_stararg)
         except OperationError, e:
             if e.match(space, space.w_TypeError):
-                w_type = space.type(w_stararg)
-                typename = w_type.getname(space)
-                raise OperationError(
+                raise operationerrfmt(
                     space.w_TypeError,
-                    space.wrap("argument after * must be "
-                               "a sequence, not %s" % (typename,)))
+                    "argument after * must be a sequence, not %T", w_stararg)
             raise
         self.arguments_w = self.arguments_w + args_w
 
@@ -116,12 +113,10 @@
                 w_keys = space.call_method(w_starstararg, "keys")
             except OperationError, e:
                 if e.match(space, space.w_AttributeError):
-                    w_type = space.type(w_starstararg)
-                    typename = w_type.getname(space)
-                    raise OperationError(
+                    raise operationerrfmt(
                         space.w_TypeError,
-                        space.wrap("argument after ** must be "
-                                   "a mapping, not %s" % (typename,)))
+                        "argument after ** must be a mapping, not %T",
+                        w_starstararg)
                 raise
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2793,8 +2793,7 @@
 
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2835,8 +2834,7 @@
 
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2881,8 +2879,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -2925,8 +2922,7 @@
 
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2971,8 +2967,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -2993,8 +2988,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3024,8 +3018,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3046,8 +3039,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3064,8 +3056,7 @@
 
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3081,8 +3072,7 @@
 
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3131,8 +3121,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3149,8 +3138,7 @@
 
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
             list_w = []
@@ -3166,8 +3154,7 @@
 
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3183,8 +3170,7 @@
 
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3234,8 +3220,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3278,8 +3263,7 @@
 
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3320,8 +3304,7 @@
 
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3341,8 +3324,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3391,8 +3373,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3415,8 +3396,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3439,8 +3419,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3489,8 +3468,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3509,8 +3487,7 @@
 
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -3530,8 +3507,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3579,8 +3555,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3603,8 +3578,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3623,8 +3597,7 @@
 
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3640,8 +3613,7 @@
 
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3690,8 +3662,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3710,8 +3681,7 @@
 
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3727,8 +3697,7 @@
 
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3776,8 +3745,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3796,8 +3764,7 @@
 
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3813,8 +3780,7 @@
 
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3862,8 +3828,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -3886,8 +3851,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -3906,8 +3870,7 @@
 
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3954,8 +3917,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -3978,8 +3940,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4002,8 +3963,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4048,8 +4008,7 @@
 
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -4065,8 +4024,7 @@
 
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
             list_w = []
@@ -4082,8 +4040,7 @@
 
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -4128,8 +4085,7 @@
 
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -4145,8 +4101,7 @@
 
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
             list_w = []
@@ -4193,8 +4148,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4217,8 +4171,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4262,8 +4215,7 @@
 
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4308,8 +4260,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4329,8 +4280,7 @@
 
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4350,8 +4300,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4399,8 +4348,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4423,8 +4371,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4447,8 +4394,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4493,8 +4439,7 @@
 
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4539,8 +4484,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4638,8 +4582,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4660,8 +4603,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4691,8 +4633,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4711,8 +4652,7 @@
 
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -4758,8 +4698,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4782,8 +4721,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4806,8 +4744,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4856,8 +4793,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4880,8 +4816,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -4929,8 +4864,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -4951,8 +4885,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5000,8 +4933,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5024,8 +4956,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5048,8 +4979,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5094,8 +5024,7 @@
 
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
             list_w = []
@@ -5111,8 +5040,7 @@
 
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -5155,8 +5083,7 @@
 
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
             list_w = []
@@ -5201,8 +5128,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5221,8 +5147,7 @@
 
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5268,8 +5193,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5288,8 +5212,7 @@
 
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5335,8 +5258,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5359,8 +5281,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5379,8 +5300,7 @@
 
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5427,8 +5347,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5447,8 +5366,7 @@
 
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5494,8 +5412,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5542,8 +5459,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5562,8 +5478,7 @@
 
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
             list_w = []
@@ -5579,8 +5494,7 @@
 
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
             list_w = []
@@ -5628,8 +5542,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5648,8 +5561,7 @@
 
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
             list_w = []
@@ -5665,8 +5577,7 @@
 
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
             list_w = []
@@ -5686,8 +5597,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5710,8 +5620,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 64:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5764,8 +5673,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5812,8 +5720,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5858,8 +5765,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5904,8 +5810,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -5928,8 +5833,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -5950,8 +5854,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6000,8 +5903,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6024,8 +5926,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6048,8 +5949,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6098,8 +5998,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6120,8 +6019,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6165,8 +6063,7 @@
 
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
             list_w = []
@@ -6186,8 +6083,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6232,8 +6128,7 @@
 
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
             list_w = []
@@ -6253,8 +6148,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6303,8 +6197,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6422,8 +6315,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6446,8 +6338,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6470,8 +6361,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6516,8 +6406,7 @@
 
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
             list_w = []
@@ -6562,8 +6451,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6834,8 +6722,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6858,8 +6745,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6878,8 +6764,7 @@
 
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
             list_w = []
@@ -6926,8 +6811,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -6948,8 +6832,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -6979,8 +6862,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7003,8 +6885,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7023,8 +6904,7 @@
 
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -7067,8 +6947,7 @@
 
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
             list_w = []
@@ -7088,8 +6967,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7113,8 +6991,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7134,8 +7011,7 @@
 
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
             list_w = []
@@ -7184,8 +7060,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7206,8 +7081,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7255,8 +7129,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7277,8 +7150,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -11,7 +11,6 @@
 from pypy.interpreter.astcompiler import optimize # For side effects
 from pypy.interpreter.pyparser.error import SyntaxError
 from pypy.tool import stdlib_opcode as ops
-from pypy.interpreter.error import OperationError
 
 
 def compile_ast(space, module, info):
@@ -21,91 +20,91 @@
 
 
 name_ops_default = misc.dict_to_switch({
-    ast.Load : ops.LOAD_NAME,
-    ast.Store : ops.STORE_NAME,
-    ast.Del : ops.DELETE_NAME
+    ast.Load: ops.LOAD_NAME,
+    ast.Store: ops.STORE_NAME,
+    ast.Del: ops.DELETE_NAME
 })
 
 name_ops_fast = misc.dict_to_switch({
-    ast.Load : ops.LOAD_FAST,
-    ast.Store : ops.STORE_FAST,
-    ast.Del : ops.DELETE_FAST
+    ast.Load: ops.LOAD_FAST,
+    ast.Store: ops.STORE_FAST,
+    ast.Del: ops.DELETE_FAST
 })
 
 name_ops_deref = misc.dict_to_switch({
-    ast.Load : ops.LOAD_DEREF,
-    ast.Store : ops.STORE_DEREF,
+    ast.Load: ops.LOAD_DEREF,
+    ast.Store: ops.STORE_DEREF,
 })
 
 name_ops_global = misc.dict_to_switch({
-    ast.Load : ops.LOAD_GLOBAL,
-    ast.Store : ops.STORE_GLOBAL,
-    ast.Del : ops.DELETE_GLOBAL
+    ast.Load: ops.LOAD_GLOBAL,
+    ast.Store: ops.STORE_GLOBAL,
+    ast.Del: ops.DELETE_GLOBAL
 })
 
 
 unary_operations = misc.dict_to_switch({
-    ast.Invert : ops.UNARY_INVERT,
-    ast.Not : ops.UNARY_NOT,
-    ast.UAdd : ops.UNARY_POSITIVE,
-    ast.USub : ops.UNARY_NEGATIVE
+    ast.Invert: ops.UNARY_INVERT,
+    ast.Not: ops.UNARY_NOT,
+    ast.UAdd: ops.UNARY_POSITIVE,
+    ast.USub: ops.UNARY_NEGATIVE
 })
 
 binary_operations = misc.dict_to_switch({
-    ast.Add : ops.BINARY_ADD,
-    ast.Sub : ops.BINARY_SUBTRACT,
-    ast.Mult : ops.BINARY_MULTIPLY,
-    ast.Mod : ops.BINARY_MODULO,
-    ast.Pow : ops.BINARY_POWER,
-    ast.LShift : ops.BINARY_LSHIFT,
-    ast.RShift : ops.BINARY_RSHIFT,
-    ast.BitOr : ops.BINARY_OR,
-    ast.BitAnd : ops.BINARY_AND,
-    ast.BitXor : ops.BINARY_XOR,
-    ast.FloorDiv : ops.BINARY_FLOOR_DIVIDE
+    ast.Add: ops.BINARY_ADD,
+    ast.Sub: ops.BINARY_SUBTRACT,
+    ast.Mult: ops.BINARY_MULTIPLY,
+    ast.Mod: ops.BINARY_MODULO,
+    ast.Pow: ops.BINARY_POWER,
+    ast.LShift: ops.BINARY_LSHIFT,
+    ast.RShift: ops.BINARY_RSHIFT,
+    ast.BitOr: ops.BINARY_OR,
+    ast.BitAnd: ops.BINARY_AND,
+    ast.BitXor: ops.BINARY_XOR,
+    ast.FloorDiv: ops.BINARY_FLOOR_DIVIDE
 })
 
 inplace_operations = misc.dict_to_switch({
-    ast.Add : ops.INPLACE_ADD,
-    ast.Sub : ops.INPLACE_SUBTRACT,
-    ast.Mult : ops.INPLACE_MULTIPLY,
-    ast.Mod : ops.INPLACE_MODULO,
-    ast.Pow : ops.INPLACE_POWER,
-    ast.LShift : ops.INPLACE_LSHIFT,
-    ast.RShift : ops.INPLACE_RSHIFT,
-    ast.BitOr : ops.INPLACE_OR,
-    ast.BitAnd : ops.INPLACE_AND,
-    ast.BitXor : ops.INPLACE_XOR,
-    ast.FloorDiv : ops.INPLACE_FLOOR_DIVIDE
+    ast.Add: ops.INPLACE_ADD,
+    ast.Sub: ops.INPLACE_SUBTRACT,
+    ast.Mult: ops.INPLACE_MULTIPLY,
+    ast.Mod: ops.INPLACE_MODULO,
+    ast.Pow: ops.INPLACE_POWER,
+    ast.LShift: ops.INPLACE_LSHIFT,
+    ast.RShift: ops.INPLACE_RSHIFT,
+    ast.BitOr: ops.INPLACE_OR,
+    ast.BitAnd: ops.INPLACE_AND,
+    ast.BitXor: ops.INPLACE_XOR,
+    ast.FloorDiv: ops.INPLACE_FLOOR_DIVIDE
 })
 
 compare_operations = misc.dict_to_switch({
-    ast.Eq : 2,
-    ast.NotEq : 3,
-    ast.Lt : 0,
-    ast.LtE : 1,
-    ast.Gt : 4,
-    ast.GtE : 5,
-    ast.In : 6,
-    ast.NotIn : 7,
-    ast.Is : 8,
-    ast.IsNot : 9
+    ast.Eq: 2,
+    ast.NotEq: 3,
+    ast.Lt: 0,
+    ast.LtE: 1,
+    ast.Gt: 4,
+    ast.GtE: 5,
+    ast.In: 6,
+    ast.NotIn: 7,
+    ast.Is: 8,
+    ast.IsNot: 9
 })
 
 subscr_operations = misc.dict_to_switch({
-    ast.AugLoad : ops.BINARY_SUBSCR,
-    ast.Load : ops.BINARY_SUBSCR,
-    ast.AugStore : ops.STORE_SUBSCR,
-    ast.Store : ops.STORE_SUBSCR,
-    ast.Del : ops.DELETE_SUBSCR
+    ast.AugLoad: ops.BINARY_SUBSCR,
+    ast.Load: ops.BINARY_SUBSCR,
+    ast.AugStore: ops.STORE_SUBSCR,
+    ast.Store: ops.STORE_SUBSCR,
+    ast.Del: ops.DELETE_SUBSCR
 })
 
 slice_operations = misc.dict_to_switch({
-    ast.AugLoad : ops.SLICE,
-    ast.Load : ops.SLICE,

From noreply at buildbot.pypy.org  Sat Jun  1 21:45:06 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  1 Jun 2013 21:45:06 +0200 (CEST)
Subject: [pypy-commit] pypy win32-fixes3: document branch
Message-ID: <20130601194506.B8DF11C0196@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: win32-fixes3
Changeset: r64704:6f7a4a142276
Date: 2013-06-01 22:42 +0300
http://bitbucket.org/pypy/pypy/changeset/6f7a4a142276/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -46,3 +46,7 @@
 
 .. branch: operrfmt-NT
 Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+

From noreply at buildbot.pypy.org  Sat Jun  1 21:45:07 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  1 Jun 2013 21:45:07 +0200 (CEST)
Subject: [pypy-commit] pypy win32-fixes3: close branch to be merged
Message-ID: <20130601194507.E7F531C0690@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: win32-fixes3
Changeset: r64705:8ea86639090a
Date: 2013-06-01 22:43 +0300
http://bitbucket.org/pypy/pypy/changeset/8ea86639090a/

Log:	close branch to be merged


From noreply at buildbot.pypy.org  Sat Jun  1 21:45:09 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  1 Jun 2013 21:45:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge win32-fixes3 into default
Message-ID: <20130601194509.704241C1107@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r64706:216ba1e7cb65
Date: 2013-06-01 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/216ba1e7cb65/

Log:	merge win32-fixes3 into default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -46,3 +46,7 @@
 
 .. branch: operrfmt-NT
 Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -578,6 +578,11 @@
 class TestNonInteractive:
     def run_with_status_code(self, cmdline, senddata='', expect_prompt=False,
             expect_banner=False, python_flags='', env=None):
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         cmdline = '%s %s "%s" %s' % (sys.executable, python_flags,
                                      app_main, cmdline)
         print 'POPEN:', cmdline
@@ -706,6 +711,11 @@
         assert 'copyright' not in data
 
     def test_non_interactive_stdout_fully_buffered(self):
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         path = getscript(r"""
             import sys, time
             sys.stdout.write('\x00(STDOUT)\n\x00')   # stays in buffers
@@ -726,6 +736,11 @@
 
     def test_non_interactive_stdout_unbuffered(self, monkeypatch):
         monkeypatch.setenv('PYTHONUNBUFFERED', '1')
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         path = getscript(r"""
             import sys, time
             sys.stdout.write('\x00(STDOUT)\n\x00')
diff --git a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
--- a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
+++ b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
@@ -41,6 +41,10 @@
     assert 'LOG_NOTICE' in d
 
 def test_resource():
+    try:
+        import lib_pypy.resource
+    except ImportError:
+        py.test.skip('no syslog on this platform')
     d = run('resource.ctc.py', '_resource_cache.py')
     assert 'RLIM_NLIMITS' in d
 
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -9,7 +9,7 @@
 link_files = []
 include_dirs = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
-    libraries = ['libeay32', 'ssleay32',
+    libraries = ['libeay32', 'ssleay32', 'zlib1',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
     includes = [
         # ssl.h includes winsock.h, which will conflict with our own
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -103,11 +103,10 @@
     _set_errno(rffi.cast(INT, errno))
 
 if os.name == 'nt':
-    is_valid_fd = rffi.llexternal(
+    is_valid_fd = jit.dont_look_inside(rffi.llexternal(
         "_PyVerify_fd", [rffi.INT], rffi.INT,
         compilation_info=errno_eci,
-        )
-    @jit.dont_look_inside
+        ))
     def validate_fd(fd):
         if not is_valid_fd(fd):
             raise OSError(get_errno(), 'Bad file descriptor')
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -10,7 +10,7 @@
 
 
 if compiler.name == "msvc":
-    libname = 'zlib'
+    libname = 'zlib1' # since version 1.1.4 and later, see http://www.zlib.net/DLL_FAQ.txt
 else:
     libname = 'z'
 eci = ExternalCompilationInfo(
diff --git a/rpython/rtyper/tool/test/test_rffi_platform.py b/rpython/rtyper/tool/test/test_rffi_platform.py
--- a/rpython/rtyper/tool/test/test_rffi_platform.py
+++ b/rpython/rtyper/tool/test/test_rffi_platform.py
@@ -288,9 +288,6 @@
     assert a % struct.calcsize("P") == 0
 
 def test_external_lib():
-    # XXX this one seems to be a bit too platform-specific. Check
-    #     how to test it on windows correctly (using so_prefix?)
-    #     and what are alternatives to LD_LIBRARY_PATH
     eci = ExternalCompilationInfo()
     c_source = """
     int f(int a, int b)
@@ -298,12 +295,17 @@
         return (a + b);
     }
     """
+    if platform.name == 'mscv':
+        c_source = '__declspec(dllexport) ' + c_source
+        libname = 'libc_lib'
+    else:
+        libname = 'c_lib'
     tmpdir = udir.join('external_lib').ensure(dir=1)
     c_file = tmpdir.join('libc_lib.c')
     c_file.write(c_source)
     l = platform.compile([c_file], eci, standalone=False)
     eci = ExternalCompilationInfo(
-        libraries = ['c_lib'],
+        libraries = [libname],
         library_dirs = [str(tmpdir)]
         )
     rffi_platform.verify_eci(eci)
diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py
--- a/rpython/translator/c/gcc/test/test_asmgcroot.py
+++ b/rpython/translator/c/gcc/test/test_asmgcroot.py
@@ -25,8 +25,8 @@
 
     @classmethod
     def make_config(cls):
-        if _MSVC and _WIN64:
-            py.test.skip("all asmgcroot tests disabled for MSVC X64")
+        if _MSVC:
+            py.test.skip("all asmgcroot tests disabled for MSVC")
         from rpython.config.translationoption import get_combined_translation_config
         config = get_combined_translation_config(translating=True)
         config.translation.gc = cls.gcpolicy

From noreply at buildbot.pypy.org  Sun Jun  2 10:04:17 2013
From: noreply at buildbot.pypy.org (juanfra684)
Date: Sun,  2 Jun 2013 10:04:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fixes to OpenBSD.
Message-ID: <20130602080417.757971C01E5@cobra.cs.uni-duesseldorf.de>

Author: Juan Francisco Cantero Hurtado <iam at juanfra.info>
Branch: 
Changeset: r64707:e92ef8459498
Date: 2013-06-02 05:40 +0200
http://bitbucket.org/pypy/pypy/changeset/e92ef8459498/

Log:	Fixes to OpenBSD.

	- Fixes the order of CFLAGS and LDFLAGS.
	- Uses the compiler within $CC with a fallback to cc.

	Both changes make the work of the packagers easier.

diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -5,11 +5,16 @@
 from rpython.translator.platform.bsd import BSD
 
 class OpenBSD(BSD):
-    DEFAULT_CC = "cc"
+    if os.environ.get("CC") is None:
+        DEFAULT_CC = "cc"
+    else:
+        DEFAULT_CC = os.environ.get("CC")
+
     name = "openbsd"
 
-    link_flags = os.environ.get("LDFLAGS", '-pthread').split()
-    cflags = os.environ.get("CFLAGS", "-O3 -pthread -fomit-frame-pointer -D_BSD_SOURCE").split()
+    link_flags = os.environ.get("LDFLAGS", "").split() + ['-pthread']
+    cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-D_BSD_SOURCE'
+             ] + os.environ.get("CFLAGS", "").split()
 
     def _libs(self, libraries):
         libraries=set(libraries + ("intl", "iconv", "compat"))

From noreply at buildbot.pypy.org  Sun Jun  2 10:25:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 10:25:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix _testcapi and _test_ctypes after
 722471a15693, sorry.
Message-ID: <20130602082500.039631C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64708:a674c4a68284
Date: 2013-06-02 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/a674c4a68284/

Log:	Fix _testcapi and _test_ctypes after 722471a15693, sorry.

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,6 +1,12 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
 def compile_shared():
     """Compile '_ctypes_test.c' into an extension module, and import it
     """
@@ -8,7 +14,6 @@
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -25,7 +30,7 @@
     object_filename = res[0]
 
     # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
+    output_filename = '_ctypes_test' + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,6 +1,12 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
 def compile_shared():
     """Compile '_testcapi.c' into an extension module, and import it
     """
@@ -8,7 +14,6 @@
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -25,7 +30,7 @@
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
+    output_filename = '_testcapi' + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')

From noreply at buildbot.pypy.org  Sun Jun  2 10:25:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 10:25:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill code duplication.
Message-ID: <20130602082501.5EDF71C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64709:18c04dc498b8
Date: 2013-06-02 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/18c04dc498b8/

Log:	Kill code duplication.

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,65 +1,7 @@
-import os, sys, imp
-import tempfile
-
-def _get_c_extension_suffix():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + _get_c_extension_suffix()
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -67,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_pypy_testcapi.py
copy from lib_pypy/_testcapi.py
copy to lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -7,8 +7,9 @@
             return ext
 
 
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
     """
     thisdir = os.path.dirname(__file__)
     output_dir = tempfile.mkdtemp()
@@ -24,13 +25,13 @@
         ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
+    res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
                            extra_preargs=ccflags)
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + _get_c_extension_suffix()
+    output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
@@ -42,7 +43,7 @@
             library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
+                        '/EXPORT:init' + modulename]
     else:
         libraries = []
         extra_ldargs = []
@@ -54,14 +55,7 @@
         libraries=libraries,
         extra_preargs=extra_ldargs)
 
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-try:
-    import cpyext
-except ImportError:
-    raise ImportError("No module named '_testcapi'")
-else:
-    compile_shared()
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,67 +1,7 @@
-import os, sys, imp
-import tempfile
-
-def _get_c_extension_suffix():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + _get_c_extension_suffix()
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
 try:
     import cpyext
 except ImportError:
     raise ImportError("No module named '_testcapi'")
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')

From noreply at buildbot.pypy.org  Sun Jun  2 12:16:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 12:16:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue #1507: silence "not holding the
 import lock" after a fork().
Message-ID: <20130602101647.573911C0FFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64710:51f304544a4b
Date: 2013-06-02 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/51f304544a4b/

Log:	issue #1507: silence "not holding the import lock" after a fork().

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -791,6 +791,10 @@
     def release_lock(self):
         me = self.space.getexecutioncontext()   # used as thread ident
         if self.lockowner is not me:
+            if self.lockowner is None:
+                # Too bad.  This situation can occur if a fork() occurred
+                # with the import lock held, and we're the child.
+                return
             if not self._can_have_lock():
                 return
             space = self.space

From noreply at buildbot.pypy.org  Sun Jun  2 13:31:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 13:31:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Backward compatibility
Message-ID: <20130602113156.0FF361C0FFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64711:3cdac1ac25eb
Date: 2013-06-02 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/3cdac1ac25eb/

Log:	Backward compatibility

diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -151,6 +151,8 @@
     'DEBUG': DEBUG,
     'NOTSET': NOTSET,
 }
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """

From noreply at buildbot.pypy.org  Sun Jun  2 18:32:50 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 18:32:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for x86/test/test_fficall on
 Windows: use a value for "abi" that
Message-ID: <20130602163250.972DF1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64712:663474f13069
Date: 2013-06-02 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/663474f13069/

Log:	Fix for x86/test/test_fficall on Windows: use a value for "abi" that
	makes sense for the backend in this test.

diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -17,7 +17,7 @@
 def get_description(atypes, rtype):
     p = lltype.malloc(CIF_DESCRIPTION, len(atypes),
                       flavor='raw', immortal=True)
-    p.abi = 42
+    p.abi = 1     # default
     p.nargs = len(atypes)
     p.rtype = rtype
     p.atypes = lltype.malloc(FFI_TYPE_PP.TO, len(atypes),

From noreply at buildbot.pypy.org  Sun Jun  2 19:22:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 19:22:56 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Updates
Message-ID: <20130602172256.A6CFC1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r55:7076fbc8975d
Date: 2013-06-02 19:20 +0200
http://bitbucket.org/pypy/stmgc/changeset/7076fbc8975d/

Log:	Updates

diff --git a/c3/doc-stmgc.txt b/c3/doc-stmgc.txt
--- a/c3/doc-stmgc.txt
+++ b/c3/doc-stmgc.txt
@@ -2,6 +2,55 @@
 Details of the interactions between STM and the GC
 ==================================================
 
+
+--------------------------
+Introduction (hand-waving)
+--------------------------
+
+When we run multiple threads, the common case is to access objects that
+have only been seen by the current thread.  Accessing the same object
+from multiple threads is possible, and handled correctly (that's the
+whole point), but a relatively rare case.
+
+So each object is classified as "public" or "protected".  New objects
+are protected until they are read by a different thread.  The point is
+to use very different mechanisms for public and for protected objects.
+Public objects are visible by all threads, but read-only in memory; to
+change them, a copy must be made, and the changes written to the copy
+(the "redolog" approach to STM).  Protected objects, on the other hand,
+are modified in-place, with (if necessary) a copy of them being made
+only for the purpose of a possible abort of the transaction (the
+"undolog" approach).
+
+This is combined with a generational GC similar to PyPy's --- but here,
+each thread gets its own nursery and does its own "minor collections",
+independently of the others.
+
+Objects start as protected, and when another thread tries to follow a
+pointer to them, then it is that other thread's job to carefully "steal"
+the object and turn it public (possibly making a copy of it if needed,
+e.g. if it was still a young object living in the original nursery).
+
+The same object can exist temporarily in multiple versions: any number
+of public copies; at most one active protected copy; and optionally one
+private copy per thread (this is the copy as currently seen by the
+transaction in progress on that thread).  The GC cleans up the
+unnecessary copies.
+
+These ideas are basically variants and extensions of the same basic idea
+of keeping multiple copies with revision numbers to track them.
+Moreover, "read barriers" and "write barriers" are used by the C program
+calling into this library in order to be sure that it is accessing the
+right version of the object.  In the current variant we can have
+extremely cheap read barriers, which are definitely a major speed
+improvement over the previous variants (and, as far as I know, over most
+of the other existing STMs).
+
+
+----------------------
+Details (more precise)
+----------------------
+
 In this document we say "high-level object" to mean an object from the
 point of the user of the library, as opposed to an "object copy", which
 is what occupies the space of one allocated piece of memory.  One
@@ -10,40 +59,41 @@
 for a globally consistent copy of all objects.  One revision is the
 result of one transaction.  A program usually has one revision per
 thread in progress, plus any number of older committed revisions.  The
-committed revisions are globally ordered.
+committed revisions are globally ordered.  This is the order that the
+multithreaded program appears to have followed serially.
 
-The object copies exist in one of three main states: they can be
-"private", "protected" or "public".  A copy is private when it belongs
-to the transaction in progress.  When that transaction commits, it
-becomes protected, and remains so as long as it is accessed only by the
-same thread.  A copy becomes public only when another thread requests
-access to it (or, more precisely, "steals" access to it).  Once public,
-a copy is immutable in memory.
+The object copies exist in one of two main states: they can be
+"protected" or "public".  A copy is also called "private" when it was
+modified by the transaction in progress; this copy is always protected
+and invisible to other threads.  When that transaction commits, all
+private copies become protected, and remain so as long as it is accessed
+only by the same thread.  A copy becomes public only when another thread
+requests access to it (or, more precisely, "steals" access to it).  Once
+public, a copy is immutable in memory.
 
 From the point of view of the generational GC, each copy is either young
 or old.  All new copies are allocated young.  They become old at the
 next minor collection.  In the common case, copies are allocated in the
 nursery, and during the next minor collection, if they survive, they are
 moved outside.  The nursery contains only young copies, but a few copies
-outside might be young too (e.g. copies of objects too large for the
+outside might be young too (e.g. object copies too large for the
 nursery).  (In fact we found out in PyPy that it's a good idea to create
 objects young even if they are outside the nursery; otherwise, a program
 that creates a lot of medium-sized objects will quickly exhaust the
 memory and trigger a lot of major collections.)  For the rest of this
 document we'll ignore young copies outside the nursery.
 
-An object that was never seen by a different thread has got at most two
-copies: the first, protected, is the copy at the latest committed
-revision; and the other, private, is the current copy.  If the
-transaction aborts we can forget the private copy and reuse the previous
-protected copy.  If the transaction commits we forget the previous
-protected copy instead; then the private copy becomes protected, like
-*all* private copies at this point.  If the object is modified again in
-the near future, we reuse the memory that was occupied by the previous
-copy to store the next private copy.  As a result, each of these two
-spaces in memory can be young or old.  If an object is no longer
-modified for long enough, the next (minor or major) GC will free one of
-the two spaces it uses.
+An object that was never seen by a different thread has got either one
+of two copies, both protected: the "main" one, used by the thread, which
+may be private or not depending on whether the object was modified in
+the current transaction; and, if the object is private but older than
+the current transaction, then it has got a secondary copy whose purpose
+is to record the state that the object had at the start of the current
+transaction.
+
+If an object is committed and then no longer modified for long enough,
+the next (minor or major) GC will free the space that was used by the
+secondary copy.
 
 The way to share data between threads goes via prebuilt objects, which
 are always public: it is their existence that gives the starting point
@@ -53,15 +103,15 @@
 1. A thread tries to write to a public object.  This is done by
 allocating a fresh private copy of the public object.  Then writes go to
 the private copy.  If the transaction commits, the private copy becomes
-protected, and the public object is made to point to it (with
+simply protected, and the public object is made to point to it (with
 multithread care).  From now on, any access to the public object from
-the same thread will work on the protected object or its future private
-copy.  Any access from a different thread will trigger "stealing", as
-explained next.
+the same thread will work on the protected copy.  Any access from a
+different thread will trigger "stealing", as explained next.
 
 2. When we are running a thread, it may try to access a public object
 but find that another thread (the "foreign thread") has committed
-changes to it .  Then we "steal" the object.  It is a read-only
+changes to it; i.e. the object has a protected copy, but belonging to a
+foreign thread.  Then we "steal" the object.  It is a read-only
 operation performed by peeking on the foreign thread's data.  The
 operation involves making a duplicate of the original copy, if it was in
 the foreign thread's nursery (so that no thread ever reads another
@@ -69,11 +119,11 @@
 original protected copy if it was not in the nursery, is then marked as
 public.  From now on nobody is allowed to change (or free) the content
 of this copy, and it becomes the current public copy.  These public
-copies accumulate: every time the same object is stolen by a different
-thread, a new public copy is made (so that unrelated threads don't have
-to worry about existing public copies being updated).  (This chain of
-objects is freed at the next major GC, which is a stop-the-world
-operation.)
+copies accumulate in case the same object is successively stolen by
+different threads.  A new public copy is made every time, so that
+unrelated threads don't have to worry about existing public copies being
+updated.  (This chain of objects is freed at the next major GC, which is
+a stop-the-world operation.)
 
 3. A subtle but important point about making a public copy is about all
 the references stored in the object: if they point to other protected
@@ -103,23 +153,23 @@
 3. the read/write barriers.
 
 Point 3 is essential for performance: we want most importantly a read
-barrier that doesn't trigger for the cases described above.  There are
-basically three cases:
+barrier that doesn't trigger for the cases described above.  The read
+barrier needs to check if a pointer P references a public copy that
+was outdated by a future revision.  This is an easy check, which can
+be implemented by checking a flag in the header of the copy.  In all
+the common cases, this flag is not set, and no actual call needs to
+be done.
 
-1. read_barrier(P) = P   [if P is directly a non-modified copy]
-
-2. read_barrier(P) = P->h_revision   [protected copy -> private copy]
-
-3. all other more complex cases, handled by a call.
-
-It is possible to compress the first two cases into C code that GCC
-compiles into a total of two or three assembler instructions, using a
-conditional move.  (Still, it is unclear so far if case 2 is worth
-inlining at every read barrier site, or should be left to the call.)
-
-The case of the write barrier is similar, but differs in the check for
-case 1: this case is reduced to only private objects.  This could be
-done simply by checking a different GC flags.
+The case of the write barrier is similar, but differs in the check we
+need to do.  We need to do a call if the object is not already private.
+For performance reasons, "being private" is not directly a flag in the
+object, because when a transaction commits, we don't want to have to
+walk all private objects to change this flag.  Instead, private objects
+have a precise negative odd number in their `h_revision` field, called
+the "local revision number".  When a transaction commits, we change the
+value of the local revision number, and all previously-private objects
+become automatically protected.  So the write barrier fast-path checks
+if the `h_revision` is equal from the local revision number.
 
 Note that this design relies on the following property: in a given copy
 of an object which was committed at revision N, all pointers points to
@@ -134,10 +184,12 @@
 
 A public object copy is either up-to-date (no more recent committed
 copy) or outdated.  The public copies need to have globally consistent
-revision numbers (produced at commit time).  If there are several copies
-of the same object, we only need to store the revision number of the
-most recent one.  The previous copies are simply outdated and need
-instead to store a pointer to a more recent copy.
+revision numbers (produced at commit time).  If there are several public
+copies of the same object, we only need to store the revision number of
+the most recent one.  The previous copies are simply outdated and need
+instead to store a pointer to a more recent copy.  We use the same field
+`h_revision` to store either the revision number or the pointer to the
+more recent copy.
 
 The important property is that committed transactions must be
 "linearized": when we look at them a posteriori, it must be as if they
@@ -150,17 +202,20 @@
 at the end of the transaction, all objects read during the transaction
 have a revision not greater than this starting time, then we have no
 "read-write" conflict (i.e. reads of an object that another thread has
-modified; there are also "write-write" conflicts).  An improvement over
-this basic model is that if, during the transaction, we are about to
-read a new object and detect a read-write conflict, we can try to
-"extend" the starting time to the value that is now stored in the global
-time variable.  If none of the objects that we have read previously have
-been modified in the interval, then the transaction would have given the
-same results if it had started at the new time.
+modified; there are also in theory "write-write" conflicts, but this
+case can be reduced to read-write conflicts if we consider that all
+writes are also reads).
+
+An improvement over this basic model is that if, during the transaction,
+we are about to read a new object and detect a read-write conflict, we
+can try to "extend" the starting time to the value that is now stored in
+the global time variable.  We need to check that none of the objects
+that we have read previously have been modified in the interval.  If
+that is the case, then the transaction would have given the same results
+if it had started at the new time.
 
 The model described above is known in the literature as the "extendable
-timestamp" model.  We apply it for public object.  It can however be
-tweaked for protected objects.
+timestamp" model.  We apply it for public object.
 
 
 Pointers and revision numbers on protected objects
@@ -169,7 +224,7 @@
 In traditional transactional systems, we have a special case to speed up
 transactions that don't do any write; but that seems too restricted to
 be useful in PyPy.  Instead, we can have a special case for transactions
-that don't write to any *public* object.  Our assumption is that
+that didn't try to write to any *public* object.  Our assumption is that
 transactions can be anywhere from very small to very large; the small
 ones are unlikely to change any object that has been seen by another
 thread.  Moreover the previous transaction in the same thread is
@@ -182,15 +237,20 @@
 global order of the committed transactions is given by the ordering of
 these 2-tuples.
 
-A commit with writes to public objects works as described above; it
+A commit which writes to public objects works as described above; it
 gives the transaction the number `(global_time, 0)`, and atomically
 increments `global_time`.
 
 A commit with no write to any public object produces the number
-`(start_time, N+1)`, provided that we didn't have any object stolen
-since `start_time`.  This condition is enough to guarantee that it is ok
-to linearize the transaction at `(start_time, N+1)` even though other
-threads might already have produced transactions linearized at some
-greater time.  Indeed, the fact that no object of ours was stolen
+`(start_time, N+1)`, provided that we didn't have any of our objects
+stolen since `start_time`.  This condition is enough to guarantee that
+it is ok to linearize the transaction at `(start_time, N+1)` even though
+other threads might already have produced transactions linearized at
+some later time.  Indeed, the fact that no object of ours was stolen
 means that no other thread's transaction depends on any object we're
-about to commit.
+about to commit.  In other words, the absence of both public writes and
+stealing is a cheap way to determine that this transaction "commutes"
+with other transactions already committed.  (My current guess is that we
+can in this way reduce the pressure over the word of memory that
+contains the shared "global time" variable, and make many very short
+transactions efficient.)

From noreply at buildbot.pypy.org  Sun Jun  2 19:22:57 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 19:22:57 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Update README.txt
Message-ID: <20130602172257.D064D1C0FFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r56:113d9570c3ad
Date: 2013-06-02 19:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/113d9570c3ad/

Log:	Update README.txt

diff --git a/README.txt b/README.txt
--- a/README.txt
+++ b/README.txt
@@ -9,7 +9,11 @@
 
 The library interface is in "c3/stmgc.h".
 
-A demo program can be found in "c3/demo1.c".
+The file "c3/doc-stmgc.txt" contains a high-level overview followed by
+more detailled explanations.
+
+A demo program can be found in "c3/demo1.c", but the code so far is
+outdated (it doesn't follow what c3/doc-stmgc describes).
 It can be built with "make debug-demo1" or "make build-demo1".
 
 The plan is to use this C code directly with PyPy, and not write

From noreply at buildbot.pypy.org  Sun Jun  2 19:34:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 19:34:47 +0200 (CEST)
Subject: [pypy-commit] stmgc default: More text.
Message-ID: <20130602173447.574D71C0FFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r57:b522cf8f5f8a
Date: 2013-06-02 19:34 +0200
http://bitbucket.org/pypy/stmgc/changeset/b522cf8f5f8a/

Log:	More text.

diff --git a/c3/doc-stmgc.txt b/c3/doc-stmgc.txt
--- a/c3/doc-stmgc.txt
+++ b/c3/doc-stmgc.txt
@@ -87,13 +87,13 @@
 of two copies, both protected: the "main" one, used by the thread, which
 may be private or not depending on whether the object was modified in
 the current transaction; and, if the object is private but older than
-the current transaction, then it has got a secondary copy whose purpose
-is to record the state that the object had at the start of the current
+the current transaction, then it has got a backup copy whose purpose is
+to record the state that the object had at the start of the current
 transaction.
 
 If an object is committed and then no longer modified for long enough,
 the next (minor or major) GC will free the space that was used by the
-secondary copy.
+backup copy.
 
 The way to share data between threads goes via prebuilt objects, which
 are always public: it is their existence that gives the starting point
@@ -166,10 +166,10 @@
 object, because when a transaction commits, we don't want to have to
 walk all private objects to change this flag.  Instead, private objects
 have a precise negative odd number in their `h_revision` field, called
-the "local revision number".  When a transaction commits, we change the
-value of the local revision number, and all previously-private objects
-become automatically protected.  So the write barrier fast-path checks
-if the `h_revision` is equal from the local revision number.
+the "local revision identifier".  When a transaction commits, we change
+the value of the local revision identifier, and all previously-private
+objects become automatically protected.  So the write barrier fast-path
+checks if the `h_revision` is equal from the local revision identifier.
 
 Note that this design relies on the following property: in a given copy
 of an object which was committed at revision N, all pointers points to
@@ -218,8 +218,8 @@
 timestamp" model.  We apply it for public object.
 
 
-Pointers and revision numbers on protected objects
---------------------------------------------------
+Commits on protected objects
+----------------------------
 
 In traditional transactional systems, we have a special case to speed up
 transactions that don't do any write; but that seems too restricted to
@@ -254,3 +254,28 @@
 can in this way reduce the pressure over the word of memory that
 contains the shared "global time" variable, and make many very short
 transactions efficient.)
+
+
+Details of protected objects
+----------------------------
+
+As described above, each thread has a "local revision identifier".  It
+is a negative odd number that changes whenever it commits a transaction.
+The important point for the write barrier is that on any object copy,
+`h_revision` must be equal to the local revision identifier if and only
+if the copy is private.  A newly allocated object is always private.
+Once the transaction commits it becomes merely protected.  Its
+`h_revision` field doesn't change (but the thread's local revision
+identifier does).  If later the write barrier triggers on it, we make a
+backup copy of the object and copy the content of the primary copy to
+it.  We also set `h_revision` in the primary copy to point to the
+backup copy: as long as `h_revision` is different from the local
+revision identifier, its exact value is otherwise not used.  In this way
+we can keep using the same backup copy in each future transaction
+that needs to write to the object.
+
+The backup copy is used in two cases.  One is if the transaction aborts;
+then we copy the content back over the regular protected copy.  The
+other case is if the object is stolen.  In that case, if the object has
+an active backup copy, we must steal this one, because the regular
+protected copy is actually private at that point in time.

From noreply at buildbot.pypy.org  Sun Jun  2 20:27:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 20:27:00 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Desccribe minor and major collections.
 Hand-waving for now.
Message-ID: <20130602182700.33A6E1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r58:3b7bd55eed2f
Date: 2013-06-02 20:26 +0200
http://bitbucket.org/pypy/stmgc/changeset/3b7bd55eed2f/

Log:	Desccribe minor and major collections. Hand-waving for now.

diff --git a/c3/doc-stmgc.txt b/c3/doc-stmgc.txt
--- a/c3/doc-stmgc.txt
+++ b/c3/doc-stmgc.txt
@@ -171,13 +171,6 @@
 objects become automatically protected.  So the write barrier fast-path
 checks if the `h_revision` is equal from the local revision identifier.
 
-Note that this design relies on the following property: in a given copy
-of an object which was committed at revision N, all pointers points to
-copies of objects which were committed at or before revision N.  This
-property is true by construction, but we must be careful not to break it
-by "optimizing" the content of a copy.  In particular the GC, during
-both minor and major collections, has to preserve this property.  
-
 
 The extendable timestamp model
 ------------------------------
@@ -279,3 +272,36 @@
 other case is if the object is stolen.  In that case, if the object has
 an active backup copy, we must steal this one, because the regular
 protected copy is actually private at that point in time.
+
+
+Minor and major collections
+---------------------------
+
+The GC needs to interact with objects being copied outside the nursery:
+we need to detect if, later, they are modified to contain a pointer to a
+nursery object.  This is the classical purpose of a write barrier in GC
+terms.  In our case, we need the write barrier's call to occur even on a
+private object freshly copied out of the nursery, the first time it is
+written to.  This is easily combined with the write barrier described
+above: when a minor collection copies objects out of the nursery,
+private objects' `h_revision` field is temporarily replaced with a
+different value.
+
+Major (global) collections are stop-the-world: when they need to occur,
+the threads are all stopped at the next safe point.  Then the problem
+is simplified to a regular complete garbage collection.  Additionally,
+as hinted above, we can compact chains of public object copies.
+
+Note that our design relies on the following property: in a given copy
+of an object which was committed at revision N, all pointers points to
+copies of objects which were committed at or before revision N.  This
+property is true by construction, but we must be careful not to break it
+by "optimizing" the content of a copy.  In particular, major collections
+have to preserve this property.  It means that the best (but still safe)
+thing to do during major collection is to compress chains of public
+objects down to one copy (the most recent one) and one stub.  We fix the
+references in existing objects to point to either the real copy or the
+stub.  This is probably a bit involved: we might have to get the current
+revision numbers of all threads, and theoretically compact each interval
+of number down to only one number, but still keep one active revision
+number per thread.

From noreply at buildbot.pypy.org  Sun Jun  2 20:48:24 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 20:48:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue #1508: fix these last two usages
 of a fixed-ascii string-to-unicode
Message-ID: <20130602184824.982921C101E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64713:64c70e584a54
Date: 2013-06-02 20:47 +0200
http://bitbucket.org/pypy/pypy/changeset/64c70e584a54/

Log:	issue #1508: fix these last two usages of a fixed-ascii string-to-
	unicode convertion, which CPython doesn't do anyway.

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -352,9 +352,8 @@
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):
-                # convert string to unicode explicitely here
-                from pypy.objspace.std.unicodetype import plain_str2unicode
-                r = plain_str2unicode(self.space, r)
+                # convert string to unicode using the default encoding
+                r = self.space.unicode_w(self.space.wrap(r))
             prec = self.prec
             if prec == -1 and self.width == 0:
                 # fast path
@@ -509,12 +508,10 @@
             result = formatter.format()
         except NeedUnicodeFormattingError:
             # fall through to the unicode case
-            from pypy.objspace.std.unicodetype import plain_str2unicode
-            fmt = plain_str2unicode(space, fmt)
+            pass
         else:
             return space.wrap(result)
-    else:
-        fmt = space.unicode_w(w_fmt)
+    fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
     return space.wrap(result)
diff --git a/pypy/objspace/std/test/test_stringobject.py b/pypy/objspace/std/test/test_stringobject.py
--- a/pypy/objspace/std/test/test_stringobject.py
+++ b/pypy/objspace/std/test/test_stringobject.py
@@ -530,6 +530,12 @@
                 del sys.modules[module_name]
             temp_sys.setdefaultencoding('utf-8')
             assert u''.join(['\xc3\xa1']) == u'\xe1'
+            #
+            assert ('\xc3\xa1:%s' % u'\xe2') == u'\xe1:\xe2'
+            class Foo(object):
+                def __repr__(self):
+                    return '\xc3\xa2'
+            assert u'\xe1:%r' % Foo() == u'\xe1:\xe2'
         finally:
             temp_sys.setdefaultencoding(old_encoding)
             sys.modules.update(self.original_modules)
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -13,22 +13,6 @@
     from pypy.objspace.std.unicodeobject import W_UnicodeObject
     return W_UnicodeObject(uni)
 
-def plain_str2unicode(space, s):
-    try:
-        return unicode(s)
-    except UnicodeDecodeError:
-        for i in range(len(s)):
-            if ord(s[i]) > 127:
-                raise OperationError(
-                    space.w_UnicodeDecodeError,
-                    space.newtuple([
-                    space.wrap('ascii'),
-                    space.wrap(s),
-                    space.wrap(i),
-                    space.wrap(i+1),
-                    space.wrap("ordinal not in range(128)")]))
-        assert False, "unreachable"
-
 
 unicode_capitalize = SMM('capitalize', 1,
                          doc='S.capitalize() -> unicode\n\nReturn a'

From noreply at buildbot.pypy.org  Sun Jun  2 21:22:16 2013
From: noreply at buildbot.pypy.org (juanfra684)
Date: Sun,  2 Jun 2013 21:22:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: Reverts the last change to CC in
 openbsd.py. PyPy reads correctly $CC.
Message-ID: <20130602192216.8FF311C01E5@cobra.cs.uni-duesseldorf.de>

Author: Juan Francisco Cantero Hurtado <iam at juanfra.info>
Branch: 
Changeset: r64714:b56af2127b16
Date: 2013-06-02 20:46 +0200
http://bitbucket.org/pypy/pypy/changeset/b56af2127b16/

Log:	Reverts the last change to CC in openbsd.py. PyPy reads correctly
	$CC.

	Reported by Laurence Tratt.

diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -5,11 +5,7 @@
 from rpython.translator.platform.bsd import BSD
 
 class OpenBSD(BSD):
-    if os.environ.get("CC") is None:
-        DEFAULT_CC = "cc"
-    else:
-        DEFAULT_CC = os.environ.get("CC")
-
+    DEFAULT_CC = "cc"
     name = "openbsd"
 
     link_flags = os.environ.get("LDFLAGS", "").split() + ['-pthread']

From noreply at buildbot.pypy.org  Sun Jun  2 21:22:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 21:22:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged in juanfra684/pypy (pull request
	#154)
Message-ID: <20130602192217.F03191C01E5@cobra.cs.uni-duesseldorf.de>

Author: arigo <armin.rigo at gmail.com>
Branch: 
Changeset: r64715:a4a78a85ceaf
Date: 2013-06-02 21:21 +0200
http://bitbucket.org/pypy/pypy/changeset/a4a78a85ceaf/

Log:	Merged in juanfra684/pypy (pull request #154)

	Reverts the last change to CC in openbsd.py. PyPy reads correctly
	$CC.

diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -5,11 +5,7 @@
 from rpython.translator.platform.bsd import BSD
 
 class OpenBSD(BSD):
-    if os.environ.get("CC") is None:
-        DEFAULT_CC = "cc"
-    else:
-        DEFAULT_CC = os.environ.get("CC")
-
+    DEFAULT_CC = "cc"
     name = "openbsd"
 
     link_flags = os.environ.get("LDFLAGS", "").split() + ['-pthread']

From noreply at buildbot.pypy.org  Sun Jun  2 22:21:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 22:21:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Hack more,
 trying to work around both CPython's test_imp.py and
Message-ID: <20130602202149.20B231C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64716:6c22c424ddb8
Date: 2013-06-02 22:21 +0200
http://bitbucket.org/pypy/pypy/changeset/6c22c424ddb8/

Log:	Hack more, trying to work around both CPython's test_imp.py and
	CPython's buggy corresponding behavior
	(http://bugs.python.org/issue18122).

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -303,7 +303,7 @@
         return _absolute_import(space, modulename, baselevel,
                                 fromlist_w, tentative)
     finally:
-        lock.release_lock()
+        lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
 def absolute_import_try(space, modulename, baselevel, fromlist_w):
@@ -788,10 +788,10 @@
             self.lockowner = me
         self.lockcounter += 1
 
-    def release_lock(self):
+    def release_lock(self, silent_after_fork):
         me = self.space.getexecutioncontext()   # used as thread ident
         if self.lockowner is not me:
-            if self.lockowner is None:
+            if self.lockowner is None and silent_after_fork:
                 # Too bad.  This situation can occur if a fork() occurred
                 # with the import lock held, and we're the child.
                 return
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -177,7 +177,7 @@
 
 def release_lock(space):
     if space.config.objspace.usemodules.thread:
-        importing.getimportlock(space).release_lock()
+        importing.getimportlock(space).release_lock(silent_after_fork=False)
 
 def reinit_lock(space):
     if space.config.objspace.usemodules.thread:

From noreply at buildbot.pypy.org  Sun Jun  2 22:23:20 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Jun 2013 22:23:20 +0200 (CEST)
Subject: [pypy-commit] pypy shared-by-default: a branch to measure if
 --shared can be enabled by default
Message-ID: <20130602202320.B835C1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: shared-by-default
Changeset: r64717:858bc45b1b06
Date: 2013-06-03 04:22 +0800
http://bitbucket.org/pypy/pypy/changeset/858bc45b1b06/

Log:	a branch to measure if --shared can be enabled by default

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -93,8 +93,7 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")]
 }
 
 module_import_dependencies = {
@@ -309,6 +308,7 @@
     # unspecified and we get None.  It shouldn't occur in translate.py though.
     type_system = config.translation.type_system
     backend = config.translation.backend
+    config.translation.suggest(shared=True)
 
     # all the good optimizations for PyPy should be listed here
     if level in ['2', '3', 'jit']:

From noreply at buildbot.pypy.org  Sun Jun  2 22:30:59 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun,  2 Jun 2013 22:30:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: avoid 0 sized stack modifications if we
 are not pushing any args
Message-ID: <20130602203059.DF25E1C01E5@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64718:1d6e2058a146
Date: 2013-06-02 09:44 -0500
http://bitbucket.org/pypy/pypy/changeset/1d6e2058a146/

Log:	avoid 0 sized stack modifications if we are not pushing any args

diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -52,6 +52,8 @@
 
     def _push_stack_args(self, stack_args, on_stack):
         assert on_stack % 8 == 0
+        if on_stack == 0:
+            return
         self._adjust_sp(-on_stack)
         self.current_sp = on_stack
         ofs = 0
@@ -71,7 +73,7 @@
             else:
                 self.mc.gen_load_int(r.ip.value, n)
                 self.mc.ADD_rr(r.sp.value, r.sp.value, r.ip.value)
-        else:
+        elif n < 0:
             n = abs(n)
             if check_imm_arg(n):
                 self.mc.SUB_ri(r.sp.value, r.sp.value, n)

From noreply at buildbot.pypy.org  Sun Jun  2 22:31:01 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun,  2 Jun 2013 22:31:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: use larger allowed imm size for VLDR
	and VSTR
Message-ID: <20130602203101.423781C01E5@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64719:c970352ba2f5
Date: 2013-06-02 15:19 -0500
http://bitbucket.org/pypy/pypy/changeset/c970352ba2f5/

Log:	use larger allowed imm size for VLDR and VSTR

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -8,6 +8,7 @@
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.opassembler import ResOpAssembler
 from rpython.jit.backend.arm.regalloc import (Regalloc,
     CoreRegisterManager, check_imm_arg, VFPRegisterManager,
@@ -961,7 +962,7 @@
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VLDR(target.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
@@ -982,7 +983,7 @@
             return self._store_core_reg(mc, source, base, ofs, cond, helper)
 
     def _store_vfp_reg(self, mc, source, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VSTR(source.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
diff --git a/rpython/jit/backend/arm/helper/regalloc.py b/rpython/jit/backend/arm/helper/regalloc.py
--- a/rpython/jit/backend/arm/helper/regalloc.py
+++ b/rpython/jit/backend/arm/helper/regalloc.py
@@ -4,7 +4,10 @@
 from rpython.jit.metainterp.history import ConstInt
 from rpython.rlib.objectmodel import we_are_translated
 
-def check_imm_arg(arg, size=0xFF, allow_zero=True):
+VMEM_imm_size=0x3FC
+default_imm_size=0xFF
+
+def check_imm_arg(arg, size=default_imm_size, allow_zero=True):
     assert not isinstance(arg, ConstInt)
     if not we_are_translated():
         if not isinstance(arg, int):

From noreply at buildbot.pypy.org  Sun Jun  2 22:31:02 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun,  2 Jun 2013 22:31:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: unify code that generates memory read
	instructions
Message-ID: <20130602203102.8849D1C01E5@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64720:641e439a5353
Date: 2013-06-02 15:22 -0500
http://bitbucket.org/pypy/pypy/changeset/641e439a5353/

Log:	unify code that generates memory read instructions

diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -15,6 +15,7 @@
                                                 gen_emit_unary_float_op,
                                                 saved_registers)
 from rpython.jit.backend.arm.helper.regalloc import check_imm_arg
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.jump import remap_frame_layout
 from rpython.jit.backend.arm.regalloc import TempBox
@@ -23,6 +24,7 @@
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import InteriorFieldDescr
 from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler
+from rpython.jit.backend.llsupport.regalloc import get_scale
 from rpython.jit.metainterp.history import (Box, AbstractFailDescr,
                                             INT, FLOAT, REF)
 from rpython.jit.metainterp.history import TargetToken
@@ -559,47 +561,8 @@
     def emit_op_getfield_gc(self, op, arglocs, regalloc, fcond):
         base_loc, ofs, res, size = arglocs
         signed = op.getdescr().is_field_signed()
-        if size.value == 8:
-            assert res.is_vfp_reg()
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            if ofs.is_reg():
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
-                base_loc = r.ip
-                ofs = imm(0)
-            else:
-                assert ofs.value % 4 == 0
-            self.mc.VLDR(res.value, base_loc.value, ofs.value)
-        elif size.value == 4:
-            if ofs.is_imm():
-                self.mc.LDR_ri(res.value, base_loc.value, ofs.value)
-            else:
-                self.mc.LDR_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 2:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSH_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSH_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 1:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSB_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSB_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_rr(res.value, base_loc.value, ofs.value)
-        else:
-            assert 0
+        scale = get_scale(size.value)
+        self._load_from_mem(res, base_loc, ofs, imm(scale), signed, fcond)
         return fcond
 
     emit_op_getfield_raw = emit_op_getfield_gc
@@ -609,39 +572,22 @@
     def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond):
         (base_loc, index_loc, res_loc,
             ofs_loc, ofs, itemsize, fieldsize) = arglocs
-        self.mc.gen_load_int(r.ip.value, itemsize.value)
-        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        scale = get_scale(fieldsize.value)
+        tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+        assert not save
+        self.mc.gen_load_int(tmploc.value, itemsize.value)
+        self.mc.MUL(tmploc.value, index_loc.value, tmploc.value)
         descr = op.getdescr()
         assert isinstance(descr, InteriorFieldDescr)
         signed = descr.fielddescr.is_field_signed()
         if ofs.value > 0:
             if ofs_loc.is_imm():
-                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+                self.mc.ADD_ri(tmploc.value, tmploc.value, ofs_loc.value)
             else:
-                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
-
-        if fieldsize.value == 8:
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            assert res_loc.is_vfp_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, 0)
-        elif fieldsize.value == 4:
-            self.mc.LDR_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 2:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 1:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value, r.ip.value)
-        else:
-            assert 0
-
+                self.mc.ADD_rr(tmploc.value, tmploc.value, ofs_loc.value)
+        ofs_loc = tmploc
+        self._load_from_mem(res_loc, base_loc, ofs_loc,
+                                imm(scale), signed, fcond)
         return fcond
 
     def emit_op_setinteriorfield_gc(self, op, arglocs, regalloc, fcond):
@@ -731,33 +677,73 @@
             self.mc.ADD_ri(r.ip.value, ofs_loc.value, imm=ofs.value)
             ofs_loc = r.ip
         #
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def _load_from_mem(self, res_loc, base_loc, ofs_loc, scale,
                                             signed=False, fcond=c.AL):
         if scale.value == 3:
             assert res_loc.is_vfp_reg()
-            assert ofs_loc.is_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, cond=fcond)
+            # vldr only supports imm offsets
+            # if the offset is in a register we add it to the base and use a
+            # tmp reg
+            if ofs_loc.is_reg():
+                tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+                assert not save
+                self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
+                base_loc = tmploc
+                ofs_loc = imm(0)
+            else:
+                assert ofs_loc.is_imm()
+                # if the ofset is too large for an imm we add it to the base and use an
+                # offset of 0
+                if check_imm_arg(ofs_loc.value, VMEM_imm_size):
+                    tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+                    assert not save
+                    self.mc.gen_load_int(tmploc.value, ofs_loc.value)
+                    self.mc.ADD_rr(tmploc.value, base_loc.value, tmploc.value)
+                    base_loc = tmploc
+                    ofs_loc = imm(0)
+                else:  # sanity check
+                    assert ofs_loc.value % 4 == 0
+            self.mc.VLDR(res_loc.value, base_loc.value, ofs_loc.value, cond=fcond)
         elif scale.value == 2:
-            self.mc.LDR_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.LDR_ri(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.LDR_rr(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 1:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         elif scale.value == 0:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         else:
             assert 0
 
@@ -770,7 +756,7 @@
         # no base offset
         assert ofs.value == 0
         signed = op.getdescr().is_item_signed()
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def emit_op_strlen(self, op, arglocs, regalloc, fcond):
@@ -993,7 +979,7 @@
                 assert result_loc.is_vfp_reg()
                 # we always have a register here, since we have to sync them
                 # before call_assembler
-                self.mc.VLDR(result_loc.value, r.r0.value, imm=ofs)
+                self.load_reg(self.mc, result_loc, r.r0, ofs=ofs)
             else:
                 assert result_loc is r.r0
                 ofs = self.cpu.unpack_arraydescr(descr)

From noreply at buildbot.pypy.org  Sun Jun  2 22:31:03 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun,  2 Jun 2013 22:31:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20130602203103.D6EBA1C01E5@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64721:d3b498962380
Date: 2013-06-02 15:23 -0500
http://bitbucket.org/pypy/pypy/changeset/d3b498962380/

Log:	merge heads

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -303,7 +303,7 @@
         return _absolute_import(space, modulename, baselevel,
                                 fromlist_w, tentative)
     finally:
-        lock.release_lock()
+        lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
 def absolute_import_try(space, modulename, baselevel, fromlist_w):
@@ -788,10 +788,10 @@
             self.lockowner = me
         self.lockcounter += 1
 
-    def release_lock(self):
+    def release_lock(self, silent_after_fork):
         me = self.space.getexecutioncontext()   # used as thread ident
         if self.lockowner is not me:
-            if self.lockowner is None:
+            if self.lockowner is None and silent_after_fork:
                 # Too bad.  This situation can occur if a fork() occurred
                 # with the import lock held, and we're the child.
                 return
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -177,7 +177,7 @@
 
 def release_lock(space):
     if space.config.objspace.usemodules.thread:
-        importing.getimportlock(space).release_lock()
+        importing.getimportlock(space).release_lock(silent_after_fork=False)
 
 def reinit_lock(space):
     if space.config.objspace.usemodules.thread:
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -352,9 +352,8 @@
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):
-                # convert string to unicode explicitely here
-                from pypy.objspace.std.unicodetype import plain_str2unicode
-                r = plain_str2unicode(self.space, r)
+                # convert string to unicode using the default encoding
+                r = self.space.unicode_w(self.space.wrap(r))
             prec = self.prec
             if prec == -1 and self.width == 0:
                 # fast path
@@ -509,12 +508,10 @@
             result = formatter.format()
         except NeedUnicodeFormattingError:
             # fall through to the unicode case
-            from pypy.objspace.std.unicodetype import plain_str2unicode
-            fmt = plain_str2unicode(space, fmt)
+            pass
         else:
             return space.wrap(result)
-    else:
-        fmt = space.unicode_w(w_fmt)
+    fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
     return space.wrap(result)
diff --git a/pypy/objspace/std/test/test_stringobject.py b/pypy/objspace/std/test/test_stringobject.py
--- a/pypy/objspace/std/test/test_stringobject.py
+++ b/pypy/objspace/std/test/test_stringobject.py
@@ -530,6 +530,12 @@
                 del sys.modules[module_name]
             temp_sys.setdefaultencoding('utf-8')
             assert u''.join(['\xc3\xa1']) == u'\xe1'
+            #
+            assert ('\xc3\xa1:%s' % u'\xe2') == u'\xe1:\xe2'
+            class Foo(object):
+                def __repr__(self):
+                    return '\xc3\xa2'
+            assert u'\xe1:%r' % Foo() == u'\xe1:\xe2'
         finally:
             temp_sys.setdefaultencoding(old_encoding)
             sys.modules.update(self.original_modules)
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -13,22 +13,6 @@
     from pypy.objspace.std.unicodeobject import W_UnicodeObject
     return W_UnicodeObject(uni)
 
-def plain_str2unicode(space, s):
-    try:
-        return unicode(s)
-    except UnicodeDecodeError:
-        for i in range(len(s)):
-            if ord(s[i]) > 127:
-                raise OperationError(
-                    space.w_UnicodeDecodeError,
-                    space.newtuple([
-                    space.wrap('ascii'),
-                    space.wrap(s),
-                    space.wrap(i),
-                    space.wrap(i+1),
-                    space.wrap("ordinal not in range(128)")]))
-        assert False, "unreachable"
-
 
 unicode_capitalize = SMM('capitalize', 1,
                          doc='S.capitalize() -> unicode\n\nReturn a'
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -17,7 +17,7 @@
 def get_description(atypes, rtype):
     p = lltype.malloc(CIF_DESCRIPTION, len(atypes),
                       flavor='raw', immortal=True)
-    p.abi = 42
+    p.abi = 1     # default
     p.nargs = len(atypes)
     p.rtype = rtype
     p.atypes = lltype.malloc(FFI_TYPE_PP.TO, len(atypes),
diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -5,11 +5,7 @@
 from rpython.translator.platform.bsd import BSD
 
 class OpenBSD(BSD):
-    if os.environ.get("CC") is None:
-        DEFAULT_CC = "cc"
-    else:
-        DEFAULT_CC = os.environ.get("CC")
-
+    DEFAULT_CC = "cc"
     name = "openbsd"
 
     link_flags = os.environ.get("LDFLAGS", "").split() + ['-pthread']

From noreply at buildbot.pypy.org  Sun Jun  2 22:56:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 22:56:08 +0200 (CEST)
Subject: [pypy-commit] buildbot default: Call the handlers in a parallel
 thread, allowing the http request from
Message-ID: <20130602205608.D324A1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r824:861d54ea624e
Date: 2013-06-02 22:55 +0200
http://bitbucket.org/pypy/buildbot/changeset/861d54ea624e/

Log:	Call the handlers in a parallel thread, allowing the http request
	from bitbucket to get immediately an answer. (For tests, we don't
	use this parallel thread.)

diff --git a/bbhook/hook.py b/bbhook/hook.py
--- a/bbhook/hook.py
+++ b/bbhook/hook.py
@@ -3,6 +3,8 @@
 import subprocess
 import sys
 import time
+import thread, Queue
+import traceback
 
 from .main import app
 from . import scm
@@ -39,7 +41,24 @@
         yield commit
 
 
-def handle(payload, test=False):
+
+def _handle_thread():
+    while True:
+        local_repo = payload = None
+        try:
+            local_repo, payload = queue.get()
+            _do_handle(local_repo, payload)
+        except:
+            traceback.print_exc()
+            print >> sys.stderr, 'payload:'
+            pprint.pprint(payload, sys.stderr)
+            print >> sys.stderr
+
+queue = Queue.Queue()
+thread.start_new_thread(_handle_thread, ())
+
+
+def handle(payload, test=True):
     path = payload['repository']['absolute_url']
     owner = payload['repository']['owner']
     local_repo = app.config['LOCAL_REPOS'].join(path)
@@ -47,6 +66,12 @@
     if not check_for_local_repo(local_repo, remote_repo, owner):
         print >> sys.stderr, 'Ignoring unknown repo', path
         return
+    if test:
+        _do_handle(local_repo, payload, test)
+    else:
+        queue.put((local_repo, payload))
+
+def _do_handle(local_repo, payload, test=False):
     scm.hg('pull', '-R', local_repo)
     for commit in get_commits(payload):
         for handler in HANDLERS:

From noreply at buildbot.pypy.org  Sun Jun  2 22:59:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Jun 2013 22:59:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Typo
Message-ID: <20130602205930.58C831C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64722:a5b60f700b88
Date: 2013-06-02 22:58 +0200
http://bitbucket.org/pypy/pypy/changeset/a5b60f700b88/

Log:	Typo

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -2,7 +2,7 @@
 PyPy on Windows
 ===============
 
-Pypy is supported on Windows platforms, starting with Windows 2000.
+PyPy is supported on Windows platforms, starting with Windows 2000.
 The following text gives some hints about how to translate the PyPy
 interpreter.
 

From noreply at buildbot.pypy.org  Sun Jun  2 23:11:15 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Jun 2013 23:11:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a comment
Message-ID: <20130602211115.6897B1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r64723:f4dc731c4c03
Date: 2013-06-03 05:10 +0800
http://bitbucket.org/pypy/pypy/changeset/f4dc731c4c03/

Log:	add a comment

diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,6 @@
+
+.. comment: this document is very incomplete, should we generate it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================

From noreply at buildbot.pypy.org  Sun Jun  2 23:11:16 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Jun 2013 23:11:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20130602211116.EC3671C01E5@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r64724:b93ade5abad3
Date: 2013-06-03 05:10 +0800
http://bitbucket.org/pypy/pypy/changeset/b93ade5abad3/

Log:	merge

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -2,7 +2,7 @@
 PyPy on Windows
 ===============
 
-Pypy is supported on Windows platforms, starting with Windows 2000.
+PyPy is supported on Windows platforms, starting with Windows 2000.
 The following text gives some hints about how to translate the PyPy
 interpreter.
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -303,7 +303,7 @@
         return _absolute_import(space, modulename, baselevel,
                                 fromlist_w, tentative)
     finally:
-        lock.release_lock()
+        lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
 def absolute_import_try(space, modulename, baselevel, fromlist_w):
@@ -788,10 +788,10 @@
             self.lockowner = me
         self.lockcounter += 1
 
-    def release_lock(self):
+    def release_lock(self, silent_after_fork):
         me = self.space.getexecutioncontext()   # used as thread ident
         if self.lockowner is not me:
-            if self.lockowner is None:
+            if self.lockowner is None and silent_after_fork:
                 # Too bad.  This situation can occur if a fork() occurred
                 # with the import lock held, and we're the child.
                 return
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -177,7 +177,7 @@
 
 def release_lock(space):
     if space.config.objspace.usemodules.thread:
-        importing.getimportlock(space).release_lock()
+        importing.getimportlock(space).release_lock(silent_after_fork=False)
 
 def reinit_lock(space):
     if space.config.objspace.usemodules.thread:
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -8,6 +8,7 @@
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.opassembler import ResOpAssembler
 from rpython.jit.backend.arm.regalloc import (Regalloc,
     CoreRegisterManager, check_imm_arg, VFPRegisterManager,
@@ -961,7 +962,7 @@
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VLDR(target.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
@@ -982,7 +983,7 @@
             return self._store_core_reg(mc, source, base, ofs, cond, helper)
 
     def _store_vfp_reg(self, mc, source, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VSTR(source.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -52,6 +52,8 @@
 
     def _push_stack_args(self, stack_args, on_stack):
         assert on_stack % 8 == 0
+        if on_stack == 0:
+            return
         self._adjust_sp(-on_stack)
         self.current_sp = on_stack
         ofs = 0
@@ -71,7 +73,7 @@
             else:
                 self.mc.gen_load_int(r.ip.value, n)
                 self.mc.ADD_rr(r.sp.value, r.sp.value, r.ip.value)
-        else:
+        elif n < 0:
             n = abs(n)
             if check_imm_arg(n):
                 self.mc.SUB_ri(r.sp.value, r.sp.value, n)
diff --git a/rpython/jit/backend/arm/helper/regalloc.py b/rpython/jit/backend/arm/helper/regalloc.py
--- a/rpython/jit/backend/arm/helper/regalloc.py
+++ b/rpython/jit/backend/arm/helper/regalloc.py
@@ -4,7 +4,10 @@
 from rpython.jit.metainterp.history import ConstInt
 from rpython.rlib.objectmodel import we_are_translated
 
-def check_imm_arg(arg, size=0xFF, allow_zero=True):
+VMEM_imm_size=0x3FC
+default_imm_size=0xFF
+
+def check_imm_arg(arg, size=default_imm_size, allow_zero=True):
     assert not isinstance(arg, ConstInt)
     if not we_are_translated():
         if not isinstance(arg, int):
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -15,6 +15,7 @@
                                                 gen_emit_unary_float_op,
                                                 saved_registers)
 from rpython.jit.backend.arm.helper.regalloc import check_imm_arg
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.jump import remap_frame_layout
 from rpython.jit.backend.arm.regalloc import TempBox
@@ -23,6 +24,7 @@
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import InteriorFieldDescr
 from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler
+from rpython.jit.backend.llsupport.regalloc import get_scale
 from rpython.jit.metainterp.history import (Box, AbstractFailDescr,
                                             INT, FLOAT, REF)
 from rpython.jit.metainterp.history import TargetToken
@@ -559,47 +561,8 @@
     def emit_op_getfield_gc(self, op, arglocs, regalloc, fcond):
         base_loc, ofs, res, size = arglocs
         signed = op.getdescr().is_field_signed()
-        if size.value == 8:
-            assert res.is_vfp_reg()
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            if ofs.is_reg():
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
-                base_loc = r.ip
-                ofs = imm(0)
-            else:
-                assert ofs.value % 4 == 0
-            self.mc.VLDR(res.value, base_loc.value, ofs.value)
-        elif size.value == 4:
-            if ofs.is_imm():
-                self.mc.LDR_ri(res.value, base_loc.value, ofs.value)
-            else:
-                self.mc.LDR_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 2:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSH_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSH_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 1:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSB_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSB_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_rr(res.value, base_loc.value, ofs.value)
-        else:
-            assert 0
+        scale = get_scale(size.value)
+        self._load_from_mem(res, base_loc, ofs, imm(scale), signed, fcond)
         return fcond
 
     emit_op_getfield_raw = emit_op_getfield_gc
@@ -609,39 +572,22 @@
     def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond):
         (base_loc, index_loc, res_loc,
             ofs_loc, ofs, itemsize, fieldsize) = arglocs
-        self.mc.gen_load_int(r.ip.value, itemsize.value)
-        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        scale = get_scale(fieldsize.value)
+        tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+        assert not save
+        self.mc.gen_load_int(tmploc.value, itemsize.value)
+        self.mc.MUL(tmploc.value, index_loc.value, tmploc.value)
         descr = op.getdescr()
         assert isinstance(descr, InteriorFieldDescr)
         signed = descr.fielddescr.is_field_signed()
         if ofs.value > 0:
             if ofs_loc.is_imm():
-                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+                self.mc.ADD_ri(tmploc.value, tmploc.value, ofs_loc.value)
             else:
-                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
-
-        if fieldsize.value == 8:
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            assert res_loc.is_vfp_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, 0)
-        elif fieldsize.value == 4:
-            self.mc.LDR_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 2:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 1:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value, r.ip.value)
-        else:
-            assert 0
-
+                self.mc.ADD_rr(tmploc.value, tmploc.value, ofs_loc.value)
+        ofs_loc = tmploc
+        self._load_from_mem(res_loc, base_loc, ofs_loc,
+                                imm(scale), signed, fcond)
         return fcond
 
     def emit_op_setinteriorfield_gc(self, op, arglocs, regalloc, fcond):
@@ -731,33 +677,73 @@
             self.mc.ADD_ri(r.ip.value, ofs_loc.value, imm=ofs.value)
             ofs_loc = r.ip
         #
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def _load_from_mem(self, res_loc, base_loc, ofs_loc, scale,
                                             signed=False, fcond=c.AL):
         if scale.value == 3:
             assert res_loc.is_vfp_reg()
-            assert ofs_loc.is_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, cond=fcond)
+            # vldr only supports imm offsets
+            # if the offset is in a register we add it to the base and use a
+            # tmp reg
+            if ofs_loc.is_reg():
+                tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+                assert not save
+                self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
+                base_loc = tmploc
+                ofs_loc = imm(0)
+            else:
+                assert ofs_loc.is_imm()
+                # if the ofset is too large for an imm we add it to the base and use an
+                # offset of 0
+                if check_imm_arg(ofs_loc.value, VMEM_imm_size):
+                    tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+                    assert not save
+                    self.mc.gen_load_int(tmploc.value, ofs_loc.value)
+                    self.mc.ADD_rr(tmploc.value, base_loc.value, tmploc.value)
+                    base_loc = tmploc
+                    ofs_loc = imm(0)
+                else:  # sanity check
+                    assert ofs_loc.value % 4 == 0
+            self.mc.VLDR(res_loc.value, base_loc.value, ofs_loc.value, cond=fcond)
         elif scale.value == 2:
-            self.mc.LDR_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.LDR_ri(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.LDR_rr(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 1:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         elif scale.value == 0:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         else:
             assert 0
 
@@ -770,7 +756,7 @@
         # no base offset
         assert ofs.value == 0
         signed = op.getdescr().is_item_signed()
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def emit_op_strlen(self, op, arglocs, regalloc, fcond):
@@ -993,7 +979,7 @@
                 assert result_loc.is_vfp_reg()
                 # we always have a register here, since we have to sync them
                 # before call_assembler
-                self.mc.VLDR(result_loc.value, r.r0.value, imm=ofs)
+                self.load_reg(self.mc, result_loc, r.r0, ofs=ofs)
             else:
                 assert result_loc is r.r0
                 ofs = self.cpu.unpack_arraydescr(descr)

From noreply at buildbot.pypy.org  Mon Jun  3 09:05:26 2013
From: noreply at buildbot.pypy.org (andrewsmedina)
Date: Mon,  3 Jun 2013 09:05:26 +0200 (CEST)
Subject: [pypy-commit] lang-js default: fixed translation.
Message-ID: <20130603070526.0BCD91C0619@cobra.cs.uni-duesseldorf.de>

Author: Andrews Medina <andrewsmedina at gmail.com>
Branch: 
Changeset: r396:e2275b282ab9
Date: 2013-05-29 00:04 -0300
http://bitbucket.org/pypy/lang-js/changeset/e2275b282ab9/

Log:	fixed translation.

diff --git a/js/builtins/array.py b/js/builtins/array.py
--- a/js/builtins/array.py
+++ b/js/builtins/array.py
@@ -56,10 +56,14 @@
     o = this.ToObject()
     from_index = get_arg(args, 0).ToUInt32()
     to_index = get_arg(args, 1).ToUInt32()
-    n = []
-    for k in xrange(from_index, to_index):
-        n.append(o.get(unicode(str(k))))
-    return _w(n)
+    from js.object_space import object_space
+    n = object_space.new_array(length=_w(to_index-from_index))
+    from js.jsobj import put_property
+    index = 0
+    for item in xrange(from_index, to_index):
+        put_property(n, unicode(str(index)), o.get(unicode(str(item))))
+        index += 1
+    return n
 
 
 # 15.4.4.7

From noreply at buildbot.pypy.org  Mon Jun  3 10:45:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  3 Jun 2013 10:45:19 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Forgot about one aspect of read
 barriers. Now they are no longer
Message-ID: <20130603084519.1335A1C010B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r59:14020650fc09
Date: 2013-06-03 10:45 +0200
http://bitbucket.org/pypy/stmgc/changeset/14020650fc09/

Log:	Forgot about one aspect of read barriers. Now they are no longer
	"extremely cheap". I hope they can still be "very cheap".

diff --git a/c3/doc-stmgc.txt b/c3/doc-stmgc.txt
--- a/c3/doc-stmgc.txt
+++ b/c3/doc-stmgc.txt
@@ -41,10 +41,10 @@
 of keeping multiple copies with revision numbers to track them.
 Moreover, "read barriers" and "write barriers" are used by the C program
 calling into this library in order to be sure that it is accessing the
-right version of the object.  In the current variant we can have
-extremely cheap read barriers, which are definitely a major speed
-improvement over the previous variants (and, as far as I know, over most
-of the other existing STMs).
+right version of the object.  In the current variant we can have very
+cheap read barriers, which are definitely a major speed improvement over
+the previous variants (and, as far as I know, over most of the other
+existing STMs).
 
 
 ----------------------
@@ -154,22 +154,34 @@
 
 Point 3 is essential for performance: we want most importantly a read
 barrier that doesn't trigger for the cases described above.  The read
-barrier needs to check if a pointer P references a public copy that
-was outdated by a future revision.  This is an easy check, which can
-be implemented by checking a flag in the header of the copy.  In all
-the common cases, this flag is not set, and no actual call needs to
-be done.
+barrier has two purposes: it needs to check that a given pointer P
+references an object that is not outdated already; and it needs to
+record the pointer in the "read set" of the current transaction.
 
-The case of the write barrier is similar, but differs in the check we
-need to do.  We need to do a call if the object is not already private.
-For performance reasons, "being private" is not directly a flag in the
-object, because when a transaction commits, we don't want to have to
-walk all private objects to change this flag.  Instead, private objects
-have a precise negative odd number in their `h_revision` field, called
-the "local revision identifier".  When a transaction commits, we change
-the value of the local revision identifier, and all previously-private
-objects become automatically protected.  So the write barrier fast-path
-checks if the `h_revision` is equal from the local revision identifier.
+The first check is easy, and can be implemented by checking a flag in
+the header of the copy.  In all the common cases, this flag is not set,
+and no actual call needs to be done.
+
+The recording in the read set is a bit more annoying.  We need to
+maintain a thread-local *set* of all accessed objects, but we don't care
+about the order or recording the occasional duplicate.  Moreover we
+don't need to record the private objects; but we do need all other
+protected objects, as well as public objects.  The best approach is
+probably to have a quick check "is it definitely recorded already?"
+inline, and do the call if the check fails.  It needs careful design to
+be done in only a few CPU instructions, but it should be possible.
+
+The case of the write barrier is similar to the first half of the read
+barrier, but differs in the check we need to do.  We need to do a call
+if the object is not already private.  For performance reasons, "being
+private" is not directly a flag in the object, because when a
+transaction commits, we don't want to have to walk all private objects
+to change this flag.  Instead, private objects have a precise negative
+odd number in their `h_revision` field, called the "local revision
+identifier".  When a transaction commits, we change the value of the
+local revision identifier, and all previously-private objects become
+automatically protected.  So the write barrier fast-path checks if the
+`h_revision` is equal from the local revision identifier.
 
 
 The extendable timestamp model

From noreply at buildbot.pypy.org  Mon Jun  3 11:00:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  3 Jun 2013 11:00:17 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Saved the day by moving the cache
 check inline in 4 cpu instructions.
Message-ID: <20130603090017.8AD381C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r60:39d1f410f3ce
Date: 2013-06-03 10:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/39d1f410f3ce/

Log:	Saved the day by moving the cache check inline in 4 cpu
	instructions.

diff --git a/c3/doc-stmgc.txt b/c3/doc-stmgc.txt
--- a/c3/doc-stmgc.txt
+++ b/c3/doc-stmgc.txt
@@ -158,18 +158,25 @@
 references an object that is not outdated already; and it needs to
 record the pointer in the "read set" of the current transaction.
 
-The first check is easy, and can be implemented by checking a flag in
-the header of the copy.  In all the common cases, this flag is not set,
-and no actual call needs to be done.
+The first check is easy, and could be implemented by checking a flag in
+the header of the copy.  But the recording in the read set is a bit more
+annoying.  We need to maintain a thread-local *set* of all accessed
+objects, but we don't care about the order or recording the occasional
+duplicate.  Moreover we don't need to record the private objects; but we
+do need all other protected objects, as well as public objects.  The
+best approach is probably to have a quick check "is it definitely
+recorded already?" inline, and do the call if the check fails.  It needs
+careful design to be done in only a few CPU instructions, but it should
+be possible.
 
-The recording in the read set is a bit more annoying.  We need to
-maintain a thread-local *set* of all accessed objects, but we don't care
-about the order or recording the occasional duplicate.  Moreover we
-don't need to record the private objects; but we do need all other
-protected objects, as well as public objects.  The best approach is
-probably to have a quick check "is it definitely recorded already?"
-inline, and do the call if the check fails.  It needs careful design to
-be done in only a few CPU instructions, but it should be possible.
+(Code like this compiles to 4 instructions in the fast path:
+
+  __thread char *read_barrier_cache;  /* thread-local cache of 64KB */
+
+  if (__builtin_expect(*(gcptr *)(read_barrier_cache + (((long)x) & 65535))
+                       != x, 0))
+      x = call_read_barrier(x);
+)
 
 The case of the write barrier is similar to the first half of the read
 barrier, but differs in the check we need to do.  We need to do a call

From noreply at buildbot.pypy.org  Mon Jun  3 11:33:44 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon,  3 Jun 2013 11:33:44 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: changed wrapping mechanism of
 interpreterProxy functions(ipfs):
Message-ID: <20130603093344.93DBA1C1527@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r435:9471f0ad6d08
Date: 2013-06-03 11:33 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/9471f0ad6d08/

Log:	changed wrapping mechanism of interpreterProxy functions(ipfs):
	signature is now given from (int, float, oop, ...) and
	unwrapping/wrapping/default return values are now supplied by the
	wrapping method failure state is set by the wrapper, not
	each function added list of all missing ipfs

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -8,6 +8,8 @@
 #   plugin setInterpreter: proxy.
 #   (plugin respondsTo: #initialiseModule) ifTrue:[plugin initialiseModule].
 #   plugin perform: primitiveName asSymbol.
+import inspect
+
 from rpython.rlib.entrypoint import entrypoint
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.exports import export_struct
@@ -20,49 +22,92 @@
 sqInt = rffi.INT
 sqLong = rffi.LONG
 sqDouble = rffi.DOUBLE
-sqIntArray = rffi.CArray(sqInt)
+sqIntArrayPtr = Ptr(rffi.CArray(sqInt))
 
 major = minor = 0
 functions = []
 
-def expose_on_virtual_machine_proxy(signature, minor=0, major=1):
-    f_ptr = Ptr(signature)
+oop = object()
+
+class ProxyFunctionFailed(error.PrimitiveFailedError):
+    pass
+
+def expose_on_virtual_machine_proxy(unwrap_spec, result_type, minor=0, major=1):
+    mapping = {oop: sqInt, int: sqInt, list: sqIntArrayPtr, bool: sqInt, float: sqDouble}
+    f_ptr = Ptr(FuncType([mapping[spec] for spec in unwrap_spec], mapping[result_type]))
     if minor < minor:
         minor = minor
     if major < major:
         major = major
     def decorator(func):
-        functions.append(("c_" + func.func_name, f_ptr, func))
-        return func
+        len_unwrap_spec = len(unwrap_spec)
+        assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1,
+                "wrong number of arguments")
+        unrolling_unwrap_spec = unrolling_iterable(enumerate(unwrap_spec))
+        def wrapped(*c_arguments):
+            assert len_unwrap_spec == len(c_arguments)
+            args = ()
+            try:
+                for i, spec in unrolling_unwrap_spec:
+                    c_arg = c_arguments[i]
+                    if spec is oop:
+                        args += (IProxy.oop_to_object(c_arg), )
+                    else:
+                        args += (c_arg, )
+                result = func(*args)
+                if result_type is oop:
+                    assert isinstance(result, model.W_Object)
+                    return IProxy.object_to_oop(result)
+                elif result_type is list:
+                    assert isinstance(result, list)
+                    return IProxy.list_to_carray(result)
+                elif result_type in (int, float, bool):
+                    assert isinstance(result, result_type)
+                    return result
+                else:
+                    return result
+            except error.PrimitiveFailedError:
+                IProxy.success_flag = False
+                if mapping[result_type] is sqInt:
+                    return 0
+                elif mapping[result_type] is sqDouble:
+                    return 0.0
+                elif mapping[result_type] is sqIntArrayPtr:
+                    return rffi.cast(sqIntArrayPtr, 0)
+                else:
+                    raise NotImplementedError(
+                        "InterpreterProxy: unknown result_type %s" % (result_type, ))
+        functions.append(("c_" + func.func_name, f_ptr, wrapped))
+        return wrapped
     return decorator
 
- at expose_on_virtual_machine_proxy(FuncType([], sqInt))
+ at expose_on_virtual_machine_proxy([], int)
 def minorVersion():
     return minor
 
- at expose_on_virtual_machine_proxy(FuncType([], sqInt))
+ at expose_on_virtual_machine_proxy([], int)
 def majorVersion():
     return major
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+ at expose_on_virtual_machine_proxy([int], int)
 def pop(nItems):
     IProxy.s_frame.pop_n(nItems)
     return 0
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt, sqInt], sqInt))
-def popthenPush(nItems, oop):
+ at expose_on_virtual_machine_proxy([int, oop], int)
+def popthenPush(nItems, w_object):
     s_frame = IProxy.s_frame
     s_frame.pop_n(nItems)
-    s_frame.push(IProxy.oop_to_object(oop))
+    s_frame.push(w_object)
     return 0
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
-def push(oop):
+ at expose_on_virtual_machine_proxy([oop], int)
+def push(w_object):
     s_frame = IProxy.s_frame
-    s_frame.push(IProxy.oop_to_object(oop))
+    s_frame.push(w_object)
     return 0
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+ at expose_on_virtual_machine_proxy([bool], int)
 def pushBool(trueOrFalse):
     s_frame = IProxy.s_frame
     if trueOrFalse:
@@ -71,71 +116,60 @@
         s_frame.push(IProxy.interp.space.w_false)
     return 0
 
- at expose_on_virtual_machine_proxy(FuncType([sqDouble], sqInt))
+ at expose_on_virtual_machine_proxy([float], int)
 def pushFloat(f):
     s_frame = IProxy.s_frame
     s_frame.push(IProxy.space.wrap_float(f))
     return 0
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+ at expose_on_virtual_machine_proxy([int], int)
 def pushInteger(n):
     s_frame = IProxy.s_frame
     s_frame.push(IProxy.space.wrap_int(n))
     return 0
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqDouble))
+ at expose_on_virtual_machine_proxy([int], float)
 def stackFloatValue(offset):
     s_frame = IProxy.s_frame
     f = s_frame.peek(offset)
     if isinstance(f, model.W_Float):
         return f.value
     else:
-        IProxy.successFlag = False
-        return 0.0
+        raise ProxyFunctionFailed
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+ at expose_on_virtual_machine_proxy([int], int)
 def stackIntegerValue(offset):
     s_frame = IProxy.s_frame
     n = s_frame.peek(offset)
-    try:
-        return IProxy.space.unwrap_int(n)
-    except error.PrimitiveFailedError:
-        IProxy.successFlag = False
-        return 0
+    return IProxy.space.unwrap_int(n)
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+ at expose_on_virtual_machine_proxy([int], oop)
 def stackObjectValue(offset):
     s_frame = IProxy.s_frame
     w_object = s_frame.peek(offset)
     if not isinstance(w_object, model.W_SmallInteger):
-        return IProxy.object_to_oop(w_object)
-    IProxy.successFlag = False
-    return 0
+        return w_object
+    raise ProxyFunctionFailed
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
+ at expose_on_virtual_machine_proxy([int], oop)
 def stackValue(offset):
     s_frame = IProxy.s_frame
-    return IProxy.object_to_oop(s_frame.peek(offset))
+    return s_frame.peek(offset)
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
-def argumentCountOf(methodOOP):
-    w_method = IProxy.oop_to_object(methodOOP)
+ at expose_on_virtual_machine_proxy([oop], int)
+def argumentCountOf(w_method):
     if isinstance(w_method, model.W_CompiledMethod):
         return w_method.argsize
-    IProxy.successFlag = False
-    return 0
+    raise ProxyFunctionFailed
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], Ptr(sqIntArray)))
-def arrayValueOf(oop):
-    w_array = IProxy.oop_to_object(oop)
-    if isinstance(w_array, model.W_WordsObject) or isinstance(w_array, model.W_BytesObject):
+ at expose_on_virtual_machine_proxy([oop], list)
+def arrayValueOf(w_array):
+    if w_array.is_array_object():
         raise NotImplementedError
-    IProxy.successFlag = False
-    return rffi.cast(Ptr(sqIntArray), 0)
+    raise ProxyFunctionFailed
 
- at expose_on_virtual_machine_proxy(FuncType([sqInt], sqInt))
-def byteSizeOf(oop):
-    w_object = IProxy.oop_to_object(oop)
+ at expose_on_virtual_machine_proxy([oop], int)
+def byteSizeOf(w_object):
     s_class = w_object.shadow_of_my_class(IProxy.space)
     size = s_class.instsize()
     if s_class.isvariable():
@@ -144,6 +178,263 @@
         size *= 4
     return size
 
+ at expose_on_virtual_machine_proxy([int, oop], list)
+def fetchArrayofObject(fieldIndex, w_object):
+    # arrayOop := self fetchPointer: fieldIndex ofObject: objectPointer.
+    # ^ self arrayValueOf: arrayOop
+    w_array = w_object.fetch(IProxy.space, fieldIndex)
+    if w_array.is_array_object():
+        raise NotImplementedError
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], oop)
+def fetchClassOf(w_object):
+    w_class = w_object.getclass(IProxy.space)
+    return w_class
+#     sqInt  (*fetchClassOf)(sqInt oop);
+#     double (*fetchFloatofObject)(sqInt fieldIndex, sqInt objectPointer);
+#     sqInt  (*fetchIntegerofObject)(sqInt fieldIndex, sqInt objectPointer);
+#     sqInt  (*fetchPointerofObject)(sqInt fieldIndex, sqInt oop);
+#     sqInt  (*obsoleteDontUseThisFetchWordofObject)(sqInt fieldFieldIndex, sqInt oop);
+#     void  *(*firstFixedField)(sqInt oop);
+#     void  *(*firstIndexableField)(sqInt oop);
+#     sqInt  (*literalofMethod)(sqInt offset, sqInt methodPointer);
+#     sqInt  (*literalCountOf)(sqInt methodPointer);
+#     sqInt  (*methodArgumentCount)(void);
+#     sqInt  (*methodPrimitiveIndex)(void);
+#     sqInt  (*primitiveIndexOf)(sqInt methodPointer);
+#     sqInt  (*sizeOfSTArrayFromCPrimitive)(void *cPtr);
+#     sqInt  (*slotSizeOf)(sqInt oop);
+#     sqInt  (*stObjectat)(sqInt array, sqInt fieldIndex);
+#     sqInt  (*stObjectatput)(sqInt array, sqInt fieldIndex, sqInt value);
+#     sqInt  (*stSizeOf)(sqInt oop);
+#     sqInt  (*storeIntegerofObjectwithValue)(sqInt fieldIndex, sqInt oop, sqInt integer);
+#     sqInt  (*storePointerofObjectwithValue)(sqInt fieldIndex, sqInt oop, sqInt valuePointer);
+
+#     /* InterpreterProxy methodsFor: 'testing' */
+
+#     sqInt (*isKindOf)(sqInt oop, char *aString);
+#     sqInt (*isMemberOf)(sqInt oop, char *aString);
+#     sqInt (*isBytes)(sqInt oop);
+#     sqInt (*isFloatObject)(sqInt oop);
+#     sqInt (*isIndexable)(sqInt oop);
+#     sqInt (*isIntegerObject)(sqInt objectPointer);
+#     sqInt (*isIntegerValue)(sqInt intValue);
+#     sqInt (*isPointers)(sqInt oop);
+#     sqInt (*isWeak)(sqInt oop);
+#     sqInt (*isWords)(sqInt oop);
+#     sqInt (*isWordsOrBytes)(sqInt oop);
+
+#     /* InterpreterProxy methodsFor: 'converting' */
+
+#     sqInt  (*booleanValueOf)(sqInt obj);
+#     sqInt  (*checkedIntegerValueOf)(sqInt intOop);
+#     sqInt  (*floatObjectOf)(double aFloat);
+#     double (*floatValueOf)(sqInt oop);
+#     sqInt  (*integerObjectOf)(sqInt value);
+#     sqInt  (*integerValueOf)(sqInt oop);
+#     sqInt  (*positive32BitIntegerFor)(sqInt integerValue);
+#     sqInt  (*positive32BitValueOf)(sqInt oop);
+
+#     /* InterpreterProxy methodsFor: 'special objects' */
+
+#     sqInt (*characterTable)(void);
+#     sqInt (*displayObject)(void);
+#     sqInt (*falseObject)(void);
+#     sqInt (*nilObject)(void);
+#     sqInt (*trueObject)(void);
+
+#     /* InterpreterProxy methodsFor: 'special classes' */
+
+#     sqInt (*classArray)(void);
+#     sqInt (*classBitmap)(void);
+#     sqInt (*classByteArray)(void);
+#     sqInt (*classCharacter)(void);
+#     sqInt (*classFloat)(void);
+#     sqInt (*classLargePositiveInteger)(void);
+#     sqInt (*classPoint)(void);
+#     sqInt (*classSemaphore)(void);
+#     sqInt (*classSmallInteger)(void);
+#     sqInt (*classString)(void);
+
+#     /* InterpreterProxy methodsFor: 'instance creation' */
+
+#     sqInt (*clone)(sqInt oop);
+#     sqInt (*instantiateClassindexableSize)(sqInt classPointer, sqInt size);
+#     sqInt (*makePointwithxValueyValue)(sqInt xValue, sqInt yValue);
+#     sqInt (*popRemappableOop)(void);
+#     sqInt (*pushRemappableOop)(sqInt oop);
+
+#     /* InterpreterProxy methodsFor: 'other' */
+
+#     sqInt (*becomewith)(sqInt array1, sqInt array2);
+#     sqInt (*byteSwapped)(sqInt w);
+#     sqInt (*failed)(void);
+#     sqInt (*fullDisplayUpdate)(void);
+#     sqInt (*fullGC)(void);
+#     sqInt (*incrementalGC)(void);
+#     sqInt (*primitiveFail)(void);
+#     sqInt (*showDisplayBitsLeftTopRightBottom)(sqInt aForm, sqInt l, sqInt t, sqInt r, sqInt b);
+#     sqInt (*signalSemaphoreWithIndex)(sqInt semaIndex);
+#     sqInt (*success)(sqInt aBoolean);
+#     sqInt (*superclassOf)(sqInt classPointer);
+
+#     /* InterpreterProxy methodsFor: 'compiler' */
+
+#     CompilerHook *(*compilerHookVector)(void);
+#     sqInt          (*setCompilerInitialized)(sqInt initFlag);
+
+# #if VM_PROXY_MINOR > 1
+
+#     /* InterpreterProxy methodsFor: 'BitBlt support' */
+
+#     sqInt (*loadBitBltFrom)(sqInt bbOop);
+#     sqInt (*copyBits)(void);
+#     sqInt (*copyBitsFromtoat)(sqInt leftX, sqInt rightX, sqInt yValue);
+
+# #endif
+
+# #if VM_PROXY_MINOR > 2
+
+#     sqInt (*classLargeNegativeInteger)(void);
+#     sqInt (*signed32BitIntegerFor)(sqInt integerValue);
+#     sqInt (*signed32BitValueOf)(sqInt oop);
+#     sqInt (*includesBehaviorThatOf)(sqInt aClass, sqInt aSuperClass);
+#     sqInt (*primitiveMethod)(void);
+
+#     /* InterpreterProxy methodsFor: 'FFI support' */
+
+#     sqInt (*classExternalAddress)(void);
+#     sqInt (*classExternalData)(void);
+#     sqInt (*classExternalFunction)(void);
+#     sqInt (*classExternalLibrary)(void);
+#     sqInt (*classExternalStructure)(void);
+#     sqInt (*ioLoadModuleOfLength)(sqInt modIndex, sqInt modLength);
+#     sqInt (*ioLoadSymbolOfLengthFromModule)(sqInt fnIndex, sqInt fnLength, sqInt handle);
+#     sqInt (*isInMemory)(sqInt address);
+
+# #endif
+
+# #if VM_PROXY_MINOR > 3
+
+#     void *(*ioLoadFunctionFrom)(char *fnName, char *modName);
+#     sqInt (*ioMicroMSecs)(void);
+
+# #endif
+
+# #if VM_PROXY_MINOR > 4
+
+# #  if !defined(sqLong)
+# #   if _MSC_VER
+# #     define sqLong __int64
+# #     define usqLong unsigned __int64
+# #   else
+# #     define sqLong long long
+# #     define usqLong unsigned long long
+# #   endif
+# #  endif
+
+#     sqInt  (*positive64BitIntegerFor)(sqLong integerValue);
+#     sqLong (*positive64BitValueOf)(sqInt oop);
+#     sqInt  (*signed64BitIntegerFor)(sqLong integerValue);
+#     sqLong (*signed64BitValueOf)(sqInt oop);
+
+# #endif
+
+# #if VM_PROXY_MINOR > 5
+#     sqInt (*isArray)(sqInt oop);
+#     sqInt (*forceInterruptCheck)(void);
+# #endif
+
+# #if VM_PROXY_MINOR > 6
+#     sqInt  (*fetchLong32ofObject)(sqInt fieldFieldIndex, sqInt oop);
+#     sqInt  (*getThisSessionID)(void);
+#     sqInt     (*ioFilenamefromStringofLengthresolveAliases)(char* aCharBuffer, char* filenameIndex, sqInt filenameLength, sqInt resolveFlag);
+#     sqInt  (*vmEndianness)(void);
+# #endif
+
+# #if VM_PROXY_MINOR > 7
+#   /* New methods for proxy version 1.8 */
+
+#   /* callbackEnter: Re-enter the interpreter loop for a callback.
+#      Arguments:
+#        callbackID: Pointer to a location receiving the callback ID
+#                    used in callbackLeave
+#      Returns: True if successful, false otherwise */
+#   sqInt (*callbackEnter)(sqInt *callbackID);
+
+#   /* callbackLeave: Leave the interpreter from a previous callback
+#      Arguments:
+#        callbackID: The ID of the callback received from callbackEnter()
+#      Returns: True if succcessful, false otherwise. */
+#   sqInt (*callbackLeave)(sqInt  callbackID);
+
+#   /* addGCRoot: Add a variable location to the garbage collector.
+#      The contents of the variable location will be updated accordingly.
+#      Arguments:
+#        varLoc: Pointer to the variable location
+#      Returns: True if successful, false otherwise. */
+#   sqInt (*addGCRoot)(sqInt *varLoc);
+
+#   /* removeGCRoot: Remove a variable location from the garbage collector.
+#      Arguments:
+#        varLoc: Pointer to the variable location
+#      Returns: True if successful, false otherwise.
+#   */
+#   sqInt (*removeGCRoot)(sqInt *varLoc);
+# #endif
+
+# #if VM_PROXY_MINOR > 8
+#     /* See interp.h and above for standard error codes. */
+#     sqInt  (*primitiveFailFor)(sqInt code);
+#     void (*(*setInterruptCheckChain)(void (*aFunction)(void)))();
+#     sqInt  (*classAlien)(void);
+#     sqInt  (*classUnsafeAlien)(void);
+#     sqInt  (*sendInvokeCallbackStackRegistersJmpbuf)(sqInt thunkPtrAsInt, sqInt stackPtrAsInt, sqInt regsPtrAsInt, sqInt jmpBufPtrAsInt);
+#     sqInt  (*reestablishContextPriorToCallback)(sqInt callbackContext);
+#     sqInt *(*getStackPointer)(void);
+#     sqInt  (*isOopImmutable)(sqInt oop);
+#     sqInt  (*isOopMutable)(sqInt oop);
+# #endif
+
+# #if VM_PROXY_MINOR > 9
+#   sqInt  (*methodArg)  (sqInt index);
+#   sqInt  (*objectArg)  (sqInt index);
+#   sqInt  (*integerArg) (sqInt index);
+#   double (*floatArg)   (sqInt index);
+#   sqInt  (*methodReturnValue) (sqInt oop);
+#   sqInt  (*topRemappableOop)  (void);
+# #endif
+
+# #if VM_PROXY_MINOR > 10
+# # define DisownVMLockOutFullGC 1
+#   sqInt (*disownVM)(sqInt flags);
+#   sqInt (*ownVM)   (sqInt threadIdAndFlags);
+#   void  (*addHighPriorityTickee)(void (*ticker)(void), unsigned periodms);
+#   void  (*addSynchronousTickee)(void (*ticker)(void), unsigned periodms, unsigned roundms);
+#   usqLong (*utcMicroseconds)(void);
+#   sqInt (*tenuringIncrementalGC)(void);
+#   sqInt (*isYoung) (sqInt anOop);
+#   sqInt (*isKindOfClass)(sqInt oop, sqInt aClass);
+#   sqInt (*primitiveErrorTable)(void);
+#   sqInt (*primitiveFailureCode)(void);
+#   sqInt (*instanceSizeOf)(sqInt aClass);
+# #endif
+
+# #if VM_PROXY_MINOR > 11
+# /* VMCallbackContext opaque type avoids all including setjmp.h & vmCallback.h */
+#   sqInt (*sendInvokeCallbackContext)(vmccp);
+#   sqInt (*returnAsThroughCallbackContext)(int, vmccp, sqInt);
+#   long  (*signedMachineIntegerValueOf)(sqInt);
+#   long  (*stackSignedMachineIntegerValue)(sqInt);
+#   unsigned long  (*positiveMachineIntegerValueOf)(sqInt);
+#   unsigned long  (*stackPositiveMachineIntegerValue)(sqInt);
+#   sqInt  (*getInterruptPending)(void);
+#   char  *(*cStringOrNullFor)(sqInt);
+#   void  *(*startOfAlienData)(sqInt);
+#   usqInt (*sizeOfAlienData)(sqInt);
+#   sqInt  (*signalNoResume)(sqInt);
+# #endif
 
 # ##############################################################################
 
@@ -179,7 +470,7 @@
         self.s_frame = None
         self.argcount = 0
         self.s_method = None
-        self.successFlag = True
+        self.success_flag = True
 
     def call(self, signature, interp, s_frame, argcount, s_method):
         self.interp = interp
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -139,6 +139,9 @@
         from spyvm.fieldtypes import obj
         return obj
 
+    def is_array_object(self):
+        return False
+
 class W_SmallInteger(W_Object):
     """Boxed integer value"""
     # TODO can we tell pypy that its never larger then 31-bit?
@@ -319,6 +322,9 @@
         from spyvm.fieldtypes import LPI
         return LPI
 
+    def is_array_object(self):
+        return True
+
 class W_Float(W_AbstractObjectWithIdentityHash):
     """Boxed float value."""
     _attrs_ = ['value']
@@ -789,6 +795,9 @@
             word += r_uint(ord(self.getchar(i))) << 8*i
         return word
 
+    def is_array_object(self):
+        return True
+
 class W_WordsObject(W_AbstractObjectWithClassReference):
     _attrs_ = ['words']
 
@@ -856,6 +865,9 @@
         return W_AbstractObjectWithClassReference.as_embellished_string(self,
             className='W_WordsObject', additionalInformation=('len=%d' % self.size()))
 
+    def is_array_object(self):
+        return True
+
 NATIVE_DEPTH = 32
 
 class W_DisplayBitmap(W_AbstractObjectWithClassReference):
@@ -911,6 +923,8 @@
     def setword(self, n, word):
         raise NotImplementedError("subclass responsibility")
 
+    def is_array_object(self):
+        return True
 
 class W_DisplayBitmap1Bit(W_DisplayBitmap):
     def getword(self, n):
@@ -1162,6 +1176,9 @@
     def has_shadow(self):
         return self._shadow is not None
 
+    def is_array_object(self):
+        return True
+
 class DetachingShadowError(Exception):
     def __init__(self, old_shadow, new_shadow_class):
         self.old_shadow = old_shadow

From noreply at buildbot.pypy.org  Mon Jun  3 15:57:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  3 Jun 2013 15:57:37 +0200 (CEST)
Subject: [pypy-commit] stmgc default: In-progress
Message-ID: <20130603135737.750031C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r61:82b9992a758c
Date: 2013-06-03 15:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/82b9992a758c/

Log:	In-progress

diff --git a/c3/doc-objects.txt b/c3/doc-objects.txt
new file mode 100644
--- /dev/null
+++ b/c3/doc-objects.txt
@@ -0,0 +1,76 @@
+
+
+
+Object copies state transitions (changes of state of the *same* copy)
+---------------------------------------------------------------------
+
+
+
+  Private freshly created
+             \                Private, with backup
+              \                  ^       \    ^
+               \                /         \   |
+         commit \       modify /    commit |  |
+                 \            /            |  | modify
+                  V          /             |  |
+           Protected, no backup            V  |
+                ^    ^              Protected, with backup
+               /     |       gc       |
+       commit /      `----------------'
+             /
+            /
+    Private copy of
+    a public obj
+
+
+
+     Protected backup copy
+                  \
+                   \
+           stealing \         commit of newer version
+                     \         ,-----------------.
+                      V        |                 V
+             Up-to-date public copy       Outdated public copy
+
+
+
+Kind of object copy                                    h_revision
+-------------------------------------------------------------------
+
+Private objects:
+- freshly created                                         PRN
+- converted from a protected obj                          PRN
+- private copy of a public obj                            PRN
+
+Protected objects:
+- converted from fresh private obj                     (old PRN)
+- converted from a private obj with backup           ptr to backup
+- backup copy of a private obj                    original h_revision
+- backup copy still attached to a protected               GT
+- original obj after GC killed the backup                 GT
+
+Public objects:
+- prebuilt object, never modified                         -1
+- other public object, never modified                     GT
+- outdated, has a protected copy               ptr to prot/priv | 2
+- outdated, target stolen                   ptr to next public copy
+
+Public stubs:
+- from stealing: like outdated public objects
+- from major GC: like outdated public objects with target stolen
+
+
+PRN = Private revision number (negative odd number)
+GT = Global time (positive odd number)
+
+
+
+Off-line data stored in the thread-local structure
+--------------------------------------------------
+
+- the PRN (private revision number): odd, negative, changes for every
+  transaction that commits
+
+- list of pairs (private converted from protected, backup copy)
+
+- dict {public obj: private copy}
diff --git a/c3/doc-stmgc.txt b/c3/doc-stmgc.txt
--- a/c3/doc-stmgc.txt
+++ b/c3/doc-stmgc.txt
@@ -12,24 +12,26 @@
 from multiple threads is possible, and handled correctly (that's the
 whole point), but a relatively rare case.
 
-So each object is classified as "public" or "protected".  New objects
-are protected until they are read by a different thread.  The point is
-to use very different mechanisms for public and for protected objects.
-Public objects are visible by all threads, but read-only in memory; to
-change them, a copy must be made, and the changes written to the copy
-(the "redolog" approach to STM).  Protected objects, on the other hand,
-are modified in-place, with (if necessary) a copy of them being made
-only for the purpose of a possible abort of the transaction (the
-"undolog" approach).
+So each object is classified as "public", "protected", or "private".
+Objects are created private, and later become protected, and stay so as
+long as they are not read by a different thread.  The point is to use
+very different mechanisms for public and for non-public objects.  Public
+objects are visible by all threads, but read-only in memory; to change
+them, a copy must be made, and the changes written to the copy (the
+"redolog" approach to STM).  Non-public objects, on the other hand, are
+modified in-place, with (if necessary) a copy of them being made only
+for the purpose of a possible abort of the transaction (the "undolog"
+approach).
 
 This is combined with a generational GC similar to PyPy's --- but here,
 each thread gets its own nursery and does its own "minor collections",
 independently of the others.
 
-Objects start as protected, and when another thread tries to follow a
-pointer to them, then it is that other thread's job to carefully "steal"
-the object and turn it public (possibly making a copy of it if needed,
-e.g. if it was still a young object living in the original nursery).
+The idea of protected objects is that when another thread tries to
+follow a pointer to them, then it is that other thread's job to
+carefully "steal" the object and turn it public (possibly making a copy
+of it if needed, e.g. if it was still a young object living in the
+original nursery).
 
 The same object can exist temporarily in multiple versions: any number
 of public copies; at most one active protected copy; and optionally one
@@ -62,14 +64,14 @@
 committed revisions are globally ordered.  This is the order that the
 multithreaded program appears to have followed serially.
 
-The object copies exist in one of two main states: they can be
-"protected" or "public".  A copy is also called "private" when it was
-modified by the transaction in progress; this copy is always protected
-and invisible to other threads.  When that transaction commits, all
-private copies become protected, and remain so as long as it is accessed
-only by the same thread.  A copy becomes public only when another thread
-requests access to it (or, more precisely, "steals" access to it).  Once
-public, a copy is immutable in memory.
+The object copies exist in one of three main states: they can be
+"public", "protected" or "private".  A private copy is a copy that was
+created or modified by the transaction in progress; this copy is always
+invisible to other threads.  When that transaction commits, all private
+copies become protected.  They remain protected as long as they are only
+accessed by the same thread.  A copy becomes public only when another
+thread requests access to it (or, more precisely, "steals" access to
+it).  Once public, a copy is immutable in memory.
 
 From the point of view of the generational GC, each copy is either young
 or old.  All new copies are allocated young.  They become old at the
@@ -83,17 +85,15 @@
 memory and trigger a lot of major collections.)  For the rest of this
 document we'll ignore young copies outside the nursery.
 
-An object that was never seen by a different thread has got either one
-of two copies, both protected: the "main" one, used by the thread, which
-may be private or not depending on whether the object was modified in
-the current transaction; and, if the object is private but older than
-the current transaction, then it has got a backup copy whose purpose is
-to record the state that the object had at the start of the current
-transaction.
+An object that was never seen by a different thread has got at most one
+private copy (if it was created or modified by the current transaction)
+and one protected copy (if it is older than the current transaction).
+If it has two copies, then the private one is the regular copy, and the
+other copy works as a backup copy that remembers the state that the
+object had at the start of the current transaction.
 
-If an object is committed and then no longer modified for long enough,
-the next (minor or major) GC will free the space that was used by the
-backup copy.
+If an object has got a backup copy but isn't modified any more, the next
+(minor or major) GC collection will free the backup copy.
 
 The way to share data between threads goes via prebuilt objects, which
 are always public: it is their existence that gives the starting point
@@ -103,7 +103,7 @@
 1. A thread tries to write to a public object.  This is done by
 allocating a fresh private copy of the public object.  Then writes go to
 the private copy.  If the transaction commits, the private copy becomes
-simply protected, and the public object is made to point to it (with
+protected, and the public object is made to point to it (with
 multithread care).  From now on, any access to the public object from
 the same thread will work on the protected copy.  Any access from a
 different thread will trigger "stealing", as explained next.
@@ -141,9 +141,8 @@
 -------------------
 
 This design is made to optimize the hopefully common case: objects we
-handle are mostly private or protected, or if they are public, they are
-mostly read-only.  We can design in consequence the following three
-points:
+handle are mostly protected, or if they are public, they are mostly
+read-only.  We can design in consequence the following three points:
 
 1. the extra data stored in the objects (GC flags, and one extra word
 called `h_revision`).
@@ -162,12 +161,12 @@
 the header of the copy.  But the recording in the read set is a bit more
 annoying.  We need to maintain a thread-local *set* of all accessed
 objects, but we don't care about the order or recording the occasional
-duplicate.  Moreover we don't need to record the private objects; but we
-do need all other protected objects, as well as public objects.  The
-best approach is probably to have a quick check "is it definitely
-recorded already?" inline, and do the call if the check fails.  It needs
-careful design to be done in only a few CPU instructions, but it should
-be possible.
+duplicate.  Moreover we don't need to record the private copies of
+objects; but we do need all the protected and public objects.  The best
+approach is probably to have a quick check "is it definitely recorded
+already?" inline, and do the call if the check fails.  It needs careful
+design to be done in only a few CPU instructions, but it should be
+possible.
 
 (Code like this compiles to 4 instructions in the fast path:
 
@@ -179,16 +178,16 @@
 )
 
 The case of the write barrier is similar to the first half of the read
-barrier, but differs in the check we need to do.  We need to do a call
-if the object is not already private.  For performance reasons, "being
-private" is not directly a flag in the object, because when a
-transaction commits, we don't want to have to walk all private objects
-to change this flag.  Instead, private objects have a precise negative
-odd number in their `h_revision` field, called the "local revision
-identifier".  When a transaction commits, we change the value of the
-local revision identifier, and all previously-private objects become
-automatically protected.  So the write barrier fast-path checks if the
-`h_revision` is equal from the local revision identifier.
+barrier.  We need to do a call to the slow path if the object is not
+already private.  For performance reasons, "being private" is not
+directly a flag in the object, because when a transaction commits, we
+don't want to have to walk all private copies to change this flag.
+Instead, private copies have a precise negative odd number in their
+`h_revision` field, called the "private revision identifier".  When a
+transaction commits, we change the value of the private revision
+identifier, and all previously-private objects become automatically
+protected.  So the write barrier fast-path checks if the `h_revision` is
+equal to the private revision identifier.
 
 
 The extendable timestamp model
@@ -268,29 +267,10 @@
 transactions efficient.)
 
 
-Details of protected objects
-----------------------------
+Object copies in detail
+-----------------------
 
-As described above, each thread has a "local revision identifier".  It
-is a negative odd number that changes whenever it commits a transaction.
-The important point for the write barrier is that on any object copy,
-`h_revision` must be equal to the local revision identifier if and only
-if the copy is private.  A newly allocated object is always private.
-Once the transaction commits it becomes merely protected.  Its
-`h_revision` field doesn't change (but the thread's local revision
-identifier does).  If later the write barrier triggers on it, we make a
-backup copy of the object and copy the content of the primary copy to
-it.  We also set `h_revision` in the primary copy to point to the
-backup copy: as long as `h_revision` is different from the local
-revision identifier, its exact value is otherwise not used.  In this way
-we can keep using the same backup copy in each future transaction
-that needs to write to the object.
-
-The backup copy is used in two cases.  One is if the transaction aborts;
-then we copy the content back over the regular protected copy.  The
-other case is if the object is stolen.  In that case, if the object has
-an active backup copy, we must steal this one, because the regular
-protected copy is actually private at that point in time.
+See doc-objects.txt.
 
 
 Minor and major collections
@@ -322,5 +302,5 @@
 references in existing objects to point to either the real copy or the
 stub.  This is probably a bit involved: we might have to get the current
 revision numbers of all threads, and theoretically compact each interval
-of number down to only one number, but still keep one active revision
+of numbers down to only one number, but still keep one active revision
 number per thread.

From noreply at buildbot.pypy.org  Mon Jun  3 17:26:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  3 Jun 2013 17:26:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: typo
Message-ID: <20130603152610.4464D1C010B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64737:699be6433207
Date: 2013-06-03 17:25 +0200
http://bitbucket.org/pypy/pypy/changeset/699be6433207/

Log:	typo

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -201,7 +201,7 @@
 
 hacking on PyPy with the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since hacking on Pypy means running tests, you will need a way to specify
+Since hacking on PyPy means running tests, you will need a way to specify
 the mingw compiler when hacking (as opposed to translating). As of
 March 2012, --cc is not a valid option for pytest.py. However if you set an
 environment variable CC to the compliter exe, testing will use it.

From noreply at buildbot.pypy.org  Mon Jun  3 18:38:33 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Mon,  3 Jun 2013 18:38:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge heads
Message-ID: <20130603163833.46C741C145C@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r64739:f556942951f9
Date: 2013-06-03 16:40 +0000
http://bitbucket.org/pypy/pypy/changeset/f556942951f9/

Log:	Merge heads

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -199,9 +199,9 @@
 
 or such, depending on your mingw64 download.
 
-hacking on Pypy with the mingw compiler
+hacking on PyPy with the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since hacking on Pypy means running tests, you will need a way to specify
+Since hacking on PyPy means running tests, you will need a way to specify
 the mingw compiler when hacking (as opposed to translating). As of
 March 2012, --cc is not a valid option for pytest.py. However if you set an
 environment variable CC to the compliter exe, testing will use it.

From noreply at buildbot.pypy.org  Mon Jun  3 18:38:31 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Mon,  3 Jun 2013 18:38:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Change how boxes do cache lookup
Message-ID: <20130603163831.A477B1C1401@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r64738:e69d28ee34e0
Date: 2013-06-03 16:40 +0000
http://bitbucket.org/pypy/pypy/changeset/e69d28ee34e0/

Log:	Change how boxes do cache lookup

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -30,7 +30,7 @@
 def new_dtype_getter(name):
     def _get_dtype(space):
         from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-        return getattr(get_dtype_cache(space), "w_%sdtype" % name)
+        return get_dtype_cache(space).dtypes_by_name[name]
 
     def new(space, w_subtype, w_value):
         dtype = _get_dtype(space)
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -273,7 +273,6 @@
 
         fields = space.getitem(w_data, space.wrap(4))
         self.set_fields(space, fields)
-        print self.itemtype
 
 class W_ComplexDtype(W_Dtype):
     def __init__(self, itemtype, num, kind, name, char, w_box_type,
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -21,7 +21,3 @@
 
         a = zeros(3)
         assert loads(dumps(sum(a))) == sum(a)
-
-    def setup_class(cls):
-        import py
-        py.test.xfail("FIXME: dtype('int32') == dtype('int32') fails (but only on 32-bit?)")

From noreply at buildbot.pypy.org  Mon Jun  3 19:24:46 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  3 Jun 2013 19:24:46 +0200 (CEST)
Subject: [pypy-commit] stmgc default: More preparation.
Message-ID: <20130603172446.CA8CD1C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r62:4456f79692e9
Date: 2013-06-03 19:24 +0200
http://bitbucket.org/pypy/stmgc/changeset/4456f79692e9/

Log:	More preparation.

diff --git a/c3/doc-objects.txt b/c3/doc-objects.txt
--- a/c3/doc-objects.txt
+++ b/c3/doc-objects.txt
@@ -8,19 +8,19 @@
 
   Private freshly created
              \                Private, with backup
-              \                  ^       \    ^
-               \                /         \   |
-         commit \       modify /    commit |  |
-                 \            /            |  | modify
-                  V          /             |  |
-           Protected, no backup            V  |
+              \                ^  .          |  ^
+               \              /  .    commit |  |
+         commit \     modify /  .            |  |
+                 \          /  . commit      |  | modify
+                  V        /  V              |  |
+           Protected, no backup              V  |
                 ^    ^              Protected, with backup
                /     |       gc       |
        commit /      `----------------'
              /
             /
-    Private copy of
-    a public obj
+    Private copy of               (the dotted arrow is followed if the
+    a public obj                   protected backup copy was stolen)
 
 
 
@@ -52,8 +52,8 @@
 Public objects:
 - prebuilt object, never modified                         -1
 - other public object, never modified                     GT
-- outdated, has a protected copy               ptr to prot/priv | 2
-- outdated, target stolen                   ptr to next public copy
+- outdated, has a protected copy              HANDLE to prot/priv copy
+- outdated, target stolen              ptr to a more recent public copy
 
 Public stubs:
 - from stealing: like outdated public objects
@@ -62,6 +62,8 @@
 
 PRN = Private revision number (negative odd number)
 GT = Global time (positive odd number)
+HANDLE = Reference to a prot/priv copy and its thread
+         (positive even number, such that: handle % 4 == 2)
 
 
 
@@ -71,6 +73,120 @@
 - the PRN (private revision number): odd, negative, changes for every
   transaction that commits
 
-- list of pairs (private converted from protected, backup copy)
+- dict active_backup_copies = {private converted from protected: backup copy}
 
-- dict {public obj: private copy}
+- dict public_to_private = {public obj: private copy}
+
+- list read_set containing the objects in the read set, with possibly
+  some duplicates (but hopefully not too many)
+
+
+
+Kind of object copy            distinguishing feature
+-------------------------------------------------------------------
+
+Any private object             h_revision == PRN
+Private with a backup          in active_backup_copies
+Backup copy                    GCFLAG_BACKUP_COPY
+Any public object              GCFLAG_PUBLIC
+Any protected object           h_revision != PRN && !GCFLAG_PUBLIC
+Stubs                          GCFLAG_STUB
+
+A public object that might \
+be key in public_to_private    has additionally GCFLAG_PUBLIC_TO_PRIVATE
+
+
+
+Read barrier
+-----------------------------------------
+
+Inline check: if P in read_barrier_cache, we don't call the slow path.
+Slow path:
+
+    if h_revision == PRN, just add P to read_barrier_cache and return
+
+    if GCFLAG_PUBLIC:
+
+        follow the chained list of h_revision's as long as they are
+        regular pointers
+
+        if it ends with an odd revision number, check that it's older
+        than start_time; extend the start timestamp if not
+
+        if it ends with a handle (L, Thread):
+
+            if Thread is the current thread: set P = L
+
+            else: do stealing and restart the read barrier
+
+        if we land on a P in read_barrier_cache: return
+
+    add P to 'read_set'
+
+    add P to 'read_barrier_cache' and return
+
+
+Handles are stored for example in a global list, and the actual handle
+encodes an index in the list.  Every entry in the list is a pointer to a
+prot/priv object --- excepted once every N positions, where it is a
+thread descriptor giving the thread to which all N-1 following pointers
+belong.  The pair (L, Thread) is thus `(list[H], list[H rounded down to
+a multiple of N])`.
+
+Stealing of an object copy L is done with the "collection lock" of
+the target Thread.  The target would also acquire its own lock in
+when doing some operations, like a minor collection, which can't
+occur in parallel with stealing.
+
+Once we have the lock, stealing is:
+
+    if the situation changed while we were waiting for the lock, return
+
+    if L has got a backup copy, turn it public;
+    else L must be protected, and we make a public copy of it
+
+    update the original P->h_revision to point directly to the new
+    public copy
+
+
+
+Write barrier
+-----------------------------------------
+
+The write barrier works for both STM purposes and for GC purposes.
+
+Inline check: if h_revision == PRN && !GCFLAG_WRITE_BARRIER, we're done.
+Slow path:
+
+    R = read_barrier(P)     # always do a full read_barrier first
+
+    if h_revision == PRN:
+
+        GC only: remove GCFLAG_WRITE_BARRIER, add R to the GC list of
+        modified old objects to trace at the next minor collection,
+        and return R
+
+    elif GCFLAG_PUBLIC:
+
+        add the flag GCFLAG_PUBLIC_TO_PRIVATE to R, if needed
+
+        make a fresh private copy L of R, with h_revision == PRN
+
+        add {R: L} in 'public_to_private'
+
+        return L
+
+    else:    # protected object
+
+        if h_revision is not a pointer:
+
+            allocate a backup copy, and attach it to h_revision
+
+        copy the object into the backup copy
+
+        change h_revision to be PRN (i.e. turn private)
+
+        if GCFLAG_WRITE_BARRIER: remove it, add R to the GC list of
+        modified old objects to trace at the next minor collection
+
+        return R

From noreply at buildbot.pypy.org  Tue Jun  4 01:35:24 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue,  4 Jun 2013 01:35:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: apply 722471a15693 from default
Message-ID: <20130603233524.0DF731C1007@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64741:43b233e93eed
Date: 2013-06-03 16:34 -0700
http://bitbucket.org/pypy/pypy/changeset/43b233e93eed/

Log:	apply 722471a15693 from default

diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -4,7 +4,7 @@
 modules (currently limited to C extensions, should accommodate C++
 extensions ASAP)."""
 
-import sys, os, re
+import sys, os, re, imp
 from distutils.core import Command
 from distutils.errors import *
 from distutils.sysconfig import customize_compiler, get_python_version
@@ -35,6 +35,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext(Command):
 
@@ -671,10 +676,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols(self, ext):
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -1,9 +1,17 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -49,18 +57,14 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
+    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
     global _config_vars
     _config_vars = g
@@ -70,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars

From noreply at buildbot.pypy.org  Tue Jun  4 01:35:22 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue,  4 Jun 2013 01:35:22 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130603233522.C28211C0619@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64740:f088ba1adc09
Date: 2013-06-03 16:30 -0700
http://bitbucket.org/pypy/pypy/changeset/f088ba1adc09/

Log:	merge default

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -58,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -80,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -151,6 +151,8 @@
     'DEBUG': DEBUG,
     'NOTSET': NOTSET,
 }
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python3')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:PyInit__ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_pypy_testcapi.py
copy from lib_pypy/_testcapi.py
copy to lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,14 +1,20 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
     """
     thisdir = os.path.dirname(__file__)
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -19,13 +25,13 @@
         ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
+    res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
                            extra_preargs=ccflags)
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
+    output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
@@ -37,7 +43,7 @@
             library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:PyInit__testcapi']
+                        '/EXPORT:PyInit_' + modulename]
     else:
         libraries = []
         extra_ldargs = []
@@ -49,14 +55,7 @@
         libraries=libraries,
         extra_preargs=extra_ldargs)
 
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-try:
-    import cpyext
-except ImportError:
-    raise ImportError("No module named '_testcapi'")
-else:
-    compile_shared()
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,62 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python3')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:PyInit__testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
 try:
     import cpyext
 except ImportError:
     raise ImportError("No module named '_testcapi'")
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,7 @@
+
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -46,3 +46,7 @@
 
 .. branch: operrfmt-NT
 Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -2,7 +2,7 @@
 PyPy on Windows
 ===============
 
-Pypy is supported on Windows platforms, starting with Windows 2000.
+PyPy is supported on Windows platforms, starting with Windows 2000.
 The following text gives some hints about how to translate the PyPy
 interpreter.
 
@@ -199,9 +199,9 @@
 
 or such, depending on your mingw64 download.
 
-hacking on Pypy with the mingw compiler
+hacking on PyPy with the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since hacking on Pypy means running tests, you will need a way to specify
+Since hacking on PyPy means running tests, you will need a way to specify
 the mingw compiler when hacking (as opposed to translating). As of
 March 2012, --cc is not a valid option for pytest.py. However if you set an
 environment variable CC to the compliter exe, testing will use it.
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -607,6 +607,11 @@
 class TestNonInteractive:
     def run_with_status_code(self, cmdline, senddata='', expect_prompt=False,
             expect_banner=False, python_flags='', env=None):
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         cmdline = '%s %s "%s" %s' % (python3, python_flags,
                                      app_main, cmdline)
         print 'POPEN:', cmdline
@@ -768,6 +773,11 @@
         assert 'copyright' not in data
 
     def test_non_interactive_stdout_fully_buffered(self):
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         path = getscript(r"""
             import sys, time
             sys.stdout.write('\x00(STDOUT)\n\x00')   # stays in buffers
@@ -789,6 +799,11 @@
 
     def test_non_interactive_stdout_unbuffered(self, monkeypatch):
         monkeypatch.setenv('PYTHONUNBUFFERED', '1')
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         path = getscript(r"""
             import sys, time
             sys.stdout.write('\x00(STDOUT)\n\x00')
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -1,3 +1,4 @@
+import sys
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 
@@ -113,6 +114,14 @@
 
 # ____________________________________________________________
 
+SF_MSVC_BITFIELDS = 1
+
+if sys.platform == 'win32':
+    DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+else:
+    DEFAULT_SFLAGS = 0
+
+
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -121,9 +130,11 @@
 def new_union_type(space, name):
     return ctypestruct.W_CTypeUnion(space, name)
 
- at unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int)
+ at unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
+             sflags=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1):
+                             totalsize=-1, totalalignment=-1,
+                             sflags=DEFAULT_SFLAGS):
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise OperationError(space.w_TypeError,
@@ -134,6 +145,8 @@
     alignment = 1
     boffset = 0         # this number is in *bits*, not bytes!
     boffsetmax = 0      # the maximum value of boffset, in bits too
+    prev_bitfield_size = 0
+    prev_bitfield_free = 0
     fields_w = space.listview(w_fields)
     fields_list = []
     fields_dict = {}
@@ -166,7 +179,15 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield
         falign = ftype.alignof()
-        if alignment < falign and (fbitsize < 0 or fname != ''):
+        do_align = True
+        if fbitsize >= 0:
+            if (sflags & SF_MSVC_BITFIELDS) == 0:
+                # GCC: anonymous bitfields (of any size) don't cause alignment
+                do_align = (fname != '')
+            else:
+                # MSVC: zero-sized bitfields don't cause alignment
+                do_align = (fbitsize > 0)
+        if alignment < falign and do_align:
             alignment = falign
         #
         if fbitsize < 0:
@@ -208,6 +229,7 @@
                 fields_dict[fname] = fld
 
             boffset += ftype.size * 8
+            prev_bitfield_size = 0
 
         else:
             # this is the case of a bitfield
@@ -243,31 +265,67 @@
                     raise operationerrfmt(space.w_TypeError,
                                           "field '%s.%s' is declared with :0",
                                           w_ctype.name, fname)
-                if boffset > field_offset_bytes * 8:
-                    field_offset_bytes += falign
-                    assert boffset < field_offset_bytes * 8
-                boffset = field_offset_bytes * 8
+                if (sflags & SF_MSVC_BITFIELDS) == 0:
+                    # GCC's notion of "ftype :0;"
+                    # pad boffset to a value aligned for "ftype"
+                    if boffset > field_offset_bytes * 8:
+                        field_offset_bytes += falign
+                        assert boffset < field_offset_bytes * 8
+                    boffset = field_offset_bytes * 8
+                else:
+                    # MSVC's notion of "ftype :0;
+                    # Mostly ignored.  It seems they only serve as
+                    # separator between other bitfields, to force them
+                    # into separate words.
+                    pass
+                prev_bitfield_size = 0
+
             else:
-                # Can the field start at the offset given by 'boffset'?  It
-                # can if it would entirely fit into an aligned ftype field.
-                bits_already_occupied = boffset - (field_offset_bytes * 8)
+                if (sflags & SF_MSVC_BITFIELDS) == 0:
+                    # GCC's algorithm
 
-                if bits_already_occupied + fbitsize > 8 * ftype.size:
-                    # it would not fit, we need to start at the next
-                    # allowed position
-                    field_offset_bytes += falign
-                    assert boffset < field_offset_bytes * 8
-                    boffset = field_offset_bytes * 8
-                    bitshift = 0
+                    # Can the field start at the offset given by 'boffset'?  It
+                    # can if it would entirely fit into an aligned ftype field.
+                    bits_already_occupied = boffset - (field_offset_bytes * 8)
+
+                    if bits_already_occupied + fbitsize > 8 * ftype.size:
+                        # it would not fit, we need to start at the next
+                        # allowed position
+                        field_offset_bytes += falign
+                        assert boffset < field_offset_bytes * 8
+                        boffset = field_offset_bytes * 8
+                        bitshift = 0
+                    else:
+                        bitshift = bits_already_occupied
+                        assert bitshift >= 0
+                    boffset += fbitsize
+
                 else:
-                    bitshift = bits_already_occupied
+                    # MSVC's algorithm
+
+                    # A bitfield is considered as taking the full width
+                    # of their declared type.  It can share some bits
+                    # with the previous field only if it was also a
+                    # bitfield and used a type of the same size.
+                    if (prev_bitfield_size == ftype.size and
+                        prev_bitfield_free >= fbitsize):
+                        # yes: reuse
+                        bitshift = 8 * prev_bitfield_size - prev_bitfield_free
+                    else:
+                        # no: start a new full field
+                        boffset = (boffset + falign*8-1) & ~(falign*8-1)
+                        boffset += ftype.size * 8
+                        bitshift = 0
+                        prev_bitfield_size = ftype.size
+                        prev_bitfield_free = 8 * prev_bitfield_size
+                    #
+                    prev_bitfield_free -= fbitsize
+                    field_offset_bytes = boffset / 8 - ftype.size
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
-            
-                boffset += fbitsize
 
         if boffset > boffsetmax:
             boffsetmax = boffset
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2757,36 +2757,57 @@
     assert wr() is None
     py.test.raises(RuntimeError, from_handle, cast(BCharP, 0))
 
-def test_bitfield_as_gcc():
+def _test_bitfield_details(flag):
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
+    BUInt = new_primitive_type("unsigned int")
     BStruct = new_struct_type("foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
-                                       ('b', BInt, 9),
-                                       ('c', BChar, -1)])
-    assert typeoffsetof(BStruct, 'c') == (BChar, 3)
-    assert sizeof(BStruct) == 4
+                                       ('b1', BInt, 9),
+                                       ('b2', BUInt, 7),
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 3)
+        assert sizeof(BStruct) == 4
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 8)
+        assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
-                                       ('c', BChar, -1)])
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
     assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
-    assert alignof(BStruct) == 1
+    if flag == 0:   # gcc
+        assert sizeof(BStruct) == 5
+        assert alignof(BStruct) == 1
+    else:           # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
-                                       ('c', BChar, -1)])
-    assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 5
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 1)
+        assert sizeof(BStruct) == 2
     assert alignof(BStruct) == 1
 
 
+def test_bitfield_as_gcc():
+    _test_bitfield_details(flag=0)
+
+def test_bitfield_as_msvc():
+    _test_bitfield_details(flag=1)
+
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "0.7"
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -280,7 +280,7 @@
         return _absolute_import(space, modulename, baselevel,
                                 fromlist_w, tentative)
     finally:
-        lock.release_lock()
+        lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
 def absolute_import_try(space, modulename, baselevel, fromlist_w):
@@ -775,9 +775,13 @@
             self.lockowner = me
         self.lockcounter += 1
 
-    def release_lock(self):
+    def release_lock(self, silent_after_fork):
         me = self.space.getexecutioncontext()   # used as thread ident
         if self.lockowner is not me:
+            if self.lockowner is None and silent_after_fork:
+                # Too bad.  This situation can occur if a fork() occurred
+                # with the import lock held, and we're the child.
+                return
             if not self._can_have_lock():
                 return
             space = self.space
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -209,7 +209,7 @@
 
 def release_lock(space):
     if space.config.objspace.usemodules.thread:
-        importing.getimportlock(space).release_lock()
+        importing.getimportlock(space).release_lock(silent_after_fork=False)
 
 def reinit_lock(space):
     if space.config.objspace.usemodules.thread:
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -30,7 +30,7 @@
 def new_dtype_getter(name):
     def _get_dtype(space):
         from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-        return getattr(get_dtype_cache(space), "w_%sdtype" % name)
+        return get_dtype_cache(space).dtypes_by_name[name]
 
     def new(space, w_subtype, w_value):
         dtype = _get_dtype(space)
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -273,7 +273,6 @@
 
         fields = space.getitem(w_data, space.wrap(4))
         self.set_fields(space, fields)
-        print self.itemtype
 
 class W_ComplexDtype(W_Dtype):
     def __init__(self, itemtype, num, kind, name, char, w_box_type,
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -21,7 +21,3 @@
 
         a = zeros(3)
         assert loads(dumps(sum(a))) == sum(a)
-
-    def setup_class(cls):
-        import py
-        py.test.xfail("FIXME: dtype('int32') == dtype('int32') fails")
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -109,7 +109,7 @@
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
                        'mmap', 'marshal', '_codecs', 'rctime', 'cppyy',
-                       '_cffi_backend', 'pyexpat', '_continuation']:
+                       '_cffi_backend', 'pyexpat', '_continuation', '_io']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -36,6 +36,10 @@
     from pypy.module.rctime.interp_time import time
     assert pypypolicy.look_inside_function(time)
 
+def test_io():
+    from pypy.module._io.interp_bytesio import W_BytesIO
+    assert pypypolicy.look_inside_function(W_BytesIO.seek_w.im_func)
+
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
     from pypy.module._random.interp_random import W_Random
diff --git a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
--- a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
+++ b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
@@ -41,6 +41,10 @@
     assert 'LOG_NOTICE' in d
 
 def test_resource():
+    try:
+        import lib_pypy.resource
+    except ImportError:
+        py.test.skip('no syslog on this platform')
     d = run('resource.ctc.py', '_resource_cache.py')
     assert 'RLIM_NLIMITS' in d
 
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -347,9 +347,8 @@
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):
-                # convert string to unicode explicitely here
-                from pypy.objspace.std.unicodetype import plain_str2unicode
-                r = plain_str2unicode(self.space, r)
+                # convert string to unicode using the default encoding
+                r = self.space.unicode_w(self.space.wrap(r))
             prec = self.prec
             if prec == -1 and self.width == 0:
                 # fast path
@@ -508,12 +507,10 @@
             result = formatter.format()
         except NeedUnicodeFormattingError:
             # fall through to the unicode case
-            from pypy.objspace.std.unicodetype import plain_str2unicode
-            fmt = plain_str2unicode(space, fmt)
+            pass
         else:
             return space.wrap(result)
-    else:
-        fmt = space.unicode_w(w_fmt)
+    fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
     return space.wrap(result)
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -12,22 +12,6 @@
     from pypy.objspace.std.unicodeobject import W_UnicodeObject
     return W_UnicodeObject(uni)
 
-def plain_str2unicode(space, s):
-    try:
-        return unicode(s)
-    except UnicodeDecodeError:
-        for i in range(len(s)):
-            if ord(s[i]) > 127:
-                raise OperationError(
-                    space.w_UnicodeDecodeError,
-                    space.newtuple([
-                    space.wrap('ascii'),
-                    space.wrapbytes(s),
-                    space.wrap(i),
-                    space.wrap(i+1),
-                    space.wrap("ordinal not in range(128)")]))
-        assert False, "unreachable"
-
 
 unicode_capitalize = SMM('capitalize', 1,
                          doc='S.capitalize() -> unicode\n\nReturn a'
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -8,6 +8,7 @@
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.opassembler import ResOpAssembler
 from rpython.jit.backend.arm.regalloc import (Regalloc,
     CoreRegisterManager, check_imm_arg, VFPRegisterManager,
@@ -961,7 +962,7 @@
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VLDR(target.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
@@ -982,7 +983,7 @@
             return self._store_core_reg(mc, source, base, ofs, cond, helper)
 
     def _store_vfp_reg(self, mc, source, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VSTR(source.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -52,6 +52,8 @@
 
     def _push_stack_args(self, stack_args, on_stack):
         assert on_stack % 8 == 0
+        if on_stack == 0:
+            return
         self._adjust_sp(-on_stack)
         self.current_sp = on_stack
         ofs = 0
@@ -71,7 +73,7 @@
             else:
                 self.mc.gen_load_int(r.ip.value, n)
                 self.mc.ADD_rr(r.sp.value, r.sp.value, r.ip.value)
-        else:
+        elif n < 0:
             n = abs(n)
             if check_imm_arg(n):
                 self.mc.SUB_ri(r.sp.value, r.sp.value, n)
diff --git a/rpython/jit/backend/arm/helper/regalloc.py b/rpython/jit/backend/arm/helper/regalloc.py
--- a/rpython/jit/backend/arm/helper/regalloc.py
+++ b/rpython/jit/backend/arm/helper/regalloc.py
@@ -4,7 +4,10 @@
 from rpython.jit.metainterp.history import ConstInt
 from rpython.rlib.objectmodel import we_are_translated
 
-def check_imm_arg(arg, size=0xFF, allow_zero=True):
+VMEM_imm_size=0x3FC
+default_imm_size=0xFF
+
+def check_imm_arg(arg, size=default_imm_size, allow_zero=True):
     assert not isinstance(arg, ConstInt)
     if not we_are_translated():
         if not isinstance(arg, int):
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -15,6 +15,7 @@
                                                 gen_emit_unary_float_op,
                                                 saved_registers)
 from rpython.jit.backend.arm.helper.regalloc import check_imm_arg
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.jump import remap_frame_layout
 from rpython.jit.backend.arm.regalloc import TempBox
@@ -23,6 +24,7 @@
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import InteriorFieldDescr
 from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler
+from rpython.jit.backend.llsupport.regalloc import get_scale
 from rpython.jit.metainterp.history import (Box, AbstractFailDescr,
                                             INT, FLOAT, REF)
 from rpython.jit.metainterp.history import TargetToken
@@ -523,35 +525,9 @@
 
     def emit_op_setfield_gc(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs, size = arglocs
-        if size.value == 8:
-            assert value_loc.is_vfp_reg()
-            # vstr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            if ofs.is_reg():
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
-                base_loc = r.ip
-                ofs = imm(0)
-            else:
-                assert ofs.value % 4 == 0
-            self.mc.VSTR(value_loc.value, base_loc.value, ofs.value)
-        elif size.value == 4:
-            if ofs.is_imm():
-                self.mc.STR_ri(value_loc.value, base_loc.value, ofs.value)
-            else:
-                self.mc.STR_rr(value_loc.value, base_loc.value, ofs.value)
-        elif size.value == 2:
-            if ofs.is_imm():
-                self.mc.STRH_ri(value_loc.value, base_loc.value, ofs.value)
-            else:
-                self.mc.STRH_rr(value_loc.value, base_loc.value, ofs.value)
-        elif size.value == 1:
-            if ofs.is_imm():
-                self.mc.STRB_ri(value_loc.value, base_loc.value, ofs.value)
-            else:
-                self.mc.STRB_rr(value_loc.value, base_loc.value, ofs.value)
-        else:
-            assert 0
+        scale = get_scale(size.value)
+        self._write_to_mem(value_loc, base_loc,
+                                ofs, imm(scale), fcond)
         return fcond
 
     emit_op_setfield_raw = emit_op_setfield_gc
@@ -559,47 +535,8 @@
     def emit_op_getfield_gc(self, op, arglocs, regalloc, fcond):
         base_loc, ofs, res, size = arglocs
         signed = op.getdescr().is_field_signed()
-        if size.value == 8:
-            assert res.is_vfp_reg()
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            if ofs.is_reg():
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
-                base_loc = r.ip
-                ofs = imm(0)
-            else:
-                assert ofs.value % 4 == 0
-            self.mc.VLDR(res.value, base_loc.value, ofs.value)
-        elif size.value == 4:
-            if ofs.is_imm():
-                self.mc.LDR_ri(res.value, base_loc.value, ofs.value)
-            else:
-                self.mc.LDR_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 2:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSH_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSH_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 1:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSB_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSB_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_rr(res.value, base_loc.value, ofs.value)
-        else:
-            assert 0
+        scale = get_scale(size.value)
+        self._load_from_mem(res, base_loc, ofs, imm(scale), signed, fcond)
         return fcond
 
     emit_op_getfield_raw = emit_op_getfield_gc
@@ -609,72 +546,44 @@
     def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond):
         (base_loc, index_loc, res_loc,
             ofs_loc, ofs, itemsize, fieldsize) = arglocs
-        self.mc.gen_load_int(r.ip.value, itemsize.value)
-        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        scale = get_scale(fieldsize.value)
+        tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+        assert not save
+        self.mc.gen_load_int(tmploc.value, itemsize.value)
+        self.mc.MUL(tmploc.value, index_loc.value, tmploc.value)
         descr = op.getdescr()
         assert isinstance(descr, InteriorFieldDescr)
         signed = descr.fielddescr.is_field_signed()
         if ofs.value > 0:
             if ofs_loc.is_imm():
-                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+                self.mc.ADD_ri(tmploc.value, tmploc.value, ofs_loc.value)
             else:
-                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
-
-        if fieldsize.value == 8:
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            assert res_loc.is_vfp_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, 0)
-        elif fieldsize.value == 4:
-            self.mc.LDR_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 2:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 1:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value, r.ip.value)
-        else:
-            assert 0
-
+                self.mc.ADD_rr(tmploc.value, tmploc.value, ofs_loc.value)
+        ofs_loc = tmploc
+        self._load_from_mem(res_loc, base_loc, ofs_loc,
+                                imm(scale), signed, fcond)
         return fcond
 
     def emit_op_setinteriorfield_gc(self, op, arglocs, regalloc, fcond):
         (base_loc, index_loc, value_loc,
             ofs_loc, ofs, itemsize, fieldsize) = arglocs
-        self.mc.gen_load_int(r.ip.value, itemsize.value)
-        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        scale = get_scale(fieldsize.value)
+        tmploc, save = self.get_tmp_reg([base_loc, index_loc, value_loc, ofs_loc])
+        assert not save
+        self.mc.gen_load_int(tmploc.value, itemsize.value)
+        self.mc.MUL(tmploc.value, index_loc.value, tmploc.value)
         if ofs.value > 0:
             if ofs_loc.is_imm():
-                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+                self.mc.ADD_ri(tmploc.value, tmploc.value, ofs_loc.value)
             else:
-                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
-        if fieldsize.value == 8:
-            # vstr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            assert value_loc.is_vfp_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
-            self.mc.VSTR(value_loc.value, r.ip.value, 0)
-        elif fieldsize.value == 4:
-            self.mc.STR_rr(value_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 2:
-            self.mc.STRH_rr(value_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 1:
-            self.mc.STRB_rr(value_loc.value, base_loc.value, r.ip.value)
-        else:
-            assert 0
+                self.mc.ADD_rr(tmploc.value, tmploc.value, ofs_loc.value)
+        self._write_to_mem(value_loc, base_loc, tmploc, imm(scale), fcond)
         return fcond
     emit_op_setinteriorfield_raw = emit_op_setinteriorfield_gc
 
     def emit_op_arraylen_gc(self, op, arglocs, regalloc, fcond):
         res, base_loc, ofs = arglocs
-        self.mc.LDR_ri(res.value, base_loc.value, ofs.value)
+        self.load_reg(self.mc, res, base_loc, ofs.value)
         return fcond
 
     def emit_op_setarrayitem_gc(self, op, arglocs, regalloc, fcond):
@@ -694,18 +603,40 @@
     def _write_to_mem(self, value_loc, base_loc, ofs_loc, scale, fcond=c.AL):
         if scale.value == 3:
             assert value_loc.is_vfp_reg()
-            assert ofs_loc.is_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value)
-            self.mc.VSTR(value_loc.value, r.ip.value, cond=fcond)
+            # vstr only supports imm offsets
+            # so if the ofset is too large we add it to the base and use an
+            # offset of 0
+            if ofs_loc.is_reg():
+                tmploc, save = self.get_tmp_reg([value_loc, base_loc, ofs_loc])
+                assert not save
+                self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
+                base_loc = tmploc
+                ofs_loc = imm(0)
+            else:
+                assert ofs_loc.is_imm()
+                assert ofs_loc.value % 4 == 0
+            self.mc.VSTR(value_loc.value, base_loc.value, ofs_loc.value)
         elif scale.value == 2:
-            self.mc.STR_rr(value_loc.value, base_loc.value, ofs_loc.value,
-                                                                    cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.STR_ri(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.STR_rr(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 1:
-            self.mc.STRH_rr(value_loc.value, base_loc.value, ofs_loc.value,
-                                                                    cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.STRH_ri(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.STRH_rr(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 0:
-            self.mc.STRB_rr(value_loc.value, base_loc.value, ofs_loc.value,
-                                                                    cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.STRB_ri(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.STRB_rr(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         else:
             assert 0
 
@@ -731,33 +662,63 @@
             self.mc.ADD_ri(r.ip.value, ofs_loc.value, imm=ofs.value)
             ofs_loc = r.ip
         #
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def _load_from_mem(self, res_loc, base_loc, ofs_loc, scale,
                                             signed=False, fcond=c.AL):
         if scale.value == 3:
             assert res_loc.is_vfp_reg()
-            assert ofs_loc.is_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, cond=fcond)
+            # vldr only supports imm offsets
+            # if the offset is in a register we add it to the base and use a
+            # tmp reg
+            if ofs_loc.is_reg():
+                tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+                assert not save
+                self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
+                base_loc = tmploc
+                ofs_loc = imm(0)
+            else:
+                assert ofs_loc.is_imm()
+                assert ofs_loc.value % 4 == 0
+            self.mc.VLDR(res_loc.value, base_loc.value, ofs_loc.value, cond=fcond)
         elif scale.value == 2:
-            self.mc.LDR_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.LDR_ri(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.LDR_rr(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 1:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         elif scale.value == 0:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         else:
             assert 0
 
@@ -770,7 +731,7 @@
         # no base offset
         assert ofs.value == 0
         signed = op.getdescr().is_item_signed()
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def emit_op_strlen(self, op, arglocs, regalloc, fcond):
@@ -993,7 +954,7 @@
                 assert result_loc.is_vfp_reg()
                 # we always have a register here, since we have to sync them
                 # before call_assembler
-                self.mc.VLDR(result_loc.value, r.r0.value, imm=ofs)
+                self.load_reg(self.mc, result_loc, r.r0, ofs=ofs)
             else:
                 assert result_loc is r.r0
                 ofs = self.cpu.unpack_arraydescr(descr)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -13,7 +13,9 @@
                                                     prepare_cmp_op,
                                                     prepare_float_op,
                                                     check_imm_arg,
-                                                    check_imm_box
+                                                    check_imm_box,
+                                                    VMEM_imm_size,
+                                                    default_imm_size,
                                                     )
 from rpython.jit.backend.arm.jump import remap_frame_layout_mixed
 from rpython.jit.backend.arm.arch import WORD, JITFRAME_FIXED_SIZE
@@ -811,7 +813,8 @@
         ofs, size, sign = unpack_fielddescr(op.getdescr())
         base_loc = self.make_sure_var_in_reg(a0, boxes)
         value_loc = self.make_sure_var_in_reg(a1, boxes)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if size < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = imm(ofs)
         else:
             ofs_loc = self.get_scratch_reg(INT, boxes)
@@ -825,7 +828,8 @@
         ofs, size, sign = unpack_fielddescr(op.getdescr())
         base_loc = self.make_sure_var_in_reg(a0)
         immofs = imm(ofs)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if size < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = immofs
         else:
             ofs_loc = self.get_scratch_reg(INT, [a0])
@@ -846,7 +850,8 @@
         base_loc = self.make_sure_var_in_reg(op.getarg(0), args)
         index_loc = self.make_sure_var_in_reg(op.getarg(1), args)
         immofs = imm(ofs)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if fieldsize < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = immofs
         else:
             ofs_loc = self.get_scratch_reg(INT, args)
@@ -865,7 +870,8 @@
         index_loc = self.make_sure_var_in_reg(op.getarg(1), args)
         value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
         immofs = imm(ofs)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if fieldsize < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = immofs
         else:
             ofs_loc = self.get_scratch_reg(INT, args)
@@ -890,8 +896,8 @@
         scale = get_scale(size)
         args = op.getarglist()
         base_loc = self.make_sure_var_in_reg(args[0], args)
+        value_loc = self.make_sure_var_in_reg(args[2], args)
         ofs_loc = self.make_sure_var_in_reg(args[1], args)
-        value_loc = self.make_sure_var_in_reg(args[2], args)
         assert check_imm_arg(ofs)
         return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)]
     prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc
diff --git a/rpython/jit/backend/arm/test/test_regalloc_mov.py b/rpython/jit/backend/arm/test/test_regalloc_mov.py
--- a/rpython/jit/backend/arm/test/test_regalloc_mov.py
+++ b/rpython/jit/backend/arm/test/test_regalloc_mov.py
@@ -436,7 +436,7 @@
         self.push(sf, e)
 
     def test_push_large_stackfloat(self):
-        sf = stack_float(100)
+        sf = stack_float(1000)
         e = [
             mi('gen_load_int', ip.value, sf.value, cond=AL),
             mi('ADD_rr', ip.value, fp.value, ip.value, cond=AL),
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -17,7 +17,7 @@
 def get_description(atypes, rtype):
     p = lltype.malloc(CIF_DESCRIPTION, len(atypes),
                       flavor='raw', immortal=True)
-    p.abi = 42
+    p.abi = 1     # default
     p.nargs = len(atypes)
     p.rtype = rtype
     p.atypes = lltype.malloc(FFI_TYPE_PP.TO, len(atypes),
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -9,7 +9,7 @@
 link_files = []
 include_dirs = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
-    libraries = ['libeay32', 'ssleay32', 'zlib',
+    libraries = ['libeay32', 'ssleay32', 'zlib1',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
     includes = [
         # ssl.h includes winsock.h, which will conflict with our own
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -107,7 +107,6 @@
         "_PyVerify_fd", [rffi.INT], rffi.INT,
         compilation_info=errno_eci,
         ))
-    @jit.dont_look_inside
     def validate_fd(fd):
         if not is_valid_fd(fd):
             raise OSError(get_errno(), 'Bad file descriptor')
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -7,7 +7,6 @@
 from struct import unpack
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 from rpython.rlib.rstruct.error import StructError
-from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
 
 class MasterReader(object):
     def __init__(self, s):
@@ -30,9 +29,9 @@
 
 def reader_for_pos(pos):
     class ReaderForPos(AbstractReader):
-        def __init__(self, mr):
+        def __init__(self, mr, bigendian):
             self.mr = mr
-            self.bigendian = native_is_bigendian
+            self.bigendian = bigendian
 
         def read(self, count):
             return self.mr.read(count)
@@ -64,6 +63,7 @@
         perform_lst = []
         miniglobals = {}
         miniglobals.update(globals())
+        miniglobals['bigendian'] = self.bigendian
         for i in rg:
             fmtdesc, rep, mask = self.formats[i]
             miniglobals['unpacker%d' % i] = fmtdesc.unpack
@@ -74,8 +74,8 @@
             else:
                 perform_lst.append('unpacker%d(reader%d, %d)' % (i, i, rep))
             miniglobals['reader_cls%d' % i] = reader_for_pos(i)
-        readers = ";".join(["reader%d = reader_cls%d(master_reader)" % (i, i)
-                             for i in rg])
+        readers = ";".join(["reader%d = reader_cls%d(master_reader, bigendian)"
+                            % (i, i) for i in rg])
         perform = ";".join(perform_lst)
         unpackers = ','.join(['reader%d.value' % i for i in rg])
         source = py.code.Source("""
diff --git a/rpython/rlib/rstruct/test/test_runpack.py b/rpython/rlib/rstruct/test/test_runpack.py
--- a/rpython/rlib/rstruct/test/test_runpack.py
+++ b/rpython/rlib/rstruct/test/test_runpack.py
@@ -26,6 +26,18 @@
         assert fn() == 123
         assert self.interpret(fn, []) == 123
 
+    def test_unpack_big_endian(self):
+        def fn():
+            return runpack(">i", "\x01\x02\x03\x04")
+        assert fn() == 0x01020304
+        assert self.interpret(fn, []) == 0x01020304
+
+    def test_unpack_double_big_endian(self):
+        def fn():
+            return runpack(">d", "testtest")
+        assert fn() == struct.unpack(">d", "testtest")[0]
+        assert self.interpret(fn, []) == struct.unpack(">d", "testtest")[0]
+
 class TestLLType(BaseTestRStruct, LLRtypeMixin):
     pass
 
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -10,7 +10,7 @@
 
 
 if compiler.name == "msvc":
-    libname = 'zlib'
+    libname = 'zlib1' # since version 1.1.4 and later, see http://www.zlib.net/DLL_FAQ.txt
 else:
     libname = 'z'
 eci = ExternalCompilationInfo(
diff --git a/rpython/rtyper/tool/test/test_rffi_platform.py b/rpython/rtyper/tool/test/test_rffi_platform.py
--- a/rpython/rtyper/tool/test/test_rffi_platform.py
+++ b/rpython/rtyper/tool/test/test_rffi_platform.py
@@ -288,9 +288,6 @@
     assert a % struct.calcsize("P") == 0
 
 def test_external_lib():
-    # XXX this one seems to be a bit too platform-specific. Check
-    #     how to test it on windows correctly (using so_prefix?)
-    #     and what are alternatives to LD_LIBRARY_PATH
     eci = ExternalCompilationInfo()
     c_source = """
     int f(int a, int b)
@@ -298,12 +295,17 @@
         return (a + b);
     }
     """
+    if platform.name == 'mscv':
+        c_source = '__declspec(dllexport) ' + c_source
+        libname = 'libc_lib'
+    else:
+        libname = 'c_lib'
     tmpdir = udir.join('external_lib').ensure(dir=1)
     c_file = tmpdir.join('libc_lib.c')
     c_file.write(c_source)
     l = platform.compile([c_file], eci, standalone=False)
     eci = ExternalCompilationInfo(
-        libraries = ['c_lib'],
+        libraries = [libname],
         library_dirs = [str(tmpdir)]
         )
     rffi_platform.verify_eci(eci)
diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py
--- a/rpython/translator/c/gcc/test/test_asmgcroot.py
+++ b/rpython/translator/c/gcc/test/test_asmgcroot.py
@@ -25,8 +25,8 @@
 
     @classmethod
     def make_config(cls):
-        if _MSVC and _WIN64:
-            py.test.skip("all asmgcroot tests disabled for MSVC X64")
+        if _MSVC:
+            py.test.skip("all asmgcroot tests disabled for MSVC")
         from rpython.config.translationoption import get_combined_translation_config
         config = get_combined_translation_config(translating=True)
         config.translation.gc = cls.gcpolicy
diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py
--- a/rpython/translator/c/test/test_genc.py
+++ b/rpython/translator/c/test/test_genc.py
@@ -37,7 +37,7 @@
     if isinstance(v, float):
         from rpython.rlib.rfloat import formatd, DTSF_ADD_DOT_0
         return formatd(v, 'r', 0, DTSF_ADD_DOT_0)
-    return v
+    return str(v)   # always return a string, to get consistent types
 
 def parse_longlong(a):
     p0, p1 = a.split(":")
@@ -205,6 +205,28 @@
     py.test.raises(Exception, f1, "world")  # check that it's really typed
 
 
+def test_int_becomes_float():
+    # used to crash "very often": the long chain of mangle() calls end
+    # up converting the return value of f() from an int to a float, but
+    # if blocks are followed in random order by the annotator, it will
+    # very likely first follow the call to llrepr_out() done after the
+    # call to f(), getting an int first (and a float only later).
+    @specialize.arg(1)
+    def mangle(x, chain):
+        if chain:
+            return mangle(x, chain[1:])
+        return x - 0.5
+    def f(x):
+        if x > 10:
+            x = mangle(x, (1,1,1,1,1,1,1,1,1,1))
+        return x + 1
+
+    f1 = compile(f, [int])
+
+    assert f1(5) == 6
+    assert f1(12) == 12.5
+
+
 def test_string_arg():
     def f(s):
         total = 0
diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -8,8 +8,9 @@
     DEFAULT_CC = "cc"
     name = "openbsd"
 
-    link_flags = os.environ.get("LDFLAGS", '-pthread').split()
-    cflags = os.environ.get("CFLAGS", "-O3 -pthread -fomit-frame-pointer -D_BSD_SOURCE").split()
+    link_flags = os.environ.get("LDFLAGS", "").split() + ['-pthread']
+    cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-D_BSD_SOURCE'
+             ] + os.environ.get("CFLAGS", "").split()
 
     def _libs(self, libraries):
         libraries=set(libraries + ("intl", "iconv", "compat"))
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -28,6 +28,8 @@
     return _get_compiler_type(cc, False)
 
 def Windows_x64(cc=None):
+    raise Exception("Win64 is not supported.  You must either build for Win32"
+                    " or contribute the missing support in PyPy.")
     return _get_compiler_type(cc, True)
     
 def _get_msvc_env(vsver, x64flag):

From noreply at buildbot.pypy.org  Tue Jun  4 02:32:15 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue,  4 Jun 2013 02:32:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: o fix identifier handling around
 space.getname (use unicode throughout)
Message-ID: <20130604003215.54E021C0F88@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64742:4042cddab652
Date: 2013-06-03 17:29 -0700
http://bitbucket.org/pypy/pypy/changeset/4042cddab652/

Log:	o fix identifier handling around space.getname (use unicode
	throughout) o space.getname\W_TypeObject.get_module_type_name now
	return unicode o space.getrepr now accepts unicode

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -68,7 +68,7 @@
             ##    con.interact()
             except OperationError, e:
                 debug("OperationError:")
-                debug(" operror-type: " + e.w_type.getname(space))
+                debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
@@ -76,7 +76,7 @@
                 space.call_function(w_run_toplevel, w_call_finish_gateway)
             except OperationError, e:
                 debug("OperationError:")
-                debug(" operror-type: " + e.w_type.getname(space))
+                debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
@@ -117,7 +117,7 @@
         except OperationError, e:
             if verbose:
                 debug("OperationError:")
-                debug(" operror-type: " + e.w_type.getname(space))
+                debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return 1
 
@@ -155,7 +155,7 @@
             stmt.exec_code(space, w_globals, w_globals)
         except OperationError, e:
             debug("OperationError:")
-            debug(" operror-type: " + e.w_type.getname(space))
+            debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
             debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return 1
         return 0
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -77,11 +77,12 @@
         raise NotImplementedError("only for interp-level user subclasses "
                                   "from typedef.py")
 
-    def getname(self, space, default='?'):
+    def getname(self, space, default=u'?'):
         try:
-            return space.str_w(space.getattr(self, space.wrap('__name__')))
+            return space.unicode_w(space.getattr(self, space.wrap('__name__')))
         except OperationError, e:
-            if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
+            if (e.match(space, space.w_TypeError) or
+                e.match(space, space.w_AttributeError)):
                 return default
             raise
 
@@ -104,10 +105,9 @@
             w_id = space.rshift(w_id, w_4)
         return ''.join(addrstring)
 
-    def getrepr(self, space, info, moreinfo=''):
-        addrstring = self.getaddrstring(space)
-        return space.wrap("<%s at 0x%s%s>" % (info, addrstring,
-                                              moreinfo))
+    def getrepr(self, space, info, moreinfo=u''):
+        addrstring = unicode(self.getaddrstring(space))
+        return space.wrap(u"<%s at 0x%s%s>" % (info, addrstring, moreinfo))
 
     def getslotvalue(self, index):
         raise NotImplementedError
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -414,15 +414,7 @@
                     elif fmt in 'NT':
                         if fmt == 'T':
                             value = space.type(value)
-                        try:
-                            w_name = space.getattr(value, space.wrap('__name__'))
-                        except OperationError as e:
-                            if not (e.match(space, space.w_TypeError) or
-                                    e.match(space, space.w_AttributeError)):
-                                raise
-                            result = u'?'
-                        else:
-                            result = space.unicode_w(w_name)
+                        result = value.getname(space)
                     else:
                         result = unicode(value)
                     lst[i + i + 1] = result
@@ -450,7 +442,7 @@
 
     %8 - The result of arg.decode('utf-8', 'strict')
     %N - The result of w_arg.getname(space)
-    %R - The result of space.str_w(space.repr(w_arg))
+    %R - The result of space.unicode_w(space.repr(w_arg))
     %T - The result of space.type(w_arg).getname(space)
 
     """
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -245,7 +245,8 @@
         return self.call_args(__args__)
 
     def descr_function_repr(self):
-        return self.getrepr(self.space, 'function %s' % (self.name,))
+        return self.getrepr(self.space, u'function %s' %
+                            (self.name.decode('utf-8'),))
 
     # delicate
     _all = {'': None}
@@ -390,7 +391,7 @@
         self.w_doc = w_doc
 
     def fget_func_name(self, space):
-        return space.wrap(self.name)
+        return space.wrap(self.name.decode('utf-8'))
 
     def fset_func_name(self, space, w_name):
         try:
@@ -492,7 +493,7 @@
         return space.wrap(method)
 
     def __repr__(self):
-        return "bound method %s" % (self.w_function.getname(self.space),)
+        return u"bound method %s" % (self.w_function.getname(self.space),)
 
     def call_args(self, args):
         space = self.space
@@ -509,8 +510,8 @@
         name = self.w_function.getname(self.space)
         w_class = space.type(self.w_instance)
         typename = w_class.getname(self.space)
-        objrepr = space.str_w(space.repr(self.w_instance))
-        s = '<bound method %s.%s of %s>' % (typename, name, objrepr)
+        objrepr = space.unicode_w(space.repr(self.w_instance))
+        s = u'<bound method %s.%s of %s>' % (typename, name, objrepr)
         return space.wrap(s)
 
     def descr_method_getattribute(self, w_attr):
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -135,8 +135,8 @@
 
     def type(self, obj):
         class Type:
-            def getname(self, space, default='?'):
-                return type(obj).__name__
+            def getname(self, space, default=u'?'):
+                return unicode(type(obj).__name__)
         return Type()
 
 
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -844,6 +844,14 @@
         c = compile('from os import &#26085;&#26412;', '', 'exec')
         assert ('&#26085;&#26412;',) in c.co_consts
 
+    def test_class_nonascii(self):
+        """
+        class &#26085;&#26412;:
+            pass
+        assert &#26085;&#26412;.__name__ == '&#26085;&#26412;'
+        assert '&#26085;&#26412;' in repr(&#26085;&#26412;)
+        """
+
     def test_cpython_issue2301(self):
         skip('XXX')
         try:
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -126,7 +126,7 @@
 
 def test_new_exception(space):
     w_error = new_exception_class(space, '_socket.error')
-    assert w_error.getname(space) == 'error'
+    assert w_error.getname(space) == u'error'
     assert space.str_w(space.repr(w_error)) == "<class '_socket.error'>"
     operr = OperationError(w_error, space.wrap("message"))
     assert operr.match(space, w_error)
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -105,7 +105,7 @@
                 w_class = space.type(seen[0].w_instance)
                 found = 'method %s of %s' % (
                     seen[0].w_function.name,
-                    w_class.getname(space))
+                    w_class.getname(space).encode('utf-8'))
             else:
                 assert isinstance(seen[0], Function)
                 found = 'builtin %s' % seen[0].name
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -138,6 +138,14 @@
             __name__ = "bar"
             assert f.__module__ == "foo"''')
 
+    def test_func_nonascii(self):
+        """
+        def &#26085;&#26412;():
+            pass
+        assert repr(&#26085;&#26412;).startswith('<function &#26085;&#26412; at ')
+        assert &#26085;&#26412;.__name__ == '&#26085;&#26412;'
+        """
+
 
 class AppTestFunction:
     def test_simple_call(self):
diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py
--- a/pypy/module/__builtin__/interp_memoryview.py
+++ b/pypy/module/__builtin__/interp_memoryview.py
@@ -140,9 +140,9 @@
 
     def descr_repr(self, space):
         if self.buf is None:
-            return self.getrepr(space, 'released memory')
+            return self.getrepr(space, u'released memory')
         else:
-            return self.getrepr(space, 'memory')
+            return self.getrepr(space, u'memory')
 
     def descr_release(self, space):
         self.buf = None
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -227,16 +227,16 @@
 
     def repr_w(self, space):
         typename = space.type(self).getname(space)
-        module = space.str_w(space.type(self).get_module())
+        module = space.unicode_w(space.type(self).get_module())
         try:
             w_name = space.getattr(self, space.wrap("name"))
         except OperationError, e:
             if not e.match(space, space.w_AttributeError):
                 raise
-            return space.wrap("<%s.%s>" % (module, typename,))
+            return space.wrap(u"<%s.%s>" % (module, typename,))
         else:
-            name_repr = space.str_w(space.repr(w_name))
-            return space.wrap("<%s.%s name=%s>" % (module, typename, name_repr))
+            name_repr = space.unicode_w(space.repr(w_name))
+            return space.wrap(u"<%s.%s name=%s>" % (module, typename, name_repr))
 
     # ______________________________________________
 
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -201,27 +201,28 @@
         w_realclass, _ = space.lookup_in_type_where(w_type, name)
         if isinstance(w_realclass, W_TypeObject):
             class_name = w_realclass.get_module_type_name()
-    return "{method '%s' of '%s' objects}" % (name, class_name)
+    return u"{method '%s' of '%s' objects}" % (name.decode('utf-8'),
+                                               class_name)
 
 
 @jit.elidable_promote()
 def create_spec_for_function(space, w_func):
     if w_func.w_module is None:
-        module = ''
+        module = u''
     else:
-        module = space.str_w(w_func.w_module)
-        if module == 'builtins':
-            module = ''
+        module = space.unicode_w(w_func.w_module)
+        if module == u'builtins':
+            module = u''
         else:
-            module += '.'
-    pre = 'built-in function ' if isinstance(w_func, BuiltinFunction) else ''
-    return '{%s%s%s}' % (pre, module, w_func.name)
+            module += u'.'
+    pre = u'built-in function ' if isinstance(w_func, BuiltinFunction) else u''
+    return u'{%s%s%s}' % (pre, module, w_func.getname(space))
 
 
 @jit.elidable_promote()
 def create_spec_for_object(space, w_obj):
     class_name = space.type(w_obj).getname(space)
-    return "{'%s' object}" % (class_name,)
+    return u"{'%s' object}" % (class_name,)
 
 
 def create_spec(space, w_arg):
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -64,8 +64,9 @@
         return space.newbool(bool(self.flags & WRITABLE))
 
     def _repr(self, space, handle):
-        conn_type = ["read-only", "write-only", "read-write"][self.flags - 1]
-        return space.wrap("<%s %s, handle %d>" % (
+        index = self.flags - 1
+        conn_type = [u"read-only", u"write-only", u"read-write"][index]
+        return space.wrap(u"<%s %s, handle %d>" % (
                 conn_type, space.type(self).getname(space), handle))
 
     def descr_repr(self, space):
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -172,15 +172,15 @@
     def descr__repr__(self, space):
         w_obj = self.dereference()
         if w_obj is None:
-            state = '; dead'
+            state = u'; dead'
         else:
             typename = space.type(w_obj).getname(space)
-            objname = w_obj.getname(space, '')
+            objname = w_obj.getname(space, u'')
             if objname:
-                state = "; to '%s' (%s)" % (typename, objname)
+                state = u"; to '%s' (%s)" % (typename, objname)
             else:
-                state = "; to '%s'" % (typename,)
-        return self.getrepr(space, self.typedef.name, state)
+                state = u"; to '%s'" % (typename,)
+        return self.getrepr(space, unicode(self.typedef.name), state)
 
 
 class W_Weakref(W_WeakrefBase):
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -61,7 +61,7 @@
         self.name = name
         self.__name__ = name
     def getname(self, space, name):
-        return self.name
+        return unicode(self.name)
 class FakeBuffer(FakeBase):
     typedname = "buffer"
     def __init__(self, val):
diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py
--- a/pypy/module/cpyext/classobject.py
+++ b/pypy/module/cpyext/classobject.py
@@ -28,9 +28,8 @@
         return space.call_args(self.w_function, __args__)
 
     def descr_repr(self, space):
-        name = space.str_w(
-            space.getattr(self.w_function, space.wrap('__name__')))
-        return self.getrepr(space, '<instancemethod %s>' % (name,))
+        return self.getrepr(space, u'<instancemethod %s>' %
+                            (space.getname(self.w_function),))
 
 InstanceMethod.typedef = TypeDef("instancemethod",
     __new__ = interp2app(InstanceMethod.descr_new),
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -124,9 +124,9 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
-        return self.getrepr(self.space,
-                            "built-in method '%s' of '%s' object" %
-                            (self.name, self.w_objclass.getname(self.space)))
+        return self.getrepr(
+            self.space, u"built-in method '%s' of '%s' object" %
+            (self.name.decode('utf-8'), self.w_objclass.getname(self.space)))
 
 PyCFunction_Check, PyCFunction_CheckExact = build_type_checkers(
     "CFunction", W_PyCFunctionObject)
@@ -143,9 +143,9 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
-        return self.getrepr(self.space,
-                            "built-in method '%s' of '%s' object" %
-                            (self.name, self.w_objclass.getname(self.space)))
+        return self.getrepr(
+            self.space, u"built-in method '%s' of '%s' object" %
+            (self.name.decode('utf-8'), self.w_objclass.getname(self.space)))
 
 
 class W_PyCWrapperObject(W_Root):
@@ -173,8 +173,8 @@
         return self.wrapper_func(space, w_self, w_args, self.func)
 
     def descr_method_repr(self):
-        return self.space.wrap("<slot wrapper '%s' of '%s' objects>" %
-                               (self.method_name,
+        return self.space.wrap(u"<slot wrapper '%s' of '%s' objects>" %
+                               (self.method_name.decode('utf-8'),
                                 self.w_objclass.getname(self.space)))
 
 def cwrapper_descr_call(space, w_self, __args__):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -471,7 +471,7 @@
         from pypy.module.cpyext.unicodeobject import _PyUnicode_AsString
         pto.c_tp_name = _PyUnicode_AsString(space, heaptype.c_ht_name)
     else:
-        pto.c_tp_name = rffi.str2charp(w_type.getname(space))
+        pto.c_tp_name = rffi.str2charp(w_type.getname(space).encode('utf-8'))
     pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out
     pto.c_tp_itemsize = 0
     # uninitialized fields:
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -131,9 +131,10 @@
 
     def descr_repr(self, space):
         if self.args_w:
-            args_repr = space.str_w(space.repr(space.newtuple(self.args_w)))
+            args_repr = space.unicode_w(
+                space.repr(space.newtuple(self.args_w)))
         else:
-            args_repr = "()"
+            args_repr = u"()"
         clsname = self.getclass(space).getname(space)
         return space.wrap(clsname + args_repr)
 
@@ -556,7 +557,7 @@
             values_w = space.fixedview(self.args_w[1])
             w_tuple = space.newtuple(values_w + [self.w_lastlineno])
             args_w = [self.args_w[0], w_tuple]
-            args_repr = space.str_w(space.repr(space.newtuple(args_w)))
+            args_repr = space.unicode_w(space.repr(space.newtuple(args_w)))
             clsname = self.getclass(space).getname(space)
             return space.wrap(clsname + args_repr)
         else:
diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -261,3 +261,13 @@
             assert e.errno == errno.ENOTDIR
         else:
             assert False, "Expected OSError"
+
+    def test_nonascii_name(self):
+        """
+        class &#26085;&#26412;(Exception):
+            pass
+        assert '&#26085;&#26412;' in repr(&#26085;&#26412;)
+        class &#26085;&#26412;2(SyntaxError):
+            pass
+        assert '&#26085;&#26412;2' in repr(&#26085;&#26412;2)
+        """
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -172,7 +172,7 @@
 
     def descr__repr__(self):
         typename = space.type(self).getname(space)
-        return space.wrap("<%s owner=%d count=%d>" % (
+        return space.wrap(u"<%s owner=%d count=%d>" % (
                 typename, self.rlock_owner, self.rlock_count))
 
     @unwrap_spec(blocking=bool, timeout=float)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1151,7 +1151,7 @@
         w_self.w_dict = w_dict
 
     def descr_repr(self, space):
-        typename = space.type(self).getname(space).decode('utf-8')
+        typename = space.type(self).getname(space)
         w_seq = space.call_function(space.w_list, self)
         seq_repr = space.unicode_w(space.repr(w_seq))
         return space.wrap(u"%s(%s)" % (typename, seq_repr))
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -106,7 +106,7 @@
 
 def _get_printable_location(w_type):
     return ('list__do_extend_from_iterable [w_type=%s]' %
-            w_type.getname(w_type.space))
+            w_type.getname(w_type.space).encode('utf-8'))
 
 
 _do_extend_jitdriver = jit.JitDriver(
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -10,13 +10,13 @@
     w_module = w_type.lookup("__module__")
     if w_module is not None:
         try:
-            modulename = space.str_w(w_module)
+            modulename = space.unicode_w(w_module)
         except OperationError, e:
             if not e.match(space, space.w_TypeError):
                 raise
         else:
-            classname = '%s.%s' % (modulename, classname)
-    return w_obj.getrepr(space, '%s object' % (classname,))
+            classname = u'%s.%s' % (modulename, classname)
+    return w_obj.getrepr(space, u'%s object' % (classname,))
 
 def descr__str__(space, w_obj):
     w_type = space.type(w_obj)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -683,9 +683,24 @@
         assert d['A'].__module__ == 'builtins'    # obscure, follows CPython
         assert repr(d['A']) == "<class 'A'>"
 
-    def test_repr_unicode(self):
+    def test_repr_nonascii(self):
         assert repr(type('&#26085;&#26412;', (), {})) == "<class '%s.&#26085;&#26412;'>" % __name__
 
+    def test_name_nonascii(self):
+        assert type('&#26085;&#26412;', (), {}).__name__ == '&#26085;&#26412;'
+
+    def test_errors_nonascii(self):
+        # Check some arbitrary error messages
+        Japan = type('&#26085;&#26412;', (), {})
+        obj = Japan()
+        for f in hex, int, len, next, open, set, 'foo'.startswith:
+            try:
+                f(obj)
+            except TypeError as e:
+                assert '&#26085;&#26412;' in str(e)
+            else:
+                assert False, 'Expected TypeError'
+
     def test_invalid_mro(self):
         class A(object):
             pass
diff --git a/pypy/objspace/std/test/test_userobject.py b/pypy/objspace/std/test/test_userobject.py
--- a/pypy/objspace/std/test/test_userobject.py
+++ b/pypy/objspace/std/test/test_userobject.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 import py
 from pypy.interpreter import gateway
 from pypy.objspace.test import test_descriptor
@@ -214,6 +215,11 @@
         s = repr(Foo())
         assert s.startswith('<a.b.c.Foo object at ')
 
+    def test_repr_nonascii(self):
+        Japan = type('&#26085;&#26412;', (), dict(__module__='&#26085;&#26412;&#22269;'))
+        s = repr(Japan())
+        assert s.startswith('<&#26085;&#26412;&#22269;.&#26085;&#26412; object at ')
+
     def test_del_attr(self):
         class Foo(object):
             def __init__(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -483,14 +483,14 @@
     def get_module_type_name(w_self):
         space = w_self.space
         w_mod = w_self.get_module()
-        if not space.isinstance_w(w_mod, space.w_str):
-            mod = 'builtins'
+        if not space.isinstance_w(w_mod, space.w_unicode):
+            mod = u'builtins'
         else:
-            mod = space.str_w(w_mod)
-        if mod != 'builtins':
-            return '%s.%s' % (mod, w_self.name)
+            mod = space.unicode_w(w_mod)
+        if mod != u'builtins':
+            return u'%s.%s' % (mod, w_self.name.decode('utf-8'))
         else:
-            return w_self.name
+            return w_self.name.decode('utf-8')
 
     def add_subclass(w_self, w_subclass):
         space = w_self.space
@@ -622,7 +622,7 @@
 
 def descr_get__name__(space, w_type):
     w_type = _check(space, w_type)
-    return space.wrap(w_type.name)
+    return space.wrap(w_type.name.decode('utf-8'))
 
 def descr_set__name__(space, w_type, w_value):
     w_type = _check(space, w_type)
@@ -1217,7 +1217,7 @@
     cycle.reverse()
     names = [cls.getname(space) for cls in cycle]
     raise OperationError(space.w_TypeError,
-        space.wrap("cycle among base classes: " + ' < '.join(names)))
+        space.wrap(u"cycle among base classes: " + u' < '.join(names)))
 
 # ____________________________________________________________
 
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -87,7 +87,7 @@
     def __init__(self, space, operr):
         self.space = space
         self.operr = operr
-        self.typename = operr.w_type.getname(space)
+        self.typename = operr.w_type.getname(space).encode('utf-8')
         self.traceback = AppTraceback(space, self.operr.get_traceback())
         debug_excs = getattr(operr, 'debug_excs', [])
         if debug_excs:

From noreply at buildbot.pypy.org  Tue Jun  4 03:35:30 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue,  4 Jun 2013 03:35:30 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix translation
Message-ID: <20130604013530.765FE1C3009@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64743:a554ac771ba3
Date: 2013-06-03 18:34 -0700
http://bitbucket.org/pypy/pypy/changeset/a554ac771ba3/

Log:	fix translation

diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py
--- a/pypy/module/cpyext/classobject.py
+++ b/pypy/module/cpyext/classobject.py
@@ -29,7 +29,7 @@
 
     def descr_repr(self, space):
         return self.getrepr(space, u'<instancemethod %s>' %
-                            (space.getname(self.w_function),))
+                            (self.w_function.getname(space),))
 
 InstanceMethod.typedef = TypeDef("instancemethod",
     __new__ = interp2app(InstanceMethod.descr_new),

From noreply at buildbot.pypy.org  Tue Jun  4 06:42:55 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue,  4 Jun 2013 06:42:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: Inline into rbigint so that comparisons
 can be constant folded.
Message-ID: <20130604044255.DD11E1C010B@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64744:661d7f7624dc
Date: 2013-06-03 20:41 -0700
http://bitbucket.org/pypy/pypy/changeset/661d7f7624dc/

Log:	Inline into rbigint so that comparisons can be constant folded.

diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -118,7 +118,7 @@
     def look_inside_function(self, func):
         mod = func.__module__ or '?'
 
-        if mod == 'rpython.rlib.rbigint' or mod == 'rpython.rlib.rlocale' or mod == 'rpython.rlib.rsocket':
+        if mod == 'rpython.rlib.rlocale' or mod == 'rpython.rlib.rsocket':
             return False
         if mod.startswith('pypy.interpreter.astcompiler.'):
             return False
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -8,7 +8,7 @@
 
 def test_bigint():
     from rpython.rlib.rbigint import rbigint
-    assert not pypypolicy.look_inside_function(rbigint.lt.im_func)
+    assert pypypolicy.look_inside_function(rbigint.lt.im_func)
 
 def test_rlocale():
     from rpython.rlib.rlocale import setlocale
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -74,7 +74,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_mixed_type_loop(self):
         def main(n):
             i = 0.0
@@ -94,7 +93,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_cached_pure_func_of_equal_fields(self):
         def main(n):
             class A(object):
@@ -196,7 +194,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_chain_of_guards(self):
         src = """
         class A(object):
@@ -220,7 +217,6 @@
         loops = log.loops_by_filename(self.filepath)
         assert len(loops) == 1
 
-
     def test_unpack_iterable_non_list_tuple(self):
         def main(n):
             import array
@@ -258,7 +254,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_dont_trace_every_iteration(self):
         def main(a, b):
             i = sa = 0
@@ -289,7 +284,6 @@
         assert log.result == 300 * (-10 % -20)
         assert log.jit_summary.tracing_no == 1
 
-
     def test_overflow_checking(self):
         """
         This test only checks that we get the expected result, not that any
@@ -298,7 +292,8 @@
         def main():
             import sys
             def f(a,b):
-                if a < 0: return -1
+                if a < 0:
+                    return -1
                 return a-b
             #
             total = sys.maxint - 2147483647
@@ -309,7 +304,6 @@
         #
         self.run_and_check(main, [])
 
-
     def test_global(self):
         log = self.run("""
         i = 0
@@ -404,3 +398,14 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(log.loops_by_id("exc_info")) == 1
+
+    def test_long_comparison(self):
+        def main(n):
+            while n:
+                12345L > 123L  # ID: long_op
+                n -= 1
+
+        log = self.run(main, [300])
+        loop, = log.loops_by_id("long_op")
+        assert log.match("""
+        """)
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -332,7 +332,7 @@
             newstr = s2.malloc(len1 + len2)
             newstr.copy_contents_from_str(s1, newstr, 0, 0, len1)
         else:
-            newstr = s1.malloc(len1 + len2)            
+            newstr = s1.malloc(len1 + len2)
             newstr.copy_contents(s1, newstr, 0, 0, len1)
         if typeOf(s2) == Ptr(STR):
             newstr.copy_contents_from_str(s2, newstr, 0, len1, len2)

From noreply at buildbot.pypy.org  Tue Jun  4 06:42:57 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue,  4 Jun 2013 06:42:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: trailing whitespace
Message-ID: <20130604044257.217031C0619@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64745:4830df760669
Date: 2013-06-03 20:44 -0700
http://bitbucket.org/pypy/pypy/changeset/4830df760669/

Log:	trailing whitespace

diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -2,12 +2,14 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.history import BoxInt, ConstInt
+
 MAXINT = maxint
 MININT = -maxint - 1
 
+
 class IntBound(object):
     _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
-    
+
     def __init__(self, lower, upper):
         self.has_upper = True
         self.has_lower = True
@@ -29,7 +31,7 @@
 
     def make_lt(self, other):
         return self.make_le(other.add(-1))
- 
+
     def make_ge(self, other):
         if other.has_lower:
             if not self.has_lower or other.lower > self.lower:
@@ -86,7 +88,7 @@
                 r = True
 
         return r
-    
+
     def add(self, offset):
         res = self.clone()
         try:
@@ -101,7 +103,7 @@
 
     def mul(self, value):
         return self.mul_bound(IntBound(value, value))
-    
+
     def add_bound(self, other):
         res = self.clone()
         if other.has_upper:
@@ -115,7 +117,7 @@
             try:
                 res.lower = ovfcheck(res.lower + other.lower)
             except OverflowError:
-                res.has_lower = False            
+                res.has_lower = False
         else:
             res.has_lower = False
         return res
@@ -133,7 +135,7 @@
             try:
                 res.lower = ovfcheck(res.lower - other.upper)
             except OverflowError:
-                res.has_lower = False            
+                res.has_lower = False
         else:
             res.has_lower = False
         return res
@@ -196,7 +198,6 @@
         else:
             return IntUnbounded()
 
-
     def contains(self, val):
         if self.has_lower and val < self.lower:
             return False
@@ -216,7 +217,7 @@
         elif self.has_upper:
             return False
         return True
-        
+
     def __repr__(self):
         if self.has_lower:
             l = '%d' % self.lower
@@ -249,7 +250,7 @@
             guards.append(op)
             op = ResOperation(rop.GUARD_TRUE, [res], None)
             guards.append(op)
-    
+
 
 class IntUpperBound(IntBound):
     def __init__(self, upper):
@@ -285,7 +286,7 @@
         self._raise()
     def make_constant(self, value):
         self._raise()
-    def intersect(self, other):        
+    def intersect(self, other):
         self._raise()
 
 def min4(t):

From noreply at buildbot.pypy.org  Tue Jun  4 06:42:59 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue,  4 Jun 2013 06:42:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20130604044259.6C9841C0F88@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64746:dd7556ff00b9
Date: 2013-06-03 23:42 -0500
http://bitbucket.org/pypy/pypy/changeset/dd7556ff00b9/

Log:	merged upstream

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -58,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -80,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -151,6 +151,8 @@
     'DEBUG': DEBUG,
     'NOTSET': NOTSET,
 }
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_testcapi.py
@@ -0,0 +1,61 @@
+import os, sys, imp
+import tempfile
+
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
+    """
+    thisdir = os.path.dirname(__file__)
+    output_dir = tempfile.mkdtemp()
+
+    from distutils.ccompiler import new_compiler
+
+    compiler = new_compiler()
+    compiler.output_dir = output_dir
+
+    # Compile .c file
+    include_dir = os.path.join(thisdir, '..', 'include')
+    if sys.platform == 'win32':
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
+    else:
+        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
+    res = compiler.compile([os.path.join(thisdir, csource)],
+                           include_dirs=[include_dir],
+                           extra_preargs=ccflags)
+    object_filename = res[0]
+
+    # set link options
+    output_filename = modulename + _get_c_extension_suffix()
+    if sys.platform == 'win32':
+        # XXX libpypy-c.lib is currently not installed automatically
+        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
+        libraries = [library, 'oleaut32']
+        extra_ldargs = ['/MANIFEST',  # needed for VC10
+                        '/EXPORT:init' + modulename]
+    else:
+        libraries = []
+        extra_ldargs = []
+
+    # link the dynamic library
+    compiler.link_shared_object(
+        [object_filename],
+        output_filename,
+        libraries=libraries,
+        extra_preargs=extra_ldargs)
+
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,62 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
 try:
     import cpyext
 except ImportError:
     raise ImportError("No module named '_testcapi'")
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,7 @@
+
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -46,3 +46,7 @@
 
 .. branch: operrfmt-NT
 Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -2,7 +2,7 @@
 PyPy on Windows
 ===============
 
-Pypy is supported on Windows platforms, starting with Windows 2000.
+PyPy is supported on Windows platforms, starting with Windows 2000.
 The following text gives some hints about how to translate the PyPy
 interpreter.
 
@@ -199,9 +199,9 @@
 
 or such, depending on your mingw64 download.
 
-hacking on Pypy with the mingw compiler
+hacking on PyPy with the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since hacking on Pypy means running tests, you will need a way to specify
+Since hacking on PyPy means running tests, you will need a way to specify
 the mingw compiler when hacking (as opposed to translating). As of
 March 2012, --cc is not a valid option for pytest.py. However if you set an
 environment variable CC to the compliter exe, testing will use it.
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -578,6 +578,11 @@
 class TestNonInteractive:
     def run_with_status_code(self, cmdline, senddata='', expect_prompt=False,
             expect_banner=False, python_flags='', env=None):
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         cmdline = '%s %s "%s" %s' % (sys.executable, python_flags,
                                      app_main, cmdline)
         print 'POPEN:', cmdline
@@ -706,6 +711,11 @@
         assert 'copyright' not in data
 
     def test_non_interactive_stdout_fully_buffered(self):
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         path = getscript(r"""
             import sys, time
             sys.stdout.write('\x00(STDOUT)\n\x00')   # stays in buffers
@@ -726,6 +736,11 @@
 
     def test_non_interactive_stdout_unbuffered(self, monkeypatch):
         monkeypatch.setenv('PYTHONUNBUFFERED', '1')
+        if os.name == 'nt':
+            try:
+                import __pypy__
+            except:
+                py.test.skip('app_main cannot run on non-pypy for windows')
         path = getscript(r"""
             import sys, time
             sys.stdout.write('\x00(STDOUT)\n\x00')
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -1,3 +1,4 @@
+import sys
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 
@@ -113,6 +114,14 @@
 
 # ____________________________________________________________
 
+SF_MSVC_BITFIELDS = 1
+
+if sys.platform == 'win32':
+    DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+else:
+    DEFAULT_SFLAGS = 0
+
+
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -121,9 +130,11 @@
 def new_union_type(space, name):
     return ctypestruct.W_CTypeUnion(space, name)
 
- at unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int)
+ at unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
+             sflags=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1):
+                             totalsize=-1, totalalignment=-1,
+                             sflags=DEFAULT_SFLAGS):
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise OperationError(space.w_TypeError,
@@ -134,6 +145,8 @@
     alignment = 1
     boffset = 0         # this number is in *bits*, not bytes!
     boffsetmax = 0      # the maximum value of boffset, in bits too
+    prev_bitfield_size = 0
+    prev_bitfield_free = 0
     fields_w = space.listview(w_fields)
     fields_list = []
     fields_dict = {}
@@ -166,7 +179,15 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield
         falign = ftype.alignof()
-        if alignment < falign and (fbitsize < 0 or fname != ''):
+        do_align = True
+        if fbitsize >= 0:
+            if (sflags & SF_MSVC_BITFIELDS) == 0:
+                # GCC: anonymous bitfields (of any size) don't cause alignment
+                do_align = (fname != '')
+            else:
+                # MSVC: zero-sized bitfields don't cause alignment
+                do_align = (fbitsize > 0)
+        if alignment < falign and do_align:
             alignment = falign
         #
         if fbitsize < 0:
@@ -208,6 +229,7 @@
                 fields_dict[fname] = fld
 
             boffset += ftype.size * 8
+            prev_bitfield_size = 0
 
         else:
             # this is the case of a bitfield
@@ -243,31 +265,67 @@
                     raise operationerrfmt(space.w_TypeError,
                                           "field '%s.%s' is declared with :0",
                                           w_ctype.name, fname)
-                if boffset > field_offset_bytes * 8:
-                    field_offset_bytes += falign
-                    assert boffset < field_offset_bytes * 8
-                boffset = field_offset_bytes * 8
+                if (sflags & SF_MSVC_BITFIELDS) == 0:
+                    # GCC's notion of "ftype :0;"
+                    # pad boffset to a value aligned for "ftype"
+                    if boffset > field_offset_bytes * 8:
+                        field_offset_bytes += falign
+                        assert boffset < field_offset_bytes * 8
+                    boffset = field_offset_bytes * 8
+                else:
+                    # MSVC's notion of "ftype :0;
+                    # Mostly ignored.  It seems they only serve as
+                    # separator between other bitfields, to force them
+                    # into separate words.
+                    pass
+                prev_bitfield_size = 0
+
             else:
-                # Can the field start at the offset given by 'boffset'?  It
-                # can if it would entirely fit into an aligned ftype field.
-                bits_already_occupied = boffset - (field_offset_bytes * 8)
+                if (sflags & SF_MSVC_BITFIELDS) == 0:
+                    # GCC's algorithm
 
-                if bits_already_occupied + fbitsize > 8 * ftype.size:
-                    # it would not fit, we need to start at the next
-                    # allowed position
-                    field_offset_bytes += falign
-                    assert boffset < field_offset_bytes * 8
-                    boffset = field_offset_bytes * 8
-                    bitshift = 0
+                    # Can the field start at the offset given by 'boffset'?  It
+                    # can if it would entirely fit into an aligned ftype field.
+                    bits_already_occupied = boffset - (field_offset_bytes * 8)
+
+                    if bits_already_occupied + fbitsize > 8 * ftype.size:
+                        # it would not fit, we need to start at the next
+                        # allowed position
+                        field_offset_bytes += falign
+                        assert boffset < field_offset_bytes * 8
+                        boffset = field_offset_bytes * 8
+                        bitshift = 0
+                    else:
+                        bitshift = bits_already_occupied
+                        assert bitshift >= 0
+                    boffset += fbitsize
+
                 else:
-                    bitshift = bits_already_occupied
+                    # MSVC's algorithm
+
+                    # A bitfield is considered as taking the full width
+                    # of their declared type.  It can share some bits
+                    # with the previous field only if it was also a
+                    # bitfield and used a type of the same size.
+                    if (prev_bitfield_size == ftype.size and
+                        prev_bitfield_free >= fbitsize):
+                        # yes: reuse
+                        bitshift = 8 * prev_bitfield_size - prev_bitfield_free
+                    else:
+                        # no: start a new full field
+                        boffset = (boffset + falign*8-1) & ~(falign*8-1)
+                        boffset += ftype.size * 8
+                        bitshift = 0
+                        prev_bitfield_size = ftype.size
+                        prev_bitfield_free = 8 * prev_bitfield_size
+                    #
+                    prev_bitfield_free -= fbitsize
+                    field_offset_bytes = boffset / 8 - ftype.size
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
-            
-                boffset += fbitsize
 
         if boffset > boffsetmax:
             boffsetmax = boffset
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2757,36 +2757,57 @@
     assert wr() is None
     py.test.raises(RuntimeError, from_handle, cast(BCharP, 0))
 
-def test_bitfield_as_gcc():
+def _test_bitfield_details(flag):
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
+    BUInt = new_primitive_type("unsigned int")
     BStruct = new_struct_type("foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
-                                       ('b', BInt, 9),
-                                       ('c', BChar, -1)])
-    assert typeoffsetof(BStruct, 'c') == (BChar, 3)
-    assert sizeof(BStruct) == 4
+                                       ('b1', BInt, 9),
+                                       ('b2', BUInt, 7),
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 3)
+        assert sizeof(BStruct) == 4
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 8)
+        assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
-                                       ('c', BChar, -1)])
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
     assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
-    assert alignof(BStruct) == 1
+    if flag == 0:   # gcc
+        assert sizeof(BStruct) == 5
+        assert alignof(BStruct) == 1
+    else:           # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
     #
     BStruct = new_struct_type("foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
-                                       ('c', BChar, -1)])
-    assert typeoffsetof(BStruct, 'c') == (BChar, 4)
-    assert sizeof(BStruct) == 5
+                                       ('c', BChar, -1)], -1, -1, -1, flag)
+    if flag == 0:   # gcc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 5
+    else:           # msvc
+        assert typeoffsetof(BStruct, 'c') == (BChar, 1)
+        assert sizeof(BStruct) == 2
     assert alignof(BStruct) == 1
 
 
+def test_bitfield_as_gcc():
+    _test_bitfield_details(flag=0)
+
+def test_bitfield_as_msvc():
+    _test_bitfield_details(flag=1)
+
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "0.7"
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -303,7 +303,7 @@
         return _absolute_import(space, modulename, baselevel,
                                 fromlist_w, tentative)
     finally:
-        lock.release_lock()
+        lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
 def absolute_import_try(space, modulename, baselevel, fromlist_w):
@@ -788,9 +788,13 @@
             self.lockowner = me
         self.lockcounter += 1
 
-    def release_lock(self):
+    def release_lock(self, silent_after_fork):
         me = self.space.getexecutioncontext()   # used as thread ident
         if self.lockowner is not me:
+            if self.lockowner is None and silent_after_fork:
+                # Too bad.  This situation can occur if a fork() occurred
+                # with the import lock held, and we're the child.
+                return
             if not self._can_have_lock():
                 return
             space = self.space
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -177,7 +177,7 @@
 
 def release_lock(space):
     if space.config.objspace.usemodules.thread:
-        importing.getimportlock(space).release_lock()
+        importing.getimportlock(space).release_lock(silent_after_fork=False)
 
 def reinit_lock(space):
     if space.config.objspace.usemodules.thread:
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -30,7 +30,7 @@
 def new_dtype_getter(name):
     def _get_dtype(space):
         from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-        return getattr(get_dtype_cache(space), "w_%sdtype" % name)
+        return get_dtype_cache(space).dtypes_by_name[name]
 
     def new(space, w_subtype, w_value):
         dtype = _get_dtype(space)
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -273,7 +273,6 @@
 
         fields = space.getitem(w_data, space.wrap(4))
         self.set_fields(space, fields)
-        print self.itemtype
 
 class W_ComplexDtype(W_Dtype):
     def __init__(self, itemtype, num, kind, name, char, w_box_type,
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -21,7 +21,3 @@
 
         a = zeros(3)
         assert loads(dumps(sum(a))) == sum(a)
-
-    def setup_class(cls):
-        import py
-        py.test.xfail("FIXME: dtype('int32') == dtype('int32') fails")
diff --git a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
--- a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
+++ b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
@@ -41,6 +41,10 @@
     assert 'LOG_NOTICE' in d
 
 def test_resource():
+    try:
+        import lib_pypy.resource
+    except ImportError:
+        py.test.skip('no syslog on this platform')
     d = run('resource.ctc.py', '_resource_cache.py')
     assert 'RLIM_NLIMITS' in d
 
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -352,9 +352,8 @@
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):
-                # convert string to unicode explicitely here
-                from pypy.objspace.std.unicodetype import plain_str2unicode
-                r = plain_str2unicode(self.space, r)
+                # convert string to unicode using the default encoding
+                r = self.space.unicode_w(self.space.wrap(r))
             prec = self.prec
             if prec == -1 and self.width == 0:
                 # fast path
@@ -509,12 +508,10 @@
             result = formatter.format()
         except NeedUnicodeFormattingError:
             # fall through to the unicode case
-            from pypy.objspace.std.unicodetype import plain_str2unicode
-            fmt = plain_str2unicode(space, fmt)
+            pass
         else:
             return space.wrap(result)
-    else:
-        fmt = space.unicode_w(w_fmt)
+    fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
     return space.wrap(result)
diff --git a/pypy/objspace/std/test/test_stringobject.py b/pypy/objspace/std/test/test_stringobject.py
--- a/pypy/objspace/std/test/test_stringobject.py
+++ b/pypy/objspace/std/test/test_stringobject.py
@@ -530,6 +530,12 @@
                 del sys.modules[module_name]
             temp_sys.setdefaultencoding('utf-8')
             assert u''.join(['\xc3\xa1']) == u'\xe1'
+            #
+            assert ('\xc3\xa1:%s' % u'\xe2') == u'\xe1:\xe2'
+            class Foo(object):
+                def __repr__(self):
+                    return '\xc3\xa2'
+            assert u'\xe1:%r' % Foo() == u'\xe1:\xe2'
         finally:
             temp_sys.setdefaultencoding(old_encoding)
             sys.modules.update(self.original_modules)
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -13,22 +13,6 @@
     from pypy.objspace.std.unicodeobject import W_UnicodeObject
     return W_UnicodeObject(uni)
 
-def plain_str2unicode(space, s):
-    try:
-        return unicode(s)
-    except UnicodeDecodeError:
-        for i in range(len(s)):
-            if ord(s[i]) > 127:
-                raise OperationError(
-                    space.w_UnicodeDecodeError,
-                    space.newtuple([
-                    space.wrap('ascii'),
-                    space.wrap(s),
-                    space.wrap(i),
-                    space.wrap(i+1),
-                    space.wrap("ordinal not in range(128)")]))
-        assert False, "unreachable"
-
 
 unicode_capitalize = SMM('capitalize', 1,
                          doc='S.capitalize() -> unicode\n\nReturn a'
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -8,6 +8,7 @@
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.opassembler import ResOpAssembler
 from rpython.jit.backend.arm.regalloc import (Regalloc,
     CoreRegisterManager, check_imm_arg, VFPRegisterManager,
@@ -961,7 +962,7 @@
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VLDR(target.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
@@ -982,7 +983,7 @@
             return self._store_core_reg(mc, source, base, ofs, cond, helper)
 
     def _store_vfp_reg(self, mc, source, base, ofs, cond=c.AL, helper=r.ip):
-        if check_imm_arg(ofs):
+        if check_imm_arg(ofs, VMEM_imm_size):
             mc.VSTR(source.value, base.value, imm=ofs, cond=cond)
         else:
             mc.gen_load_int(helper.value, ofs, cond=cond)
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -52,6 +52,8 @@
 
     def _push_stack_args(self, stack_args, on_stack):
         assert on_stack % 8 == 0
+        if on_stack == 0:
+            return
         self._adjust_sp(-on_stack)
         self.current_sp = on_stack
         ofs = 0
@@ -71,7 +73,7 @@
             else:
                 self.mc.gen_load_int(r.ip.value, n)
                 self.mc.ADD_rr(r.sp.value, r.sp.value, r.ip.value)
-        else:
+        elif n < 0:
             n = abs(n)
             if check_imm_arg(n):
                 self.mc.SUB_ri(r.sp.value, r.sp.value, n)
diff --git a/rpython/jit/backend/arm/helper/regalloc.py b/rpython/jit/backend/arm/helper/regalloc.py
--- a/rpython/jit/backend/arm/helper/regalloc.py
+++ b/rpython/jit/backend/arm/helper/regalloc.py
@@ -4,7 +4,10 @@
 from rpython.jit.metainterp.history import ConstInt
 from rpython.rlib.objectmodel import we_are_translated
 
-def check_imm_arg(arg, size=0xFF, allow_zero=True):
+VMEM_imm_size=0x3FC
+default_imm_size=0xFF
+
+def check_imm_arg(arg, size=default_imm_size, allow_zero=True):
     assert not isinstance(arg, ConstInt)
     if not we_are_translated():
         if not isinstance(arg, int):
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -15,6 +15,7 @@
                                                 gen_emit_unary_float_op,
                                                 saved_registers)
 from rpython.jit.backend.arm.helper.regalloc import check_imm_arg
+from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.jump import remap_frame_layout
 from rpython.jit.backend.arm.regalloc import TempBox
@@ -23,6 +24,7 @@
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import InteriorFieldDescr
 from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler
+from rpython.jit.backend.llsupport.regalloc import get_scale
 from rpython.jit.metainterp.history import (Box, AbstractFailDescr,
                                             INT, FLOAT, REF)
 from rpython.jit.metainterp.history import TargetToken
@@ -523,35 +525,9 @@
 
     def emit_op_setfield_gc(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs, size = arglocs
-        if size.value == 8:
-            assert value_loc.is_vfp_reg()
-            # vstr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            if ofs.is_reg():
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
-                base_loc = r.ip
-                ofs = imm(0)
-            else:
-                assert ofs.value % 4 == 0
-            self.mc.VSTR(value_loc.value, base_loc.value, ofs.value)
-        elif size.value == 4:
-            if ofs.is_imm():
-                self.mc.STR_ri(value_loc.value, base_loc.value, ofs.value)
-            else:
-                self.mc.STR_rr(value_loc.value, base_loc.value, ofs.value)
-        elif size.value == 2:
-            if ofs.is_imm():
-                self.mc.STRH_ri(value_loc.value, base_loc.value, ofs.value)
-            else:
-                self.mc.STRH_rr(value_loc.value, base_loc.value, ofs.value)
-        elif size.value == 1:
-            if ofs.is_imm():
-                self.mc.STRB_ri(value_loc.value, base_loc.value, ofs.value)
-            else:
-                self.mc.STRB_rr(value_loc.value, base_loc.value, ofs.value)
-        else:
-            assert 0
+        scale = get_scale(size.value)
+        self._write_to_mem(value_loc, base_loc,
+                                ofs, imm(scale), fcond)
         return fcond
 
     emit_op_setfield_raw = emit_op_setfield_gc
@@ -559,47 +535,8 @@
     def emit_op_getfield_gc(self, op, arglocs, regalloc, fcond):
         base_loc, ofs, res, size = arglocs
         signed = op.getdescr().is_field_signed()
-        if size.value == 8:
-            assert res.is_vfp_reg()
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            if ofs.is_reg():
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
-                base_loc = r.ip
-                ofs = imm(0)
-            else:
-                assert ofs.value % 4 == 0
-            self.mc.VLDR(res.value, base_loc.value, ofs.value)
-        elif size.value == 4:
-            if ofs.is_imm():
-                self.mc.LDR_ri(res.value, base_loc.value, ofs.value)
-            else:
-                self.mc.LDR_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 2:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSH_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSH_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRH_rr(res.value, base_loc.value, ofs.value)
-        elif size.value == 1:
-            if ofs.is_imm():
-                if signed:
-                    self.mc.LDRSB_ri(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_ri(res.value, base_loc.value, ofs.value)
-            else:
-                if signed:
-                    self.mc.LDRSB_rr(res.value, base_loc.value, ofs.value)
-                else:
-                    self.mc.LDRB_rr(res.value, base_loc.value, ofs.value)
-        else:
-            assert 0
+        scale = get_scale(size.value)
+        self._load_from_mem(res, base_loc, ofs, imm(scale), signed, fcond)
         return fcond
 
     emit_op_getfield_raw = emit_op_getfield_gc
@@ -609,72 +546,44 @@
     def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond):
         (base_loc, index_loc, res_loc,
             ofs_loc, ofs, itemsize, fieldsize) = arglocs
-        self.mc.gen_load_int(r.ip.value, itemsize.value)
-        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        scale = get_scale(fieldsize.value)
+        tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+        assert not save
+        self.mc.gen_load_int(tmploc.value, itemsize.value)
+        self.mc.MUL(tmploc.value, index_loc.value, tmploc.value)
         descr = op.getdescr()
         assert isinstance(descr, InteriorFieldDescr)
         signed = descr.fielddescr.is_field_signed()
         if ofs.value > 0:
             if ofs_loc.is_imm():
-                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+                self.mc.ADD_ri(tmploc.value, tmploc.value, ofs_loc.value)
             else:
-                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
-
-        if fieldsize.value == 8:
-            # vldr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            assert res_loc.is_vfp_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, 0)
-        elif fieldsize.value == 4:
-            self.mc.LDR_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 2:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 1:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value, r.ip.value)
-            else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value, r.ip.value)
-        else:
-            assert 0
-
+                self.mc.ADD_rr(tmploc.value, tmploc.value, ofs_loc.value)
+        ofs_loc = tmploc
+        self._load_from_mem(res_loc, base_loc, ofs_loc,
+                                imm(scale), signed, fcond)
         return fcond
 
     def emit_op_setinteriorfield_gc(self, op, arglocs, regalloc, fcond):
         (base_loc, index_loc, value_loc,
             ofs_loc, ofs, itemsize, fieldsize) = arglocs
-        self.mc.gen_load_int(r.ip.value, itemsize.value)
-        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        scale = get_scale(fieldsize.value)
+        tmploc, save = self.get_tmp_reg([base_loc, index_loc, value_loc, ofs_loc])
+        assert not save
+        self.mc.gen_load_int(tmploc.value, itemsize.value)
+        self.mc.MUL(tmploc.value, index_loc.value, tmploc.value)
         if ofs.value > 0:
             if ofs_loc.is_imm():
-                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+                self.mc.ADD_ri(tmploc.value, tmploc.value, ofs_loc.value)
             else:
-                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
-        if fieldsize.value == 8:
-            # vstr only supports imm offsets
-            # so if the ofset is too large we add it to the base and use an
-            # offset of 0
-            assert value_loc.is_vfp_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
-            self.mc.VSTR(value_loc.value, r.ip.value, 0)
-        elif fieldsize.value == 4:
-            self.mc.STR_rr(value_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 2:
-            self.mc.STRH_rr(value_loc.value, base_loc.value, r.ip.value)
-        elif fieldsize.value == 1:
-            self.mc.STRB_rr(value_loc.value, base_loc.value, r.ip.value)
-        else:
-            assert 0
+                self.mc.ADD_rr(tmploc.value, tmploc.value, ofs_loc.value)
+        self._write_to_mem(value_loc, base_loc, tmploc, imm(scale), fcond)
         return fcond
     emit_op_setinteriorfield_raw = emit_op_setinteriorfield_gc
 
     def emit_op_arraylen_gc(self, op, arglocs, regalloc, fcond):
         res, base_loc, ofs = arglocs
-        self.mc.LDR_ri(res.value, base_loc.value, ofs.value)
+        self.load_reg(self.mc, res, base_loc, ofs.value)
         return fcond
 
     def emit_op_setarrayitem_gc(self, op, arglocs, regalloc, fcond):
@@ -694,18 +603,40 @@
     def _write_to_mem(self, value_loc, base_loc, ofs_loc, scale, fcond=c.AL):
         if scale.value == 3:
             assert value_loc.is_vfp_reg()
-            assert ofs_loc.is_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value)
-            self.mc.VSTR(value_loc.value, r.ip.value, cond=fcond)
+            # vstr only supports imm offsets
+            # so if the ofset is too large we add it to the base and use an
+            # offset of 0
+            if ofs_loc.is_reg():
+                tmploc, save = self.get_tmp_reg([value_loc, base_loc, ofs_loc])
+                assert not save
+                self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
+                base_loc = tmploc
+                ofs_loc = imm(0)
+            else:
+                assert ofs_loc.is_imm()
+                assert ofs_loc.value % 4 == 0
+            self.mc.VSTR(value_loc.value, base_loc.value, ofs_loc.value)
         elif scale.value == 2:
-            self.mc.STR_rr(value_loc.value, base_loc.value, ofs_loc.value,
-                                                                    cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.STR_ri(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.STR_rr(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 1:
-            self.mc.STRH_rr(value_loc.value, base_loc.value, ofs_loc.value,
-                                                                    cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.STRH_ri(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.STRH_rr(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 0:
-            self.mc.STRB_rr(value_loc.value, base_loc.value, ofs_loc.value,
-                                                                    cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.STRB_ri(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.STRB_rr(value_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         else:
             assert 0
 
@@ -731,33 +662,63 @@
             self.mc.ADD_ri(r.ip.value, ofs_loc.value, imm=ofs.value)
             ofs_loc = r.ip
         #
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def _load_from_mem(self, res_loc, base_loc, ofs_loc, scale,
                                             signed=False, fcond=c.AL):
         if scale.value == 3:
             assert res_loc.is_vfp_reg()
-            assert ofs_loc.is_reg()
-            self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value)
-            self.mc.VLDR(res_loc.value, r.ip.value, cond=fcond)
+            # vldr only supports imm offsets
+            # if the offset is in a register we add it to the base and use a
+            # tmp reg
+            if ofs_loc.is_reg():
+                tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
+                assert not save
+                self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
+                base_loc = tmploc
+                ofs_loc = imm(0)
+            else:
+                assert ofs_loc.is_imm()
+                assert ofs_loc.value % 4 == 0
+            self.mc.VLDR(res_loc.value, base_loc.value, ofs_loc.value, cond=fcond)
         elif scale.value == 2:
-            self.mc.LDR_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                self.mc.LDR_ri(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
+            else:
+                self.mc.LDR_rr(res_loc.value, base_loc.value,
+                                ofs_loc.value, cond=fcond)
         elif scale.value == 1:
-            if signed:
-                self.mc.LDRSH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRH_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRH_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         elif scale.value == 0:
-            if signed:
-                self.mc.LDRSB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+            if ofs_loc.is_imm():
+                if signed:
+                    self.mc.LDRSB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_ri(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
             else:
-                self.mc.LDRB_rr(res_loc.value, base_loc.value,
-                                 ofs_loc.value, cond=fcond)
+                if signed:
+                    self.mc.LDRSB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
+                else:
+                    self.mc.LDRB_rr(res_loc.value, base_loc.value,
+                                        ofs_loc.value, cond=fcond)
         else:
             assert 0
 
@@ -770,7 +731,7 @@
         # no base offset
         assert ofs.value == 0
         signed = op.getdescr().is_item_signed()
-        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed, fcond)
         return fcond
 
     def emit_op_strlen(self, op, arglocs, regalloc, fcond):
@@ -993,7 +954,7 @@
                 assert result_loc.is_vfp_reg()
                 # we always have a register here, since we have to sync them
                 # before call_assembler
-                self.mc.VLDR(result_loc.value, r.r0.value, imm=ofs)
+                self.load_reg(self.mc, result_loc, r.r0, ofs=ofs)
             else:
                 assert result_loc is r.r0
                 ofs = self.cpu.unpack_arraydescr(descr)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -13,7 +13,9 @@
                                                     prepare_cmp_op,
                                                     prepare_float_op,
                                                     check_imm_arg,
-                                                    check_imm_box
+                                                    check_imm_box,
+                                                    VMEM_imm_size,
+                                                    default_imm_size,
                                                     )
 from rpython.jit.backend.arm.jump import remap_frame_layout_mixed
 from rpython.jit.backend.arm.arch import WORD, JITFRAME_FIXED_SIZE
@@ -811,7 +813,8 @@
         ofs, size, sign = unpack_fielddescr(op.getdescr())
         base_loc = self.make_sure_var_in_reg(a0, boxes)
         value_loc = self.make_sure_var_in_reg(a1, boxes)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if size < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = imm(ofs)
         else:
             ofs_loc = self.get_scratch_reg(INT, boxes)
@@ -825,7 +828,8 @@
         ofs, size, sign = unpack_fielddescr(op.getdescr())
         base_loc = self.make_sure_var_in_reg(a0)
         immofs = imm(ofs)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if size < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = immofs
         else:
             ofs_loc = self.get_scratch_reg(INT, [a0])
@@ -846,7 +850,8 @@
         base_loc = self.make_sure_var_in_reg(op.getarg(0), args)
         index_loc = self.make_sure_var_in_reg(op.getarg(1), args)
         immofs = imm(ofs)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if fieldsize < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = immofs
         else:
             ofs_loc = self.get_scratch_reg(INT, args)
@@ -865,7 +870,8 @@
         index_loc = self.make_sure_var_in_reg(op.getarg(1), args)
         value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
         immofs = imm(ofs)
-        if check_imm_arg(ofs):
+        ofs_size = default_imm_size if fieldsize < 8 else VMEM_imm_size
+        if check_imm_arg(ofs, size=ofs_size):
             ofs_loc = immofs
         else:
             ofs_loc = self.get_scratch_reg(INT, args)
@@ -890,8 +896,8 @@
         scale = get_scale(size)
         args = op.getarglist()
         base_loc = self.make_sure_var_in_reg(args[0], args)
+        value_loc = self.make_sure_var_in_reg(args[2], args)
         ofs_loc = self.make_sure_var_in_reg(args[1], args)
-        value_loc = self.make_sure_var_in_reg(args[2], args)
         assert check_imm_arg(ofs)
         return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)]
     prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc
diff --git a/rpython/jit/backend/arm/test/test_regalloc_mov.py b/rpython/jit/backend/arm/test/test_regalloc_mov.py
--- a/rpython/jit/backend/arm/test/test_regalloc_mov.py
+++ b/rpython/jit/backend/arm/test/test_regalloc_mov.py
@@ -436,7 +436,7 @@
         self.push(sf, e)
 
     def test_push_large_stackfloat(self):
-        sf = stack_float(100)
+        sf = stack_float(1000)
         e = [
             mi('gen_load_int', ip.value, sf.value, cond=AL),
             mi('ADD_rr', ip.value, fp.value, ip.value, cond=AL),
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -17,7 +17,7 @@
 def get_description(atypes, rtype):
     p = lltype.malloc(CIF_DESCRIPTION, len(atypes),
                       flavor='raw', immortal=True)
-    p.abi = 42
+    p.abi = 1     # default
     p.nargs = len(atypes)
     p.rtype = rtype
     p.atypes = lltype.malloc(FFI_TYPE_PP.TO, len(atypes),
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -9,7 +9,7 @@
 link_files = []
 include_dirs = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
-    libraries = ['libeay32', 'ssleay32',
+    libraries = ['libeay32', 'ssleay32', 'zlib1',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
     includes = [
         # ssl.h includes winsock.h, which will conflict with our own
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -103,11 +103,10 @@
     _set_errno(rffi.cast(INT, errno))
 
 if os.name == 'nt':
-    is_valid_fd = rffi.llexternal(
+    is_valid_fd = jit.dont_look_inside(rffi.llexternal(
         "_PyVerify_fd", [rffi.INT], rffi.INT,
         compilation_info=errno_eci,
-        )
-    @jit.dont_look_inside
+        ))
     def validate_fd(fd):
         if not is_valid_fd(fd):
             raise OSError(get_errno(), 'Bad file descriptor')
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -7,7 +7,6 @@
 from struct import unpack
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 from rpython.rlib.rstruct.error import StructError
-from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
 
 class MasterReader(object):
     def __init__(self, s):
@@ -30,9 +29,9 @@
 
 def reader_for_pos(pos):
     class ReaderForPos(AbstractReader):
-        def __init__(self, mr):
+        def __init__(self, mr, bigendian):
             self.mr = mr
-            self.bigendian = native_is_bigendian
+            self.bigendian = bigendian
 
         def read(self, count):
             return self.mr.read(count)
@@ -64,6 +63,7 @@
         perform_lst = []
         miniglobals = {}
         miniglobals.update(globals())
+        miniglobals['bigendian'] = self.bigendian
         for i in rg:
             fmtdesc, rep, mask = self.formats[i]
             miniglobals['unpacker%d' % i] = fmtdesc.unpack
@@ -74,8 +74,8 @@
             else:
                 perform_lst.append('unpacker%d(reader%d, %d)' % (i, i, rep))
             miniglobals['reader_cls%d' % i] = reader_for_pos(i)
-        readers = ";".join(["reader%d = reader_cls%d(master_reader)" % (i, i)
-                             for i in rg])
+        readers = ";".join(["reader%d = reader_cls%d(master_reader, bigendian)"
+                            % (i, i) for i in rg])
         perform = ";".join(perform_lst)
         unpackers = ','.join(['reader%d.value' % i for i in rg])
         source = py.code.Source("""
diff --git a/rpython/rlib/rstruct/test/test_runpack.py b/rpython/rlib/rstruct/test/test_runpack.py
--- a/rpython/rlib/rstruct/test/test_runpack.py
+++ b/rpython/rlib/rstruct/test/test_runpack.py
@@ -26,6 +26,18 @@
         assert fn() == 123
         assert self.interpret(fn, []) == 123
 
+    def test_unpack_big_endian(self):
+        def fn():
+            return runpack(">i", "\x01\x02\x03\x04")
+        assert fn() == 0x01020304
+        assert self.interpret(fn, []) == 0x01020304
+
+    def test_unpack_double_big_endian(self):
+        def fn():
+            return runpack(">d", "testtest")
+        assert fn() == struct.unpack(">d", "testtest")[0]
+        assert self.interpret(fn, []) == struct.unpack(">d", "testtest")[0]
+
 class TestLLType(BaseTestRStruct, LLRtypeMixin):
     pass
 
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -10,7 +10,7 @@
 
 
 if compiler.name == "msvc":
-    libname = 'zlib'
+    libname = 'zlib1' # since version 1.1.4 and later, see http://www.zlib.net/DLL_FAQ.txt
 else:
     libname = 'z'
 eci = ExternalCompilationInfo(
diff --git a/rpython/rtyper/tool/test/test_rffi_platform.py b/rpython/rtyper/tool/test/test_rffi_platform.py
--- a/rpython/rtyper/tool/test/test_rffi_platform.py
+++ b/rpython/rtyper/tool/test/test_rffi_platform.py
@@ -288,9 +288,6 @@
     assert a % struct.calcsize("P") == 0
 
 def test_external_lib():
-    # XXX this one seems to be a bit too platform-specific. Check
-    #     how to test it on windows correctly (using so_prefix?)
-    #     and what are alternatives to LD_LIBRARY_PATH
     eci = ExternalCompilationInfo()
     c_source = """
     int f(int a, int b)
@@ -298,12 +295,17 @@
         return (a + b);
     }
     """
+    if platform.name == 'mscv':
+        c_source = '__declspec(dllexport) ' + c_source
+        libname = 'libc_lib'
+    else:
+        libname = 'c_lib'
     tmpdir = udir.join('external_lib').ensure(dir=1)
     c_file = tmpdir.join('libc_lib.c')
     c_file.write(c_source)
     l = platform.compile([c_file], eci, standalone=False)
     eci = ExternalCompilationInfo(
-        libraries = ['c_lib'],
+        libraries = [libname],
         library_dirs = [str(tmpdir)]
         )
     rffi_platform.verify_eci(eci)
diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py
--- a/rpython/translator/c/gcc/test/test_asmgcroot.py
+++ b/rpython/translator/c/gcc/test/test_asmgcroot.py
@@ -25,8 +25,8 @@
 
     @classmethod
     def make_config(cls):
-        if _MSVC and _WIN64:
-            py.test.skip("all asmgcroot tests disabled for MSVC X64")
+        if _MSVC:
+            py.test.skip("all asmgcroot tests disabled for MSVC")
         from rpython.config.translationoption import get_combined_translation_config
         config = get_combined_translation_config(translating=True)
         config.translation.gc = cls.gcpolicy
diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py
--- a/rpython/translator/c/test/test_genc.py
+++ b/rpython/translator/c/test/test_genc.py
@@ -37,7 +37,7 @@
     if isinstance(v, float):
         from rpython.rlib.rfloat import formatd, DTSF_ADD_DOT_0
         return formatd(v, 'r', 0, DTSF_ADD_DOT_0)
-    return v
+    return str(v)   # always return a string, to get consistent types
 
 def parse_longlong(a):
     p0, p1 = a.split(":")
@@ -205,6 +205,28 @@
     py.test.raises(Exception, f1, "world")  # check that it's really typed
 
 
+def test_int_becomes_float():
+    # used to crash "very often": the long chain of mangle() calls end
+    # up converting the return value of f() from an int to a float, but
+    # if blocks are followed in random order by the annotator, it will
+    # very likely first follow the call to llrepr_out() done after the
+    # call to f(), getting an int first (and a float only later).
+    @specialize.arg(1)
+    def mangle(x, chain):
+        if chain:
+            return mangle(x, chain[1:])
+        return x - 0.5
+    def f(x):
+        if x > 10:
+            x = mangle(x, (1,1,1,1,1,1,1,1,1,1))
+        return x + 1
+
+    f1 = compile(f, [int])
+
+    assert f1(5) == 6
+    assert f1(12) == 12.5
+
+
 def test_string_arg():
     def f(s):
         total = 0
diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -8,8 +8,9 @@
     DEFAULT_CC = "cc"
     name = "openbsd"
 
-    link_flags = os.environ.get("LDFLAGS", '-pthread').split()
-    cflags = os.environ.get("CFLAGS", "-O3 -pthread -fomit-frame-pointer -D_BSD_SOURCE").split()
+    link_flags = os.environ.get("LDFLAGS", "").split() + ['-pthread']
+    cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-D_BSD_SOURCE'
+             ] + os.environ.get("CFLAGS", "").split()
 
     def _libs(self, libraries):
         libraries=set(libraries + ("intl", "iconv", "compat"))
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -28,6 +28,8 @@
     return _get_compiler_type(cc, False)
 
 def Windows_x64(cc=None):
+    raise Exception("Win64 is not supported.  You must either build for Win32"
+                    " or contribute the missing support in PyPy.")
     return _get_compiler_type(cc, True)
     
 def _get_msvc_env(vsver, x64flag):

From noreply at buildbot.pypy.org  Tue Jun  4 10:37:55 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Jun 2013 10:37:55 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Mention a trade-off to explore
Message-ID: <20130604083755.BA70A1C1527@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r63:20644865dbf8
Date: 2013-06-04 10:37 +0200
http://bitbucket.org/pypy/stmgc/changeset/20644865dbf8/

Log:	Mention a trade-off to explore

diff --git a/c3/doc-objects.txt b/c3/doc-objects.txt
--- a/c3/doc-objects.txt
+++ b/c3/doc-objects.txt
@@ -126,6 +126,12 @@
     add P to 'read_barrier_cache' and return
 
 
+It is unclear if it's better to stick all private objects into the
+read_barrier_cache, or to extend the read barrier's inline code to also
+check if h_revision == PRN (trade-off of polluting the cache with
+private objects which might be a majority, vs. making the inline check
+larger).
+
 Handles are stored for example in a global list, and the actual handle
 encodes an index in the list.  Every entry in the list is a pointer to a
 prot/priv object --- excepted once every N positions, where it is a

From noreply at buildbot.pypy.org  Tue Jun  4 11:19:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Jun 2013 11:19:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged in
 kostialopuhin/pypy/ctypes-byref (pull request #152)
Message-ID: <20130604091923.E8ADF1C12FE@cobra.cs.uni-duesseldorf.de>

Author: arigo <armin.rigo at gmail.com>
Branch: 
Changeset: r64748:aae5e8afa929
Date: 2013-06-04 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/aae5e8afa929/

Log:	Merged in kostialopuhin/pypy/ctypes-byref (pull request #152)

	Fix for getting ctypes.byref contents

diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
@@ -108,6 +108,13 @@
 
             py.test.raises(TypeError, delitem, p, 0)
 
+    def test_byref(self):
+        for ct, pt in zip(ctype_types, python_types):
+            i = ct(42)
+            p = byref(i)
+            assert type(p._obj) is ct
+            assert p._obj.value == 42
+
     def test_pointer_to_pointer(self):
         x = c_int(32)
         y = c_int(42)

From noreply at buildbot.pypy.org  Tue Jun  4 11:19:22 2013
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Tue,  4 Jun 2013 11:19:22 +0200 (CEST)
Subject: [pypy-commit] pypy ctypes-byref: test and fix for getting
	ctypes.byref contents
Message-ID: <20130604091922.9B7071C0651@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin <kostia.lopuhin at gmail.com>
Branch: ctypes-byref
Changeset: r64747:3404eb318128
Date: 2013-05-20 00:23 +0400
http://bitbucket.org/pypy/pypy/changeset/3404eb318128/

Log:	test and fix for getting ctypes.byref contents

diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
@@ -108,6 +108,13 @@
 
             py.test.raises(TypeError, delitem, p, 0)
 
+    def test_byref(self):
+        for ct, pt in zip(ctype_types, python_types):
+            i = ct(42)
+            p = byref(i)
+            assert type(p._obj) is ct
+            assert p._obj.value == 42
+
     def test_pointer_to_pointer(self):
         x = c_int(32)
         y = c_int(42)

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:32 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Implement skipping asserts when -O
	option is set
Message-ID: <20130604095832.82A6C1C1527@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64750:5e679b2d667e
Date: 2013-04-28 04:23 -0500
http://bitbucket.org/pypy/pypy/changeset/5e679b2d667e/

Log:	Implement skipping asserts when -O option is set

diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jabs_op('JUMP_IF_NOT_DEBUG', 204)     # Target address
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,7 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "remove docstrings when importing modules (like CPython -OO)",
+               "skip assert statements and remove docstrings when importing modules",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -91,12 +91,13 @@
     space = option.make_objspace(config)
 
     if interactiveconfig.optimize:
-        flags = space.sys.get('flags').getitems_copy()
+        flags_w = space.sys.get('flags').getitems_copy()
         #change optimize flag's value
-        flags[6] = space.wrap(2)
-        flags = type(space.sys.get('flags'))(flags)
-        flags.user_setup(space, space.sys.get('flags').w__class__)
-        space.sys.w_dict.setitem(space.wrap('flags'), flags)
+        import pdb; pdb.set_trace()
+        flags_w[6] = space.wrap(2)
+        w_flags = type(space.sys.get('flags'))(flags_w)
+        w_flags.user_setup(space, space.sys.get('flags').w__class__)
+        space.sys.w_dict.setitem(space.wrap('flags'), w_flags)
 
     space._starttime = starttime
     space.setitem(space.sys.w_dict, space.wrap('executable'),
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,10 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
 
 -OO
-    Remove docstrings when importing modules (like CPython -OO).
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -4,8 +4,8 @@
 """
 options:
   -i             inspect interactively after running script
-  -O             dummy optimization flag for compatibility with C Python
-  -OO            remove docstrings when importing modules (like CPython -OO)
+  -O             skip assert statements
+  -OO            remove docstrings when importing modules in addition to -O
   -c cmd         program passed in as CMD (terminates option list)
   -S             do not 'import site' on initialization
   -u             unbuffered binary stdout and stderr
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -634,6 +634,7 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+    ops.JUMP_IF_NOT_DEBUG : 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -373,6 +373,7 @@
     def visit_Assert(self, asrt):
         self.update_position(asrt.lineno)
         end = self.new_block()
+        self.emit_jump(ops.JUMP_IF_NOT_DEBUG, end, True)
         asrt.test.accept_jump_if(self, True, end)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "AssertionError")
         if asrt.msg:
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -844,6 +844,23 @@
         yield self.check, dict_w, "C4.__doc__", 'docstring'
         yield self.check, dict_w, "__doc__", None
 
+    def test_assert_skipping(self):
+        space = self.space
+        source = """if 1:
+        assert False
+        """
+        w_saved_flags = space.sys.get('flags')
+        flags_w = space.sys.get('flags').getitems_copy()
+        flags_w[6] = space.wrap(1)
+        w_flags = type(space.sys.get('flags'))(flags_w)
+        w_flags.user_setup(space, w_saved_flags.w__class__)
+        space.sys.w_dict.setitem(space.wrap('flags'), w_flags)
+        try:
+            self.run(source)
+        finally:
+            space.sys.w_dict.setitem(space.wrap('flags'), w_saved_flags)
+
+
 class AppTestCompiler:
 
     def test_docstring_not_loaded(self):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -876,6 +876,15 @@
         self.popvalue()
         return next_instr
 
+    def JUMP_IF_NOT_DEBUG(self, target, next_instr):
+        try:
+            optimize = self.space.sys.get_flag('optimize')
+        except:
+            optimize = 0
+        if optimize >= 1:
+            return target
+        return next_instr
+
     def GET_ITER(self, oparg, next_instr):
         w_iterable = self.popvalue()
         w_iterator = self.space.iter(w_iterable)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -800,6 +800,9 @@
         self.popvalue()
         return next_instr
 
+    def JUMP_IF_NOT_DEBUG(self, target, next_instr):
+        return next_instr
+
     def GET_ITER(self, oparg, next_instr):
         w_iterable = self.popvalue()
         w_iterator = self.space.iter(w_iterable)

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:31 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add support for -OO flag
Message-ID: <20130604095831.0F9FB1C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64749:711d2a861680
Date: 2013-04-26 17:31 -0500
http://bitbucket.org/pypy/pypy/changeset/711d2a861680/

Log:	Add support for -OO flag

diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,7 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "remove docstrings when importing modules (like CPython -OO)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -90,6 +90,14 @@
 
     space = option.make_objspace(config)
 
+    if interactiveconfig.optimize:
+        flags = space.sys.get('flags').getitems_copy()
+        #change optimize flag's value
+        flags[6] = space.wrap(2)
+        flags = type(space.sys.get('flags'))(flags)
+        flags.user_setup(space, space.sys.get('flags').w__class__)
+        space.sys.w_dict.setitem(space.wrap('flags'), flags)
+
     space._starttime = starttime
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -18,6 +18,9 @@
 -O
     Dummy optimization flag for compatibility with C Python.
 
+-OO
+    Remove docstrings when importing modules (like CPython -OO).
+
 -c *cmd*
     Program passed in as CMD (terminates option list).
 
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -5,6 +5,7 @@
 options:
   -i             inspect interactively after running script
   -O             dummy optimization flag for compatibility with C Python
+  -OO            remove docstrings when importing modules (like CPython -OO)
   -c cmd         program passed in as CMD (terminates option list)
   -S             do not 'import site' on initialization
   -u             unbuffered binary stdout and stderr
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -245,6 +245,8 @@
         if w_len is None:
             w_len = space.len(self.w_consts)
             space.setitem(self.w_consts, w_key, w_len)
+        if space.int_w(w_len) == 0:
+            self.scope.doc_removable = False
         return space.int_w(w_len)
 
     def _make_key(self, obj):
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -255,6 +255,7 @@
                 start = 1
                 doc_expr.walkabout(self)
                 self.name_op("__doc__", ast.Store)
+                self.scope.doc_removable = True
             for i in range(start, len(body)):
                 body[i].walkabout(self)
             return True
@@ -1189,7 +1190,10 @@
         tree.walkabout(self)
 
     def _get_code_flags(self):
-        return 0
+        flags = 0
+        if self.scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
+        return flags
 
 
 class AbstractFunctionCodeGenerator(PythonCodeGenerator):
@@ -1216,6 +1220,8 @@
             flags |= consts.CO_VARARGS
         if scope.has_keywords_arg:
             flags |= consts.CO_VARKEYWORDS
+        if scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
         if not self.cell_vars and not self.free_vars:
             flags |= consts.CO_NOFREE
         return PythonCodeGenerator._get_code_flags(self) | flags
@@ -1232,6 +1238,7 @@
             doc_expr = None
         if doc_expr is not None:
             self.add_const(doc_expr.s)
+            self.scope.doc_removable = True
             start = 1
         else:
             self.add_const(self.space.w_None)
@@ -1294,3 +1301,9 @@
         self._handle_body(cls.body)
         self.emit_op(ops.LOAD_LOCALS)
         self.emit_op(ops.RETURN_VALUE)
+
+    def _get_code_flags(self):
+        flags = 0
+        if self.scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
+        return PythonCodeGenerator._get_code_flags(self) | flags
diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -15,6 +15,8 @@
 CO_FUTURE_WITH_STATEMENT = 0x8000
 CO_FUTURE_PRINT_FUNCTION = 0x10000
 CO_FUTURE_UNICODE_LITERALS = 0x20000
+#pypy specific:
+CO_KILL_DOCSTRING = 0x100000
 
 PyCF_SOURCE_IS_UTF8 = 0x0100
 PyCF_DONT_IMPLY_DEDENT = 0x0200
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -42,6 +42,7 @@
         self.has_free = False
         self.child_has_free = False
         self.nested = False
+        self.doc_removable = False
 
     def lookup(self, name):
         """Find the scope of identifier 'name'."""
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -812,6 +812,37 @@
         """
         self.simple_test(source, 'ok', 1)
 
+    def test_remove_docstring(self):
+        source = '"module_docstring"\n' + """if 1:
+        def f1():
+            'docstring'
+        def f2():
+            'docstring'
+            return 'docstring'
+        class C1():
+            'docstring'
+        class C2():
+            __doc__ = 'docstring'
+        class C3():
+            field = 'not docstring'
+        class C4():
+            'docstring'
+            field = 'docstring'
+        """
+        code_w = compile_with_astcompiler(source, 'exec', self.space)
+        code_w.remove_docstrings(self.space)
+        dict_w = self.space.newdict();
+        code_w.exec_code(self.space, dict_w, dict_w)
+
+        yield self.check, dict_w, "f1.__doc__", None
+        yield self.check, dict_w, "f2.__doc__", 'docstring'
+        yield self.check, dict_w, "C1.__doc__", None
+        yield self.check, dict_w, "C2.__doc__", 'docstring'
+        yield self.check, dict_w, "C3.field", 'not docstring'
+        yield self.check, dict_w, "C4.field", 'docstring'
+        yield self.check, dict_w, "C4.__doc__", 'docstring'
+        yield self.check, dict_w, "C4.__doc__", 'docstring'
+        yield self.check, dict_w, "__doc__", None
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -12,7 +12,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
-    CO_GENERATOR)
+    CO_GENERATOR, CO_KILL_DOCSTRING)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import compute_hash
@@ -218,6 +218,13 @@
                 return w_first
         return space.w_None
 
+    def remove_docstrings(self, space):
+        if self.co_flags & CO_KILL_DOCSTRING:
+            self.co_consts_w[0] = space.w_None
+        for co_w in self.co_consts_w:
+            if isinstance(co_w, PyCode):
+                co_w.remove_docstrings(space)
+
     def _to_code(self):
         """For debugging only."""
         consts = [None] * len(self.co_consts_w)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -913,6 +913,13 @@
             if not space.is_true(space.sys.get('dont_write_bytecode')):
                 write_compiled_module(space, code_w, cpathname, mode, mtime)
 
+    try:
+        optimize = space.sys.get_flag('optimize')
+    except:
+        optimize = 0
+    if optimize >= 2:
+        code_w.remove_docstrings(space)
+
     update_code_filenames(space, code_w, pathname)
     exec_code_module(space, w_mod, code_w)
 
@@ -1007,6 +1014,13 @@
                               "Bad magic number in %s", cpathname)
     #print "loading pyc file:", cpathname
     code_w = read_compiled_module(space, cpathname, source)
+    try:
+        optimize = space.sys.get_flag('optimize')
+    except:
+        optimize = 0
+    if optimize >= 2:
+        code_w.remove_docstrings(space)
+
     exec_code_module(space, w_mod, code_w)
 
     return w_mod

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:33 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make changing sys.flag.optimize in
 pyinteractive.py cleaner
Message-ID: <20130604095833.CB7351C1528@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64751:bcac3e99fa6f
Date: 2013-04-28 04:55 -0500
http://bitbucket.org/pypy/pypy/changeset/bcac3e99fa6f/

Log:	Make changing sys.flag.optimize in pyinteractive.py cleaner

diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -90,19 +90,19 @@
 
     space = option.make_objspace(config)
 
-    if interactiveconfig.optimize:
-        flags_w = space.sys.get('flags').getitems_copy()
-        #change optimize flag's value
-        import pdb; pdb.set_trace()
-        flags_w[6] = space.wrap(2)
-        w_flags = type(space.sys.get('flags'))(flags_w)
-        w_flags.user_setup(space, space.sys.get('flags').w__class__)
-        space.sys.w_dict.setitem(space.wrap('flags'), w_flags)
-
     space._starttime = starttime
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        flags_w = space.sys.get('flags').getitems_copy()
+        #change the optimize flag's value
+        flags_w[6] = space.wrap(2)
+        space.appexec([space.wrap(flags_w)], """(flags):
+            import sys
+            sys.flags = type(sys.flags)(flags)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -852,9 +852,11 @@
         w_saved_flags = space.sys.get('flags')
         flags_w = space.sys.get('flags').getitems_copy()
         flags_w[6] = space.wrap(1)
-        w_flags = type(space.sys.get('flags'))(flags_w)
-        w_flags.user_setup(space, w_saved_flags.w__class__)
-        space.sys.w_dict.setitem(space.wrap('flags'), w_flags)
+        space.appexec([space.wrap(flags_w)], """(flags):
+            import sys
+            sys.flags = type(sys.flags)(flags)
+        """)
+
         try:
             self.run(source)
         finally:

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:35 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move sys.flags changing to applevel code
Message-ID: <20130604095835.1E2601C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64752:6deea9fd59b6
Date: 2013-04-28 15:21 -0500
http://bitbucket.org/pypy/pypy/changeset/6deea9fd59b6/

Log:	Move sys.flags changing to applevel code

diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -95,11 +95,11 @@
                   space.wrap(argv[0]))
 
     if interactiveconfig.optimize:
-        flags_w = space.sys.get('flags').getitems_copy()
         #change the optimize flag's value
-        flags_w[6] = space.wrap(2)
-        space.appexec([space.wrap(flags_w)], """(flags):
+        space.appexec([], """():
             import sys
+            flags = list(sys.flags)
+            flags[6] = 2
             sys.flags = type(sys.flags)(flags)
         """)
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -850,10 +850,10 @@
         assert False
         """
         w_saved_flags = space.sys.get('flags')
-        flags_w = space.sys.get('flags').getitems_copy()
-        flags_w[6] = space.wrap(1)
-        space.appexec([space.wrap(flags_w)], """(flags):
+        space.appexec([], """():
             import sys
+            flags = list(sys.flags)
+            flags[6] = 2
             sys.flags = type(sys.flags)(flags)
         """)
 

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:37 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make -O set __debug__ to False and
 change JUMP_IF_NOT_DEBUG to a relative jump
Message-ID: <20130604095837.016B11C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64753:a000ef4ae952
Date: 2013-04-29 00:57 -0500
http://bitbucket.org/pypy/pypy/changeset/a000ef4ae952/

Log:	Make -O set __debug__ to False and change JUMP_IF_NOT_DEBUG to a
	relative jump

diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,6 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
-jabs_op('JUMP_IF_NOT_DEBUG', 204)     # Target address
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Target address
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -95,12 +95,14 @@
                   space.wrap(argv[0]))
 
     if interactiveconfig.optimize:
-        #change the optimize flag's value
+        #change the optimize flag's value and set __debug__ to False
         space.appexec([], """():
             import sys
             flags = list(sys.flags)
             flags[6] = 2
             sys.flags = type(sys.flags)(flags)
+            import __builtin__
+            setattr(__builtin__, '__debug__', False)
         """)
 
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -450,6 +450,10 @@
         sys.py3kwarning = bool(sys.flags.py3k_warning)
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
+        if sys.flags.optimize >= 1:
+            import __builtin__
+            setattr(__builtin__, '__debug__', False)
+
         if sys.py3kwarning:
             print >> sys.stderr, (
                 "Warning: pypy does not implement py3k warnings")

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:38 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add caching for the debug flag for
	skipping asserts
Message-ID: <20130604095838.2CB8A1C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64754:c8d597cf7e85
Date: 2013-04-29 04:29 -0500
http://bitbucket.org/pypy/pypy/changeset/c8d597cf7e85/

Log:	Add caching for the debug flag for skipping asserts

diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -101,8 +101,8 @@
             flags = list(sys.flags)
             flags[6] = 2
             sys.flags = type(sys.flags)(flags)
-            import __builtin__
-            setattr(__builtin__, '__debug__', False)
+            import __pypy__
+            __pypy__.set_debug(False)
         """)
 
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -451,8 +451,8 @@
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
         if sys.flags.optimize >= 1:
-            import __builtin__
-            setattr(__builtin__, '__debug__', False)
+            import __pypy__
+            __pypy__.set_debug(False)
 
         if sys.py3kwarning:
             print >> sys.stderr, (
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -846,21 +846,17 @@
 
     def test_assert_skipping(self):
         space = self.space
+        mod = space.getbuiltinmodule('__pypy__')
+        w_set_debug = space.getattr(mod, space.wrap('set_debug'))
+        space.call_function(w_set_debug, space.w_False)
+
         source = """if 1:
         assert False
         """
-        w_saved_flags = space.sys.get('flags')
-        space.appexec([], """():
-            import sys
-            flags = list(sys.flags)
-            flags[6] = 2
-            sys.flags = type(sys.flags)(flags)
-        """)
-
         try:
             self.run(source)
         finally:
-            space.sys.w_dict.setitem(space.wrap('flags'), w_saved_flags)
+            space.call_function(w_set_debug, space.w_True)
 
 
 class AppTestCompiler:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -877,11 +877,7 @@
         return next_instr
 
     def JUMP_IF_NOT_DEBUG(self, target, next_instr):
-        try:
-            optimize = self.space.sys.get_flag('optimize')
-        except:
-            optimize = 0
-        if optimize >= 1:
+        if not self.space.sys.debug:
             return target
         return next_instr
 
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -57,6 +57,7 @@
         'newlist_hint'              : 'interp_magic.newlist_hint',
         'newdict'                   : 'interp_dict.newdict',
         'dictstrategy'              : 'interp_dict.dictstrategy',
+        'set_debug'                 : 'interp_magic.set_debug',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -101,3 +101,11 @@
 @unwrap_spec(sizehint=int)
 def newlist_hint(space, sizehint):
     return space.newlist_hint(sizehint)
+
+ at unwrap_spec(debug=bool)
+def set_debug(space, debug):
+    print debug
+    space.sys.debug = debug
+    space.setitem(space.builtin.w_dict,
+                  space.wrap('__debug__'),
+                  space.wrap(debug))
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -7,7 +7,7 @@
 
 class Module(MixedModule):
     """Sys Builtin Module. """
-    _immutable_fields_ = ["defaultencoding?"]
+    _immutable_fields_ = ["defaultencoding?", "debug?"]
 
     def __init__(self, space, w_name):
         """NOT_RPYTHON""" # because parent __init__ isn't
@@ -18,6 +18,7 @@
         self.w_default_encoder = None
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
+        self.debug = True
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))', 

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:39 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Code cleanup (stray print,
 bad variable name, and bare except)
Message-ID: <20130604095839.748BC1C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64755:83e557521804
Date: 2013-04-29 12:20 -0500
http://bitbucket.org/pypy/pypy/changeset/83e557521804/

Log:	Code cleanup (stray print, bad variable name, and bare except)

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -221,9 +221,9 @@
     def remove_docstrings(self, space):
         if self.co_flags & CO_KILL_DOCSTRING:
             self.co_consts_w[0] = space.w_None
-        for co_w in self.co_consts_w:
+        for w_co in self.co_consts_w:
             if isinstance(co_w, PyCode):
-                co_w.remove_docstrings(space)
+                w_co.remove_docstrings(space)
 
     def _to_code(self):
         """For debugging only."""
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -104,7 +104,6 @@
 
 @unwrap_spec(debug=bool)
 def set_debug(space, debug):
-    print debug
     space.sys.debug = debug
     space.setitem(space.builtin.w_dict,
                   space.wrap('__debug__'),
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -915,7 +915,7 @@
 
     try:
         optimize = space.sys.get_flag('optimize')
-    except:
+    except Exception:
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)
@@ -1016,7 +1016,7 @@
     code_w = read_compiled_module(space, cpathname, source)
     try:
         optimize = space.sys.get_flag('optimize')
-    except:
+    except Exception:
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:40 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Code cleanup (stray print,
 bad variable name, and bare except)
Message-ID: <20130604095840.B65341C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64756:773b31fdbf56
Date: 2013-04-29 12:39 -0500
http://bitbucket.org/pypy/pypy/changeset/773b31fdbf56/

Log:	Code cleanup (stray print, bad variable name, and bare except)

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -222,7 +222,7 @@
         if self.co_flags & CO_KILL_DOCSTRING:
             self.co_consts_w[0] = space.w_None
         for w_co in self.co_consts_w:
-            if isinstance(co_w, PyCode):
+            if isinstance(w_co, PyCode):
                 w_co.remove_docstrings(space)
 
     def _to_code(self):

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:41 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add to the remove docstring tests and
	fix a comment
Message-ID: <20130604095841.E84C51C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64757:235a08c4b2de
Date: 2013-05-09 21:50 -0500
http://bitbucket.org/pypy/pypy/changeset/235a08c4b2de/

Log:	Add to the remove docstring tests and fix a comment

diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,6 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
-jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Target address
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Distance to target address
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -819,6 +819,9 @@
         def f2():
             'docstring'
             return 'docstring'
+        def f3():
+            'foo'
+            return 'bar'
         class C1():
             'docstring'
         class C2():
@@ -836,6 +839,9 @@
 
         yield self.check, dict_w, "f1.__doc__", None
         yield self.check, dict_w, "f2.__doc__", 'docstring'
+        yield self.check, dict_w, "f2()", 'docstring'
+        yield self.check, dict_w, "f3.__doc__", None
+        yield self.check, dict_w, "f3()", 'bar'
         yield self.check, dict_w, "C1.__doc__", None
         yield self.check, dict_w, "C2.__doc__", 'docstring'
         yield self.check, dict_w, "C3.field", 'not docstring'

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:45 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge pypy/pypy default
Message-ID: <20130604095845.EFBF11C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64758:3dda6aa42f0c
Date: 2013-05-10 17:23 +0400
http://bitbucket.org/pypy/pypy/changeset/3dda6aa42f0c/

Log:	Merge pypy/pypy default

diff too long, truncating to 2000 out of 6102 lines

diff --git a/lib-python/2.7/distutils/command/install.py b/lib-python/2.7/distutils/command/install.py
--- a/lib-python/2.7/distutils/command/install.py
+++ b/lib-python/2.7/distutils/command/install.py
@@ -474,8 +474,8 @@
 
     def select_scheme (self, name):
         # it's the caller's problem if they supply a bad name!
-        if hasattr(sys, 'pypy_version_info') and not (
-                name.endswith('_user') or name.endswith('_home')):
+        if (hasattr(sys, 'pypy_version_info') and
+            not name.endswith(('_user', '_home'))):
             name = 'pypy'
         scheme = INSTALL_SCHEMES[name]
         for key in SCHEME_KEYS:
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,10 +122,10 @@
         compiler.compiler_so.extend(['-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CFLAGS" in os.environ:
-            cflags = os.environ["CFLAGS"]
-            compiler.compiler.append(cflags)
-            compiler.compiler_so.append(cflags)
-            compiler.linker_so.append(cflags)
+            cflags = os.environ["CFLAGS"].split()
+            compiler.compiler.extend(cflags)
+            compiler.compiler_so.extend(cflags)
+            compiler.linker_so.extend(cflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -1953,7 +1953,11 @@
                 if key is None:
                     callback(None, modname, '')
                 else:
-                    desc = split(__import__(modname).__doc__ or '', '\n')[0]
+                    try:
+                        module_doc = __import__(modname).__doc__
+                    except ImportError:
+                        module_doc = None
+                    desc = split(module_doc or '', '\n')[0]
                     if find(lower(modname + ' - ' + desc), key) >= 0:
                         callback(None, modname, desc)
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -167,7 +167,6 @@
             return
         if '_fields_' not in self.__dict__:
             self._fields_ = []
-            self._names = []
             _set_shape(self, [], self._is_union)
 
     __setattr__ = struct_setattr
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -370,7 +370,10 @@
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
             BType = ffi._get_cached_btype(tp)
-            value = backendlib.load_function(BType, name)
+            try:
+                value = backendlib.load_function(BType, name)
+            except KeyError:
+                raise AttributeError(name)
             library.__dict__[name] = value
             return
         #
diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py
--- a/lib_pypy/ctypes_config_cache/dumpcache.py
+++ b/lib_pypy/ctypes_config_cache/dumpcache.py
@@ -1,25 +1,21 @@
-import os
+import sys, os
 from ctypes_configure import dumpcache
-from rpython.jit.backend import detect_cpu
 
 def dumpcache2(basename, config):
-    model = detect_cpu.autodetect_main_model_and_size()
-    filename = '_%s_%s_.py' % (basename, model)
+    size = 32 if sys.maxint <= 2**32 else 64
+    filename = '_%s_%s_.py' % (basename, size)
     dumpcache.dumpcache(__file__, filename, config)
     #
     filename = os.path.join(os.path.dirname(__file__),
                             '_%s_cache.py' % (basename,))
     g = open(filename, 'w')
     print >> g, '''\
-try:
-    from __pypy__ import cpumodel
-except ImportError:
-    from rpython.jit.backend import detect_cpu
-    cpumodel = detect_cpu.autodetect_main_model_and_size()
+import sys
+_size = 32 if sys.maxint <= 2**32 else 64
 # XXX relative import, should be removed together with
 # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib
-mod = __import__("_%s_%%s_" %% (cpumodel,),
-                 globals(), locals(), ["*"])
-globals().update(mod.__dict__)\
+_mod = __import__("_%s_%%s_" %% (_size,),
+                  globals(), locals(), ["*"])
+globals().update(_mod.__dict__)\
 ''' % (basename,)
     g.close()
diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py
--- a/lib_pypy/ctypes_config_cache/rebuild.py
+++ b/lib_pypy/ctypes_config_cache/rebuild.py
@@ -25,13 +25,12 @@
         sys.path[:] = path
 
 def try_rebuild():
-    from rpython.jit.backend import detect_cpu
-    model = detect_cpu.autodetect_main_model_and_size()
-    # remove the files '_*_model_.py'
+    size = 32 if sys.maxint <= 2**32 else 64
+    # remove the files '_*_size_.py'
     left = {}
     for p in os.listdir(_dirpath):
-        if p.startswith('_') and (p.endswith('_%s_.py' % model) or
-                                  p.endswith('_%s_.pyc' % model)):
+        if p.startswith('_') and (p.endswith('_%s_.py' % size) or
+                                  p.endswith('_%s_.pyc' % size)):
             os.unlink(os.path.join(_dirpath, p))
         elif p.startswith('_') and (p.endswith('_.py') or
                                     p.endswith('_.pyc')):
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -655,7 +655,8 @@
     mkdtemp = classmethod(mkdtemp)
 
     def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3,
-                          lock_timeout = 172800):   # two days
+                          lock_timeout = 172800,   # two days
+                          min_timeout = 300):      # five minutes
         """ return unique directory with a number greater than the current
             maximum one.  The number is assumed to start directly after prefix.
             if keep is true directories with a number less than (maxnum-keep)
@@ -723,6 +724,20 @@
             for path in rootdir.listdir():
                 num = parse_num(path)
                 if num is not None and num <= (maxnum - keep):
+                    if min_timeout:
+                        # NB: doing this is needed to prevent (or reduce
+                        # a lot the chance of) the following situation:
+                        # 'keep+1' processes call make_numbered_dir() at
+                        # the same time, they create dirs, but then the
+                        # last process notices the first dir doesn't have
+                        # (yet) a .lock in it and kills it.
+                        try:
+                            t1 = path.lstat().mtime
+                            t2 = lockfile.lstat().mtime
+                            if abs(t2-t1) < min_timeout:
+                                continue   # skip directories too recent
+                        except py.error.Error:
+                            continue   # failure to get a time, better skip
                     lf = path.join('.lock')
                     try:
                         t1 = lf.lstat().mtime
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -84,13 +84,14 @@
     '_multiprocessing': [('objspace.usemodules.rctime', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
+    'cppyy': [('objspace.usemodules.cpyext', True)],
     }
 module_suggests = {
     # the reason you want _rawffi is for ctypes, which
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext"),
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
                ("translation.shared", sys.platform == "win32")],
 }
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0-beta1'
+release = '2.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -46,7 +46,7 @@
 2. Install build-time dependencies.  On a Debian box these are::
 
      [user at debian-box ~]$ sudo apt-get install \
-     gcc make python-dev libffi-dev lib-sqlite3-dev pkg-config \
+     gcc make python-dev libffi-dev libsqlite3-dev pkg-config \
      libz-dev libbz2-dev libncurses-dev libexpat1-dev \
      libssl-dev libgc-dev python-sphinx python-greenlet
 
@@ -105,7 +105,7 @@
 
     $ ./pypy-c
     Python 2.7.3 (7e4f0faa3d51, Nov 22 2012, 10:35:18)
-    [PyPy 2.0.0-beta1 with GCC 4.7.1] on linux2
+    [PyPy 2.0.0 with GCC 4.7.1] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     And now for something completely different: ``RPython magically makes you rich
     and famous (says so on the tin)''
@@ -235,7 +235,7 @@
 the ``bin/pypy`` executable.
 
 To install PyPy system wide on unix-like systems, it is recommended to put the
-whole hierarchy alone (e.g. in ``/opt/pypy2.0-beta1``) and put a symlink to the
+whole hierarchy alone (e.g. in ``/opt/pypy2.0``) and put a symlink to the
 ``pypy`` executable into ``/usr/bin`` or ``/usr/local/bin``
 
 If the executable fails to find suitable libraries, it will report
diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst
--- a/pypy/doc/getting-started.rst
+++ b/pypy/doc/getting-started.rst
@@ -53,10 +53,10 @@
 PyPy is ready to be executed as soon as you unpack the tarball or the zip
 file, with no need to install it in any specific location::
 
-    $ tar xf pypy-2.0-beta1-linux.tar.bz2
-    $ ./pypy-2.0-beta1/bin/pypy
+    $ tar xf pypy-2.0.tar.bz2
+    $ ./pypy-2.0/bin/pypy
     Python 2.7.3 (7e4f0faa3d51, Nov 22 2012, 10:35:18)
-    [PyPy 2.0.0-beta1 with GCC 4.7.1] on linux2
+    [PyPy 2.0.0 with GCC 4.7.1] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     And now for something completely different: ``PyPy is an exciting technology
     that lets you to write fast, portable, multi-platform interpreters with less
@@ -75,14 +75,14 @@
 
     $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
 
-    $ ./pypy-2.0-beta1/bin/pypy distribute_setup.py
+    $ ./pypy-2.0/bin/pypy distribute_setup.py
 
-    $ ./pypy-2.0-beta1/bin/pypy get-pip.py
+    $ ./pypy-2.0/bin/pypy get-pip.py
 
-    $ ./pypy-2.0-beta1/bin/pip install pygments  # for example
+    $ ./pypy-2.0/bin/pip install pygments  # for example
 
-3rd party libraries will be installed in ``pypy-2.0-beta1/site-packages``, and
-the scripts in ``pypy-2.0-beta1/bin``.
+3rd party libraries will be installed in ``pypy-2.0/site-packages``, and
+the scripts in ``pypy-2.0/bin``.
 
 Installing using virtualenv
 ---------------------------
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -40,7 +40,7 @@
 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.0 beta 2`_: the latest official release
+* `Release 2.0`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
@@ -110,7 +110,7 @@
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.0 beta 2`: http://pypy.org/download.html
+.. _`Release 2.0`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
diff --git a/pypy/doc/release-2.0.0.rst b/pypy/doc/release-2.0.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.0.rst
@@ -0,0 +1,71 @@
+============================
+PyPy 2.0 - Einstein Sandwich
+============================
+
+We're pleased to announce PyPy 2.0. This is a stable release that brings
+a swath of bugfixes, small performance improvements and compatibility fixes.
+PyPy 2.0 is a big step for us and we hope in the future we'll be able to
+provide stable releases more often.
+
+You can download the PyPy 2.0 release here:
+
+    http://pypy.org/download.html
+
+The two biggest changes since PyPy 1.9 are:
+
+* stackless is now supported including greenlets, which means eventlet
+  and gevent should work (but read below about gevent)
+
+* PyPy now contains release 0.6 of `cffi`_ as a builtin module, which
+  is preferred way of calling C from Python that works well on PyPy
+
+.. _`cffi`: http://cffi.readthedocs.org
+
+If you're using PyPy for anything, it would help us immensely if you fill out
+the following survey: http://bit.ly/pypysurvey This is for the developers
+eyes and we will not make any information public without your agreement.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Windows 64 work is still stalling, we would welcome a volunteer
+to handle that. ARM support is on the way, as you can see from the recently
+released alpha for ARM.
+
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
+
+Highlights
+==========
+
+* Stackless including greenlets should work. For gevent, you need to check
+  out `pypycore`_ and use the `pypy-hacks`_ branch of gevent.
+
+* cffi is now a module included with PyPy.  (`cffi`_ also exists for
+  CPython; the two versions should be fully compatible.)  It is the
+  preferred way of calling C from Python that works on PyPy.
+
+* Callbacks from C are now JITted, which means XML parsing is much faster.
+
+* A lot of speed improvements in various language corners, most of them small,
+  but speeding up some particular corners a lot.
+
+* The JIT was refactored to emit machine code which manipulates a "frame"
+  that lives on the heap rather than on the stack.  This is what makes
+  Stackless work, and it could bring another future speed-up (not done yet).
+
+* A lot of stability issues fixed.
+
+* Refactoring much of the numpypy array classes, which resulted in removal of
+  lazy expression evaluation. On the other hand, we now have more complete
+  dtype support and support more array attributes.  
+
+.. _`pypycore`: https://github.com/gevent-on-pypy/pypycore/
+.. _`pypy-hacks`: https://github.com/schmir/gevent/tree/pypy-hacks
+
+Cheers,
+fijal, arigo and the PyPy team
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-2.0.rst
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,140 +1,9 @@
 ======================
-What's new in PyPy 2.0
+What's new in PyPy 2.1
 ======================
 
-.. this is a revision shortly after release-2.0-beta1
-.. startrev: 0e6161a009c6
+.. this is a revision shortly after release-2.0
+.. startrev: a13c07067613
 
-.. branch: split-rpython
-Split rpython and pypy into seperate directories
-
-.. branch: callback-jit
-Callbacks from C are now better JITted
-
-.. branch: fix-jit-logs
-
-.. branch: remove-globals-in-jit
-
-.. branch: length-hint
-Implement __lenght_hint__ according to PEP 424
-
-.. branch: numpypy-longdouble
-Long double support for numpypy
-
-.. branch: numpypy-disable-longdouble
-Since r_longdouble support is missing, disable all longdouble and derivative
-dtypes using ENABLED_LONG_DOUBLE = False
-
-.. branch: numpypy-real-as-view
-Convert real, imag from ufuncs to views. This involves the beginning of
-view() functionality
-
-.. branch: indexing-by-array
-Adds indexing by scalar, adds int conversion from scalar and single element array,
-fixes compress, indexing by an array with a smaller shape and the indexed object.
-
-.. branch: str-dtype-improvement
-Allow concatenation of str and numeric arrays
-
-.. branch: signatures
-Improved RPython typing
-
-.. branch: rpython-bytearray
-Rudimentary support for bytearray in RPython
-
-.. branch: refactor-call_release_gil
-Fix a bug which caused cffi to return the wrong result when calling a C
-function which calls a Python callback which forces the frames
-
-.. branch: virtual-raw-mallocs
-JIT optimizations which make cffi calls even faster, by removing the need to
-allocate a temporary buffer where to store the arguments.
-
-.. branch: improve-docs-2
-Improve documents and straighten out links
-
-.. branch: fast-newarray
-Inline the fast path of newarray in the assembler.
-Disabled on ARM until we fix issues.
-
-.. branch: reflex-support
-Allow dynamic loading of a (Reflex) backend that implements the C-API needed
-to provide reflection information
-
-.. branches we don't care about
-.. branch: autoreds
-.. branch: kill-faking
-.. branch: improved_ebnfparse_error
-.. branch: task-decorator
-.. branch: fix-e4fa0b2
-.. branch: win32-fixes
-.. branch: numpy-unify-methods
-.. branch: fix-version-tool
-.. branch: popen2-removal
-.. branch: pickle-dumps
-.. branch: scalar_get_set
-
-.. branch: release-2.0-beta1
-
-.. branch: remove-PYPY_NOT_MAIN_FILE
-
-.. branch: missing-jit-operations
-
-.. branch: fix-lookinside-iff-oopspec
-Fixed the interaction between two internal tools for controlling the JIT.
-
-.. branch: inline-virtualref-2
-Better optimized certain types of frame accesses in the JIT, particularly
-around exceptions that escape the function they were raised in.
-
-.. branch: missing-ndarray-attributes
-Some missing attributes from ndarrays
-
-.. branch: cleanup-tests
-Consolidated the lib_pypy/pypy_test and pypy/module/test_lib_pypy tests into
-one directory for reduced confusion and so they all run nightly.
-
-.. branch: unquote-faster
-.. branch: urlparse-unquote-faster
-
-.. branch: signal-and-thread
-Add "__pypy__.thread.signals_enabled", a context manager. Can be used in a
-non-main thread to enable the processing of signal handlers in that thread.
-
-.. branch: coding-guide-update-rlib-refs
-.. branch: rlib-doc-rpython-refs
-.. branch: clean-up-remaining-pypy-rlib-refs
-
-.. branch: enumerate-rstr
-Support enumerate() over rstr types.
-
-.. branch: cleanup-numpypy-namespace
-Cleanup _numpypy and numpypy namespaces to more closely resemble numpy.
-
-.. branch: kill-flowobjspace
-Random cleanups to hide FlowObjSpace from public view.
-
-.. branch: vendor-rename
-
-.. branch: jitframe-on-heap
-Moves optimized JIT frames from stack to heap. As a side effect it enables
-stackless to work well with the JIT on PyPy. Also removes a bunch of code from
-the GC which fixes cannot find gc roots.
-
-.. branch: pycon2013-doc-fixes
-Documentation fixes after going through the docs at PyCon 2013 sprint.
-
-.. branch: extregistry-refactor
-
-.. branch: remove-list-smm
-.. branch: bridge-logging
-.. branch: curses_cffi
-cffi implementation of _curses
-
-.. branch: sqlite-cffi
-cffi implementation of sqlite3
-
-.. branch: release-2.0-beta2
-.. branch: unbreak-freebsd
-
-.. branch: virtualref-virtualizable
+.. branch: numpy-pickle
+Pickling of numpy arrays and dtypes (including record dtypes)
diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py
--- a/pypy/goal/getnightly.py
+++ b/pypy/goal/getnightly.py
@@ -6,8 +6,14 @@
 
 if sys.platform.startswith('linux'):
     arch = 'linux'
+    cmd = 'wget "%s"'
+    tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'"
+if sys.platform.startswith('darwin'):
+    arch = 'osx'
+    cmd = 'curl -O "%s"'
+    tar = "tar -x -v --strip-components=2 -f %s '*/bin/pypy'"
 else:
-    print 'Cannot determine the platform, please update this scrip'
+    print 'Cannot determine the platform, please update this script'
     sys.exit(1)
 
 if sys.maxint == 2**63 - 1:
@@ -23,10 +29,9 @@
 tmp = py.path.local.mkdtemp()
 mydir = tmp.chdir()
 print 'Downloading pypy to', tmp
-if os.system('wget "%s"' % url) != 0:
+if os.system(cmd % url) != 0:
     sys.exit(1)
 
 print 'Extracting pypy binary'
 mydir.chdir()
-os.system("tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'" % tmp.join(filename))
-
+os.system(tar % tmp.join(filename))
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -22,20 +22,22 @@
 
 # __________  Entry point  __________
 
+
 def create_entry_point(space, w_dict):
-    w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
-    w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
-    w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish))
-    w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup))
-    withjit = space.config.objspace.usemodules.pypyjit
+    if w_dict is not None: # for tests
+        w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
+        w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
+        w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish))
+        w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup))
+        withjit = space.config.objspace.usemodules.pypyjit
 
     def entry_point(argv):
         if withjit:
             from rpython.jit.backend.hlinfo import highleveljitinfo
             highleveljitinfo.sys_executable = argv[0]
 
-        #debug("entry point starting") 
-        #for arg in argv: 
+        #debug("entry point starting")
+        #for arg in argv:
         #    debug(" argv -> " + arg)
         if len(argv) > 2 and argv[1] == '--heapsize':
             # Undocumented option, handled at interp-level.
@@ -71,7 +73,35 @@
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
-    return entry_point
+
+    # register the minimal equivalent of running a small piece of code. This
+    # should be used as sparsely as possible, just to register callbacks
+
+    from rpython.rlib.entrypoint import entrypoint
+    from rpython.rtyper.lltypesystem import rffi
+
+    @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
+    def pypy_execute_source(ll_source):
+        source = rffi.charp2str(ll_source)
+        return _pypy_execute_source(source)
+
+    w_globals = space.newdict()
+    space.setitem(w_globals, space.wrap('__builtins__'),
+                  space.builtin_modules['__builtin__'])
+
+    def _pypy_execute_source(source):
+        try:
+            compiler = space.createcompiler()
+            stmt = compiler.compile(source, 'c callback', 'exec', 0)
+            stmt.exec_code(space, w_globals, w_globals)
+        except OperationError, e:
+            debug("OperationError:")
+            debug(" operror-type: " + e.w_type.getname(space))
+            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            return 1
+        return 0
+
+    return entry_point, _pypy_execute_source # for tests
 
 def call_finish(space):
     space.finish()
@@ -219,7 +249,7 @@
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy, pypy_hooks
         return PyPyJitPolicy(pypy_hooks)
-    
+
     def get_entry_point(self, config):
         from pypy.tool.lib_pypy import import_from_lib_pypy
         rebuild = import_from_lib_pypy('ctypes_config_cache/rebuild')
@@ -232,7 +262,7 @@
         app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main')
         app.hidden_applevel = False
         w_dict = app.getwdict(space)
-        entry_point = create_entry_point(space, w_dict)
+        entry_point, _ = create_entry_point(space, w_dict)
 
         return entry_point, None, PyPyAnnotatorPolicy(single_space = space)
 
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -1,21 +1,41 @@
 #! /usr/bin/env python
 # App-level version of py.py.
 # See test/test_app_main.
+
+# Missing vs CPython: -d, -OO, -t, -v, -x, -3
+"""\
+Options and arguments (and corresponding environment variables):
+-B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
+-c cmd : program passed in as string (terminates option list)
+-E     : ignore PYTHON* environment variables (such as PYTHONPATH)
+-h     : print this help message and exit (also --help)
+-i     : inspect interactively after running script; forces a prompt even
+         if stdin does not appear to be a terminal; also PYTHONINSPECT=x
+-m mod : run library module as a script (terminates option list)
+-O     : skip assert statements
+-OO    : remove docstrings when importing modules in addition to -O
+-R     : ignored (see http://bugs.python.org/issue14621)
+-Q arg : division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew
+-s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
+-S     : don't imply 'import site' on initialization
+-u     : unbuffered binary stdout and stderr; also PYTHONUNBUFFERED=x
+-V     : print the Python version number and exit (also --version)
+-W arg : warning control; arg is action:message:category:module:lineno
+         also PYTHONWARNINGS=arg
+file   : program read from script file
+-      : program read from stdin (default; interactive mode if a tty)
+arg ...: arguments passed to program in sys.argv[1:]
+PyPy options and arguments:
+--info : print translation information about this PyPy executable
 """
-options:
-  -i             inspect interactively after running script
-  -O             skip assert statements
-  -OO            remove docstrings when importing modules in addition to -O
-  -c cmd         program passed in as CMD (terminates option list)
-  -S             do not 'import site' on initialization
-  -u             unbuffered binary stdout and stderr
-  -h, --help     show this help message and exit
-  -m mod         library module to be run as a script (terminates option list)
-  -W arg         warning control (arg is action:message:category:module:lineno)
-  -E             ignore environment variables (such as PYTHONPATH)
-  -R             ignored (see http://bugs.python.org/issue14621)
-  --version      print the PyPy version
-  --info         print translation information about this PyPy executable
+USAGE1 = __doc__
+# Missing vs CPython: PYTHONHOME, PYTHONCASEOK
+USAGE2 = """
+Other environment variables:
+PYTHONSTARTUP: file executed on interactive startup (no default)
+PYTHONPATH   : %r-separated list of directories prefixed to the
+               default module search path.  The result is sys.path.
+PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.
 """
 
 import sys
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -27,7 +27,7 @@
         new_inst = mod.get('generator_new')
         w        = space.wrap
         if self.frame:
-            w_frame = w(self.frame)
+            w_frame = self.frame._reduce_state(space)
         else:
             w_frame = space.w_None
 
@@ -36,7 +36,20 @@
             w(self.running),
             ]
 
-        return space.newtuple([new_inst, space.newtuple(tup)])
+        return space.newtuple([new_inst, space.newtuple([]),
+                               space.newtuple(tup)])
+
+    def descr__setstate__(self, space, w_args):
+        from rpython.rlib.objectmodel import instantiate
+        args_w = space.unpackiterable(w_args)
+        w_framestate, w_running = args_w
+        if space.is_w(w_framestate, space.w_None):
+            self.frame = None
+        else:
+            frame = instantiate(space.FrameClass)   # XXX fish
+            frame.descr__setstate__(space, w_framestate)
+            GeneratorIterator.__init__(self, frame)
+        self.running = self.space.is_true(w_running)
 
     def descr__iter__(self):
         """x.__iter__() <==> iter(x)"""
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -13,6 +13,7 @@
     # imported yet, and when it has been, it is mod.__dict__.items() just
     # after startup().
     w_initialdict = None
+    lazy = False
 
     def __init__(self, space, w_name):
         """ NOT_RPYTHON """
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -16,10 +16,9 @@
 from rpython.tool.stdlib_opcode import host_bytecode_spec
 
 # Define some opcodes used
-g = globals()
 for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY
 POP_BLOCK END_FINALLY'''.split():
-    g[op] = stdlib_opcode.opmap[op]
+    globals()[op] = stdlib_opcode.opmap[op]
 HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT
 
 class PyFrame(eval.Frame):
@@ -304,11 +303,17 @@
     @jit.dont_look_inside
     def descr__reduce__(self, space):
         from pypy.interpreter.mixedmodule import MixedModule
-        from pypy.module._pickle_support import maker # helper fns
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
         new_inst = mod.get('frame_new')
-        w        = space.wrap
+        w_tup_state = self._reduce_state(space)
+        nt = space.newtuple
+        return nt([new_inst, nt([]), w_tup_state])
+
+    @jit.dont_look_inside
+    def _reduce_state(self, space):
+        from pypy.module._pickle_support import maker # helper fns
+        w = space.wrap
         nt = space.newtuple
 
         cells = self._getcells()
@@ -359,8 +364,7 @@
             w(self.instr_prev_plus_one),
             w_cells,
             ]
-
-        return nt([new_inst, nt([]), nt(tup_state)])
+        return nt(tup_state)
 
     @jit.dont_look_inside
     def descr__setstate__(self, space, w_args):
diff --git a/pypy/interpreter/test2/mymodule.py b/pypy/interpreter/test/mymodule.py
rename from pypy/interpreter/test2/mymodule.py
rename to pypy/interpreter/test/mymodule.py
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_app_main.py
@@ -0,0 +1,966 @@
+"""
+Tests for the entry point of pypy-c, app_main.py.
+"""
+from __future__ import with_statement
+import py
+import sys, os, re, runpy, subprocess
+from rpython.tool.udir import udir
+from contextlib import contextmanager
+from pypy.conftest import pypydir
+
+banner = sys.version.splitlines()[0]
+
+app_main = os.path.join(os.path.realpath(os.path.dirname(__file__)), os.pardir, 'app_main.py')
+app_main = os.path.abspath(app_main)
+
+_counter = 0
+def _get_next_path(ext='.py'):
+    global _counter
+    p = udir.join('demo_test_app_main_%d%s' % (_counter, ext))
+    _counter += 1
+    return p
+
+def getscript(source):
+    p = _get_next_path()
+    p.write(str(py.code.Source(source)))
+    return str(p)
+
+def getscript_pyc(space, source):
+    p = _get_next_path()
+    p.write(str(py.code.Source(source)))
+    w_dir = space.wrap(str(p.dirpath()))
+    w_modname = space.wrap(p.purebasename)
+    space.appexec([w_dir, w_modname], """(dir, modname):
+        import sys
+        d = sys.modules.copy()
+        sys.path.insert(0, dir)
+        __import__(modname)
+        sys.path.pop(0)
+        for key in sys.modules.keys():
+            if key not in d:
+                del sys.modules[key]
+    """)
+    p = str(p) + 'c'
+    assert os.path.isfile(p)   # the .pyc file should have been created above
+    return p
+
+def getscript_in_dir(source):
+    pdir = _get_next_path(ext='')
+    p = pdir.ensure(dir=1).join('__main__.py')
+    p.write(str(py.code.Source(source)))
+    # return relative path for testing purposes 
+    return py.path.local().bestrelpath(pdir)
+
+demo_script = getscript("""
+    print 'hello'
+    print 'Name:', __name__
+    print 'File:', __file__
+    import sys
+    print 'Exec:', sys.executable
+    print 'Argv:', sys.argv
+    print 'goodbye'
+    myvalue = 6*7
+    """)
+
+crashing_demo_script = getscript("""
+    print 'Hello2'
+    myvalue2 = 11
+    ooups
+    myvalue2 = 22
+    print 'Goodbye2'   # should not be reached
+    """)
+
+
+class TestParseCommandLine:
+    def check_options(self, options, sys_argv, **expected):
+        assert sys.argv == sys_argv
+        for key, value in expected.items():
+            assert options[key] == value
+        for key, value in options.items():
+            if key not in expected:
+                assert not value, (
+                    "option %r has unexpectedly the value %r" % (key, value))
+
+    def check(self, argv, env, **expected):
+        import StringIO
+        from pypy.interpreter import app_main
+        saved_env = os.environ.copy()
+        saved_sys_argv = sys.argv[:]
+        saved_sys_stdout = sys.stdout
+        saved_sys_stderr = sys.stdout
+        app_main.os = os
+        try:
+            os.environ.update(env)
+            sys.stdout = sys.stderr = StringIO.StringIO()
+            try:
+                options = app_main.parse_command_line(argv)
+            except SystemExit:
+                output = expected['output_contains']
+                assert output in sys.stdout.getvalue()
+            else:
+                self.check_options(options, **expected)
+        finally:
+            os.environ.clear()
+            os.environ.update(saved_env)
+            sys.argv[:] = saved_sys_argv
+            sys.stdout = saved_sys_stdout
+            sys.stderr = saved_sys_stderr
+
+    def test_all_combinations_I_can_think_of(self):
+        self.check([], {}, sys_argv=[''], run_stdin=True)
+        self.check(['-'], {}, sys_argv=['-'], run_stdin=True)
+        self.check(['-S'], {}, sys_argv=[''], run_stdin=True, no_site=1)
+        self.check(['-OO'], {}, sys_argv=[''], run_stdin=True, optimize=2)
+        self.check(['-O', '-O'], {}, sys_argv=[''], run_stdin=True, optimize=2)
+        self.check(['-Qnew'], {}, sys_argv=[''], run_stdin=True, division_new=1)
+        self.check(['-Qold'], {}, sys_argv=[''], run_stdin=True, division_new=0)
+        self.check(['-Qwarn'], {}, sys_argv=[''], run_stdin=True, division_warning=1)
+        self.check(['-Qwarnall'], {}, sys_argv=[''], run_stdin=True,
+                   division_warning=2)
+        self.check(['-Q', 'new'], {}, sys_argv=[''], run_stdin=True, division_new=1)
+        self.check(['-SOQnew'], {}, sys_argv=[''], run_stdin=True,
+                   no_site=1, optimize=1, division_new=1)
+        self.check(['-SOQ', 'new'], {}, sys_argv=[''], run_stdin=True,
+                   no_site=1, optimize=1, division_new=1)
+        self.check(['-i'], {}, sys_argv=[''], run_stdin=True,
+                   interactive=1, inspect=1)
+        self.check(['-?'], {}, output_contains='usage:')
+        self.check(['-h'], {}, output_contains='usage:')
+        self.check(['-S', '-tO', '-h'], {}, output_contains='usage:')
+        self.check(['-S', '-thO'], {}, output_contains='usage:')
+        self.check(['-S', '-tO', '--help'], {}, output_contains='usage:')
+        self.check(['-S', '-tO', '--info'], {}, output_contains='translation')
+        self.check(['-S', '-tO', '--version'], {}, output_contains='Python')
+        self.check(['-S', '-tOV'], {}, output_contains='Python')
+        self.check(['--jit', 'foobar', '-S'], {}, sys_argv=[''],
+                   run_stdin=True, no_site=1)
+        self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass')
+        self.check(['-cpass'], {}, sys_argv=['-c'], run_command='pass')
+        self.check(['-cpass','x'], {}, sys_argv=['-c','x'], run_command='pass')
+        self.check(['-Sc', 'pass'], {}, sys_argv=['-c'], run_command='pass',
+                   no_site=1)
+        self.check(['-Scpass'], {}, sys_argv=['-c'], run_command='pass', no_site=1)
+        self.check(['-c', '', ''], {}, sys_argv=['-c', ''], run_command='')
+        self.check(['-mfoo', 'bar', 'baz'], {}, sys_argv=['foo', 'bar', 'baz'],
+                   run_module=True)
+        self.check(['-m', 'foo', 'bar', 'baz'], {}, sys_argv=['foo', 'bar', 'baz'],
+                   run_module=True)
+        self.check(['-Smfoo', 'bar', 'baz'], {}, sys_argv=['foo', 'bar', 'baz'],
+                   run_module=True, no_site=1)
+        self.check(['-Sm', 'foo', 'bar', 'baz'], {}, sys_argv=['foo', 'bar', 'baz'],
+                   run_module=True, no_site=1)
+        self.check(['-', 'foo', 'bar'], {}, sys_argv=['-', 'foo', 'bar'],
+                   run_stdin=True)
+        self.check(['foo', 'bar'], {}, sys_argv=['foo', 'bar'])
+        self.check(['foo', '-i'], {}, sys_argv=['foo', '-i'])
+        self.check(['-i', 'foo'], {}, sys_argv=['foo'], interactive=1, inspect=1)
+        self.check(['--', 'foo'], {}, sys_argv=['foo'])
+        self.check(['--', '-i', 'foo'], {}, sys_argv=['-i', 'foo'])
+        self.check(['--', '-', 'foo'], {}, sys_argv=['-', 'foo'], run_stdin=True)
+        self.check(['-Wbog'], {}, sys_argv=[''], warnoptions=['bog'], run_stdin=True)
+        self.check(['-W', 'ab', '-SWc'], {}, sys_argv=[''], warnoptions=['ab', 'c'],
+                   run_stdin=True, no_site=1)
+
+        self.check([], {'PYTHONDEBUG': '1'}, sys_argv=[''], run_stdin=True, debug=1)
+        self.check([], {'PYTHONDONTWRITEBYTECODE': '1'}, sys_argv=[''], run_stdin=True, dont_write_bytecode=1)
+        self.check([], {'PYTHONNOUSERSITE': '1'}, sys_argv=[''], run_stdin=True, no_user_site=1)
+        self.check([], {'PYTHONUNBUFFERED': '1'}, sys_argv=[''], run_stdin=True, unbuffered=1)
+        self.check([], {'PYTHONVERBOSE': '1'}, sys_argv=[''], run_stdin=True, verbose=1)
+
+    def test_sysflags(self):
+        flags = (
+            ("debug", "-d", "1"),
+            ("py3k_warning", "-3", "1"),
+            ("division_warning", "-Qwarn", "1"),
+            ("division_warning", "-Qwarnall", "2"),
+            ("division_new", "-Qnew", "1"),
+            (["inspect", "interactive"], "-i", "1"),
+            ("optimize", "-O", "1"),
+            ("optimize", "-OO", "2"),
+            ("dont_write_bytecode", "-B", "1"),
+            ("no_user_site", "-s", "1"),
+            ("no_site", "-S", "1"),
+            ("ignore_environment", "-E", "1"),
+            ("tabcheck", "-t", "1"),
+            ("tabcheck", "-tt", "2"),
+            ("verbose", "-v", "1"),
+            ("unicode", "-U", "1"),
+            ("bytes_warning", "-b", "1"),
+        )
+        for flag, opt, value in flags:
+            if isinstance(flag, list):   # this is for inspect&interactive
+                expected = {}
+                for flag1 in flag:
+                    expected[flag1] = int(value)
+            else:
+                expected = {flag: int(value)}
+            self.check([opt, '-c', 'pass'], {}, sys_argv=['-c'],
+                       run_command='pass', **expected)
+
+    def test_sysflags_envvar(self, monkeypatch):
+        monkeypatch.setenv('PYTHONNOUSERSITE', '1')
+        expected = {"no_user_site": True}
+        self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass', **expected)
+
+
+class TestInteraction:
+    """
+    These tests require pexpect (UNIX-only).
+    http://pexpect.sourceforge.net/
+    """
+    def _spawn(self, *args, **kwds):
+        try:
+            import pexpect
+        except ImportError, e:
+            py.test.skip(str(e))
+        else:
+            # Version is of the style "0.999" or "2.1".  Older versions of
+            # pexpect try to get the fileno of stdin, which generally won't
+            # work with py.test (due to sys.stdin being a DontReadFromInput
+            # instance).
+            version = map(int, pexpect.__version__.split('.'))
+
+            # I only tested 0.999 and 2.1.  The former does not work, the
+            # latter does.  Feel free to refine this measurement.
+            # -exarkun, 17/12/2007
+            if version < [2, 1]:
+                py.test.skip(
+                    "pexpect version too old, requires 2.1 or newer: %r" % (
+                        pexpect.__version__,))
+
+        kwds.setdefault('timeout', 10)
+        print 'SPAWN:', ' '.join([args[0]] + args[1]), kwds
+        child = pexpect.spawn(*args, **kwds)
+        child.logfile = sys.stdout
+        return child
+
+    def spawn(self, argv):
+        return self._spawn(sys.executable, [app_main] + argv)
+
+    def test_interactive(self):
+        child = self.spawn([])
+        child.expect('Python ')   # banner
+        child.expect('>>> ')      # prompt
+        child.sendline('[6*7]')
+        child.expect(re.escape('[42]'))
+        child.sendline('def f(x):')
+        child.expect(re.escape('... '))
+        child.sendline('    return x + 100')
+        child.expect(re.escape('... '))
+        child.sendline('')
+        child.expect('>>> ')
+        child.sendline('f(98)')
+        child.expect('198')
+        child.expect('>>> ')
+        child.sendline('__name__')
+        child.expect("'__main__'")
+        child.expect('>>> ')
+        child.sendline('import sys')
+        child.expect('>>> ')
+        child.sendline("'' in sys.path")
+        child.expect("True")
+
+    def test_help(self):
+        # test that -h prints the usage, including the name of the executable
+        # which should be /full/path/to/app_main.py in this case
+        child = self.spawn(['-h'])
+        child.expect(r'usage: .*app_main.py \[option\]')
+        child.expect('PyPy options and arguments:')
+
+    def test_run_script(self):
+        child = self.spawn([demo_script])
+        idx = child.expect(['hello', 'Python ', '>>> '])
+        assert idx == 0   # no banner or prompt
+        child.expect(re.escape("Name: __main__"))
+        child.expect(re.escape('File: ' + demo_script))
+        child.expect(re.escape('Exec: ' + app_main))
+        child.expect(re.escape('Argv: ' + repr([demo_script])))
+        child.expect('goodbye')
+
+    def test_run_script_with_args(self):
+        argv = [demo_script, 'hello', 'world']
+        child = self.spawn(argv)
+        child.expect(re.escape('Argv: ' + repr(argv)))
+        child.expect('goodbye')
+
+    def test_no_such_script(self):
+        import errno
+        msg = os.strerror(errno.ENOENT)   # 'No such file or directory'
+        child = self.spawn(['xxx-no-such-file-xxx'])
+        child.expect(re.escape(msg))
+
+    def test_option_i(self):
+        argv = [demo_script, 'foo', 'bar']
+        child = self.spawn(['-i'] + argv)
+        idx = child.expect(['hello', re.escape(banner)])
+        assert idx == 0      # no banner
+        child.expect(re.escape('File: ' + demo_script))
+        child.expect(re.escape('Argv: ' + repr(argv)))
+        child.expect('goodbye')
+        idx = child.expect(['>>> ', re.escape(banner)])
+        assert idx == 0      # prompt, but still no banner
+        child.sendline('myvalue * 102')
+        child.expect('4284')
+        child.sendline('__name__')
+        child.expect('__main__')
+
+    def test_option_i_crashing(self):
+        argv = [crashing_demo_script, 'foo', 'bar']
+        child = self.spawn(['-i'] + argv)
+        idx = child.expect(['Hello2', re.escape(banner)])
+        assert idx == 0      # no banner
+        child.expect('NameError')
+        child.sendline('myvalue2 * 1001')
+        child.expect('11011')
+        child.sendline('import sys; sys.argv')
+        child.expect(re.escape(repr(argv)))
+        child.sendline('sys.last_type.__name__')
+        child.expect(re.escape(repr('NameError')))
+
+    def test_options_i_c(self):
+        child = self.spawn(['-i', '-c', 'x=555'])
+        idx = child.expect(['>>> ', re.escape(banner)])
+        assert idx == 0      # prompt, but no banner
+        child.sendline('x')
+        child.expect('555')
+        child.sendline('__name__')
+        child.expect('__main__')
+        child.sendline('import sys; sys.argv')
+        child.expect(re.escape("['-c']"))
+
+    def test_options_i_c_crashing(self, monkeypatch):
+        monkeypatch.setenv('PYTHONPATH', None)
+        child = self.spawn(['-i', '-c', 'x=666;foobar'])
+        child.expect('NameError')
+        idx = child.expect(['>>> ', re.escape(banner)])
+        assert idx == 0      # prompt, but no banner
+        child.sendline('x')
+        child.expect('666')
+        child.sendline('__name__')
+        child.expect('__main__')
+        child.sendline('import sys; sys.argv')
+        child.expect(re.escape("['-c']"))
+        child.sendline('sys.last_type.__name__')
+        child.expect(re.escape(repr('NameError')))
+
+    def test_atexit(self):
+        child = self.spawn([])
+        child.expect('>>> ')
+        child.sendline('def f(): print "foobye"')
+        child.sendline('')
+        child.sendline('import atexit; atexit.register(f)')
+        child.sendline('6*7')
+        child.expect('42')
+        # pexpect's sendeof() is confused by py.test capturing, though
+        # I think that it is a bug of sendeof()
+        old = sys.stdin
+        try:
+            sys.stdin = child
+            child.sendeof()
+        finally:
+            sys.stdin = old
+        child.expect('foobye')
+
+    def test_pythonstartup(self, monkeypatch):
+        monkeypatch.setenv('PYTHONPATH', None)
+        monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script)
+        child = self.spawn([])
+        child.expect(re.escape(banner))
+        child.expect('Traceback')
+        child.expect('NameError')
+        child.expect('>>> ')
+        child.sendline('[myvalue2]')
+        child.expect(re.escape('[11]'))
+        child.expect('>>> ')
+
+        child = self.spawn(['-i', demo_script])
+        for line in ['hello', 'goodbye', '>>> ']:
+            idx = child.expect([line, 'Hello2'])
+            assert idx == 0    # no PYTHONSTARTUP run here
+        child.sendline('myvalue2')
+        child.expect('Traceback')
+        child.expect('NameError')
+
+    def test_pythonstartup_file1(self, monkeypatch):
+        monkeypatch.setenv('PYTHONPATH', None)
+        monkeypatch.setenv('PYTHONSTARTUP', demo_script)
+        child = self.spawn([])
+        child.expect('File: [^\n]+\.py')
+        child.expect('goodbye')
+        child.expect('>>> ')
+        child.sendline('[myvalue]')
+        child.expect(re.escape('[42]'))
+        child.expect('>>> ')
+        child.sendline('__file__')
+        child.expect('Traceback')
+        child.expect('NameError')
+
+    def test_pythonstartup_file2(self, monkeypatch):
+        monkeypatch.setenv('PYTHONPATH', None)
+        monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script)
+        child = self.spawn([])
+        child.expect('Traceback')
+        child.expect('>>> ')
+        child.sendline('__file__')
+        child.expect('Traceback')
+        child.expect('NameError')
+
+    def test_ignore_python_startup(self):
+        old = os.environ.get('PYTHONSTARTUP', '')
+        try:
+            os.environ['PYTHONSTARTUP'] = crashing_demo_script
+            child = self.spawn(['-E'])
+            child.expect(re.escape(banner))
+            index = child.expect(['Traceback', '>>> '])
+            assert index == 1      # no traceback
+        finally:
+            os.environ['PYTHONSTARTUP'] = old
+
+    def test_ignore_python_inspect(self):
+        os.environ['PYTHONINSPECT_'] = '1'
+        try:
+            child = self.spawn(['-E', '-c', 'pass'])
+            from pexpect import EOF
+            index = child.expect(['>>> ', EOF])
+            assert index == 1      # no prompt
+        finally:
+            del os.environ['PYTHONINSPECT_']
+
+    def test_python_path_keeps_duplicates(self):
+        old = os.environ.get('PYTHONPATH', '')
+        try:
+            os.environ['PYTHONPATH'] = 'foobarbaz:foobarbaz'
+            child = self.spawn(['-c', 'import sys; print sys.path'])
+            child.expect(r"\['', 'foobarbaz', 'foobarbaz', ")
+        finally:
+            os.environ['PYTHONPATH'] = old
+
+    def test_ignore_python_path(self):
+        old = os.environ.get('PYTHONPATH', '')
+        try:
+            os.environ['PYTHONPATH'] = 'foobarbaz'
+            child = self.spawn(['-E', '-c', 'import sys; print sys.path'])
+            from pexpect import EOF
+            index = child.expect(['foobarbaz', EOF])
+            assert index == 1      # no foobarbaz
+        finally:
+            os.environ['PYTHONPATH'] = old
+
+    def test_unbuffered(self):
+        line = 'import os,sys;sys.stdout.write(str(789));os.read(0,1)'
+        child = self.spawn(['-u', '-c', line])
+        child.expect('789')    # expect to see it before the timeout hits
+        child.sendline('X')
+
+    def test_options_i_m(self, monkeypatch):
+        if sys.platform == "win32":
+            skip("close_fds is not supported on Windows platforms")
+        if not hasattr(runpy, '_run_module_as_main'):
+            skip("requires CPython >= 2.6")
+        p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
+        p = os.path.abspath(p)
+        monkeypatch.chdir(os.path.dirname(app_main))
+        child = self.spawn(['-i',
+                            '-m', 'test.mymodule',
+                            'extra'])
+        child.expect('mymodule running')
+        child.expect('Name: __main__')
+        child.expect(re.escape('File: ' + p))
+        child.expect(re.escape('Argv: ' + repr([p, 'extra'])))
+        child.expect('>>> ')
+        child.sendline('somevalue')
+        child.expect(re.escape(repr("foobar")))
+        child.expect('>>> ')
+        child.sendline('import sys')
+        child.sendline('"test" in sys.modules')
+        child.expect('True')
+        child.sendline('"test.mymodule" in sys.modules')
+        child.expect('False')
+        child.sendline('sys.path[0]')
+        child.expect("''")
+
+    def test_option_i_noexit(self):
+        child = self.spawn(['-i', '-c', 'import sys; sys.exit(1)'])
+        child.expect('Traceback')
+        child.expect('SystemExit: 1')
+
+    def test_options_u_i(self):
+        if sys.platform == "win32":
+            skip("close_fds is not supported on Windows platforms")
+        import subprocess, select, os
+        python = sys.executable
+        pipe = subprocess.Popen([python, app_main, "-u", "-i"],
+                                stdout=subprocess.PIPE,
+                                stdin=subprocess.PIPE,
+                                stderr=subprocess.STDOUT,
+                                bufsize=0, close_fds=True)
+        iwtd, owtd, ewtd = select.select([pipe.stdout], [], [], 5)
+        assert iwtd    # else we timed out
+        data = os.read(pipe.stdout.fileno(), 1024)
+        assert data.startswith('Python')
+
+    def test_paste_several_lines_doesnt_mess_prompt(self):
+        py.test.skip("this can only work if readline is enabled")
+        child = self.spawn([])
+        child.expect('>>> ')
+        child.sendline('if 1:\n    print 42\n')
+        child.expect('...     print 42')
+        child.expect('... ')
+        child.expect('42')
+        child.expect('>>> ')
+
+    def test_pythoninspect(self):
+        os.environ['PYTHONINSPECT_'] = '1'
+        try:
+            path = getscript("""
+                print 6*7
+                """)
+            child = self.spawn([path])
+            child.expect('42')
+            child.expect('>>> ')
+        finally:
+            del os.environ['PYTHONINSPECT_']
+
+    def test_set_pythoninspect(self):
+        path = getscript("""
+            import os
+            os.environ['PYTHONINSPECT'] = '1'
+            print 6*7
+            """)
+        child = self.spawn([path])
+        child.expect('42')
+        child.expect('>>> ')
+
+    def test_clear_pythoninspect(self):
+        os.environ['PYTHONINSPECT_'] = '1'
+        try:
+            path = getscript("""
+                import os
+                del os.environ['PYTHONINSPECT']
+                """)
+            child = self.spawn([path])
+            child.expect('>>> ')
+        finally:
+            del os.environ['PYTHONINSPECT_']
+
+    def test_stdout_flushes_before_stdin_blocks(self):
+        # This doesn't really test app_main.py, but a behavior that
+        # can only be checked on top of py.py with pexpect.
+        path = getscript("""
+            import sys
+            sys.stdout.write('Are you suggesting coconuts migrate? ')
+            line = sys.stdin.readline()
+            assert line.rstrip() == 'Not at all. They could be carried.'
+            print 'A five ounce bird could not carry a one pound coconut.'
+            """)
+        py_py = os.path.join(pypydir, 'bin', 'pyinteractive.py')
+        child = self._spawn(sys.executable, [py_py, '-S', path])
+        child.expect('Are you suggesting coconuts migrate?', timeout=120)
+        child.sendline('Not at all. They could be carried.')
+        child.expect('A five ounce bird could not carry a one pound coconut.')
+
+    def test_no_space_before_argument(self, monkeypatch):
+        if not hasattr(runpy, '_run_module_as_main'):
+            skip("requires CPython >= 2.6")
+        child = self.spawn(['-cprint "hel" + "lo"'])
+        child.expect('hello')
+
+        monkeypatch.chdir(os.path.dirname(app_main))
+        child = self.spawn(['-mtest.mymodule'])
+        child.expect('mymodule running')
+
+    def test_ps1_only_if_interactive(self):
+        argv = ['-c', 'import sys; print hasattr(sys, "ps1")']
+        child = self.spawn(argv)
+        child.expect('False')
+
+
+class TestNonInteractive:
+    def run_with_status_code(self, cmdline, senddata='', expect_prompt=False,
+            expect_banner=False, python_flags='', env=None):
+        cmdline = '%s %s "%s" %s' % (sys.executable, python_flags,
+                                     app_main, cmdline)
+        print 'POPEN:', cmdline
+        process = subprocess.Popen(
+            cmdline,
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+            shell=True, env=env,
+            universal_newlines=True
+        )
+        child_in, child_out_err = process.stdin, process.stdout
+        child_in.write(senddata)
+        child_in.close()
+        data = child_out_err.read()
+        child_out_err.close()
+        process.wait()
+        assert (banner in data) == expect_banner   # no banner unless expected
+        assert ('>>> ' in data) == expect_prompt   # no prompt unless expected
+        return data, process.returncode
+
+    def run(self, *args, **kwargs):
+        data, status = self.run_with_status_code(*args, **kwargs)
+        return data
+
+    def test_script_on_stdin(self):
+        for extraargs, expected_argv in [
+            ('',              ['']),
+            ('-',             ['-']),
+            ('- hello world', ['-', 'hello', 'world']),
+            ]:
+            data = self.run('%s < "%s"' % (extraargs, demo_script))
+            assert "hello" in data
+            assert "Name: __main__" in data
+            assert "File: <stdin>" in data
+            assert ("Exec: " + app_main) in data
+            assert ("Argv: " + repr(expected_argv)) in data
+            assert "goodbye" in data
+
+    def test_run_crashing_script(self):
+        data = self.run('"%s"' % (crashing_demo_script,))
+        assert 'Hello2' in data
+        assert 'NameError' in data
+        assert 'Goodbye2' not in data
+
+    def test_crashing_script_on_stdin(self):
+        data = self.run(' < "%s"' % (crashing_demo_script,))
+        assert 'Hello2' in data
+        assert 'NameError' in data
+        assert 'Goodbye2' not in data
+
+    def test_option_W(self):
+        data = self.run('-W d -c "print 42"')
+        assert '42' in data
+        data = self.run('-Wd -c "print 42"')
+        assert '42' in data
+
+    def test_option_W_crashing(self):
+        data = self.run('-W')
+        assert "Argument expected for the '-W' option" in data
+
+    def test_option_W_arg_ignored(self):
+        data = self.run('-Wc')
+        assert "Invalid -W option ignored: invalid action: 'c'" in data
+
+    def test_option_W_arg_ignored2(self):
+        data = self.run('-W-W')
+        assert "Invalid -W option ignored: invalid action:" in data
+
+    def test_option_c(self):
+        data = self.run('-c "print 6**5"')
+        assert '7776' in data
+
+    def test_no_pythonstartup(self, monkeypatch):
+        monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script)
+        data = self.run('"%s"' % (demo_script,))
+        assert 'Hello2' not in data
+        data = self.run('-c pass')
+        assert 'Hello2' not in data
+
+    def test_pythonwarnings(self, monkeypatch):
+        # PYTHONWARNINGS_ is special cased by app_main: we cannot directly set
+        # PYTHONWARNINGS because else the warnings raised from within pypy are
+        # turned in errors.
+        monkeypatch.setenv('PYTHONWARNINGS_', "once,error")
+        data = self.run('-W ignore -W default '
+                        '-c "import sys; print sys.warnoptions"')
+        assert "['ignore', 'default', 'once', 'error']" in data
+
+    def test_option_m(self, monkeypatch):
+        if not hasattr(runpy, '_run_module_as_main'):
+            skip("requires CPython >= 2.6")
+        p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
+        p = os.path.abspath(p)
+        monkeypatch.chdir(os.path.dirname(app_main))
+        data = self.run('-m test.mymodule extra')
+        assert 'mymodule running' in data
+        assert 'Name: __main__' in data
+        # ignoring case for windows. abspath behaves different from autopath
+        # concerning drive letters right now.
+        assert ('File: ' + p) in data
+        assert ('Argv: ' + repr([p, 'extra'])) in data
+
+    def test_pythoninspect_doesnt_override_isatty(self):
+        os.environ['PYTHONINSPECT_'] = '1'
+        try:
+            data = self.run('', senddata='6*7\nprint 2+3\n')
+            assert data == '5\n'
+        finally:
+            del os.environ['PYTHONINSPECT_']
+
+    def test_i_flag_overrides_isatty(self):
+        data = self.run('-i', senddata='6*7\nraise SystemExit\n',
+                              expect_prompt=True, expect_banner=True)
+        assert '42\n' in data
+        # if a file name is passed, the banner is never printed but
+        # we get a prompt anyway
+        cmdline = '-i %s' % getscript("""
+            print 'hello world'
+            """)
+        data = self.run(cmdline, senddata='6*7\nraise SystemExit\n',
+                                 expect_prompt=True, expect_banner=False)
+        assert 'hello world\n' in data
+        assert '42\n' in data
+
+    def test_option_S_copyright(self):
+        data = self.run('-S -i', expect_prompt=True, expect_banner=True)
+        assert 'copyright' not in data
+
+    def test_non_interactive_stdout_fully_buffered(self):
+        path = getscript(r"""
+            import sys, time
+            sys.stdout.write('\x00(STDOUT)\n\x00')   # stays in buffers
+            time.sleep(1)
+            sys.stderr.write('\x00[STDERR]\n\x00')
+            time.sleep(1)
+            # stdout flushed automatically here
+            """)
+        cmdline = '%s -u "%s" %s' % (sys.executable, app_main, path)
+        print 'POPEN:', cmdline
+        child_in, child_out_err = os.popen4(cmdline)
+        data = child_out_err.read(11)
+        assert data == '\x00[STDERR]\n\x00'    # from stderr
+        child_in.close()
+        data = child_out_err.read(11)
+        assert data == '\x00(STDOUT)\n\x00'    # from stdout
+        child_out_err.close()
+
+    def test_non_interactive_stdout_unbuffered(self, monkeypatch):
+        monkeypatch.setenv('PYTHONUNBUFFERED', '1')
+        path = getscript(r"""
+            import sys, time
+            sys.stdout.write('\x00(STDOUT)\n\x00')
+            time.sleep(1)
+            sys.stderr.write('\x00[STDERR]\n\x00')
+            time.sleep(1)
+            # stdout flushed automatically here
+            """)
+        cmdline = '%s -E "%s" %s' % (sys.executable, app_main, path)
+        print 'POPEN:', cmdline
+        child_in, child_out_err = os.popen4(cmdline)
+        data = child_out_err.read(11)
+        assert data == '\x00(STDOUT)\n\x00'    # from stderr
+        data = child_out_err.read(11)
+        assert data == '\x00[STDERR]\n\x00'    # from stdout
+        child_out_err.close()
+        child_in.close()
+
+    def test_proper_sys_path(self, tmpdir):
+        data = self.run('-c "import _ctypes"', python_flags='-S')
+        if data.startswith('Traceback'):
+            py.test.skip("'python -S' cannot import extension modules: "
+                         "see probably http://bugs.python.org/issue586680")
+
+        @contextmanager
+        def chdir_and_unset_pythonpath(new_cwd):
+            old_cwd = new_cwd.chdir()
+            old_pythonpath = os.getenv('PYTHONPATH')
+            os.unsetenv('PYTHONPATH')
+            try:
+                yield
+            finally:
+                old_cwd.chdir()
+                # Can't call putenv with a None argument.
+                if old_pythonpath is not None:
+                    os.putenv('PYTHONPATH', old_pythonpath)
+
+        tmpdir.join('site.py').write('print "SHOULD NOT RUN"')
+        runme_py = tmpdir.join('runme.py')
+        runme_py.write('print "some text"')
+
+        cmdline = str(runme_py)
+
+        with chdir_and_unset_pythonpath(tmpdir):
+            data = self.run(cmdline, python_flags='-S')
+
+        assert data == "some text\n"
+
+        runme2_py = tmpdir.mkdir('otherpath').join('runme2.py')
+        runme2_py.write('print "some new text"\n'
+                        'import sys\n'
+                        'print sys.path\n')
+
+        cmdline2 = str(runme2_py)
+
+        with chdir_and_unset_pythonpath(tmpdir):
+            data = self.run(cmdline2, python_flags='-S')
+        assert data.startswith("some new text\n")
+        assert repr(str(tmpdir.join('otherpath'))) in data
+        assert "''" not in data
+
+        data = self.run('-c "import sys; print sys.path"')
+        assert data.startswith("[''")
+
+    def test_pyc_commandline_argument(self):
+        p = getscript_pyc(self.space, "print 6*7\n")
+        assert os.path.isfile(p) and p.endswith('.pyc')
+        data = self.run(p)
+        assert data == 'in _run_compiled_module\n'
+
+    def test_main_in_dir_commandline_argument(self):
+        if not hasattr(runpy, '_run_module_as_main'):
+            skip("requires CPython >= 2.6")
+        p = getscript_in_dir('import sys; print sys.argv[0]\n')
+        data = self.run(p)
+        assert data == p + '\n'
+        data = self.run(p + os.sep)
+        assert data == p + os.sep + '\n'
+
+    def test_getfilesystemencoding(self):
+        py.test.skip("encoding is only set if stdout.isatty(), test is flawed")
+        if sys.version_info < (2, 7):
+            skip("test requires Python >= 2.7")
+        p = getscript_in_dir("""
+        import sys
+        sys.stdout.write(u'15\u20ac')
+        sys.stdout.flush()
+        """)
+        env = os.environ.copy()
+        env["LC_CTYPE"] = 'en_US.UTF-8'
+        data = self.run(p, env=env)
+        assert data == '15\xe2\x82\xac'
+
+    def test_pythonioencoding(self):
+        if sys.version_info < (2, 7):
+            skip("test requires Python >= 2.7")
+        for encoding, expected in [
+            ("iso-8859-15", "15\xa4"),
+            ("utf-8", '15\xe2\x82\xac'),
+            ("utf-16-le", '1\x005\x00\xac\x20'),
+            ("iso-8859-1:ignore", "15"),
+            ("iso-8859-1:replace", "15?"),
+            ("iso-8859-1:backslashreplace", "15\\u20ac"),
+        ]:
+            p = getscript_in_dir("""
+            import sys
+            sys.stdout.write(u'15\u20ac')
+            sys.stdout.flush()
+            """)
+            env = os.environ.copy()
+            env["PYTHONIOENCODING"] = encoding
+            data = self.run(p, env=env)
+            assert data == expected
+
+    def test_sys_exit_pythonioencoding(self):
+        if sys.version_info < (2, 7):
+            skip("test required Python >= 2.7")
+        p = getscript_in_dir("""
+        import sys
+        sys.exit(u'15\u20ac')
+        """)
+        env = os.environ.copy()
+        env["PYTHONIOENCODING"] = "utf-8"
+        data, status = self.run_with_status_code(p, env=env)
+        assert status == 1
+        assert data.startswith("15\xe2\x82\xac")
+
+
+class TestAppMain:
+    def test_print_info(self):
+        from pypy.interpreter import app_main
+        import sys, cStringIO
+        prev_so = sys.stdout
+        prev_ti = getattr(sys, 'pypy_translation_info', 'missing')
+        sys.pypy_translation_info = {
+            'translation.foo': True,
+            'translation.bar': 42,
+            'translation.egg.something': None,
+            'objspace.x': 'hello',
+        }
+        try:
+            sys.stdout = f = cStringIO.StringIO()
+            py.test.raises(SystemExit, app_main.print_info)
+        finally:
+            sys.stdout = prev_so
+            if prev_ti == 'missing':
+                del sys.pypy_translation_info
+            else:
+                sys.pypy_translation_info = prev_ti
+        assert f.getvalue() == ("[objspace]\n"
+                                "    x = 'hello'\n"
+                                "[translation]\n"
+                                "    bar = 42\n"
+                                "    [egg]\n"
+                                "        something = None\n"
+                                "    foo = True\n")
+
+
+class AppTestAppMain:
+    def setup_class(self):
+        # ----------------------------------------
+        # setup code for test_setup_bootstrap_path
+        # ----------------------------------------
+        from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION
+        cpy_ver = '%d.%d' % CPYTHON_VERSION[:2]
+
+        goal_dir = os.path.dirname(app_main)
+        # build a directory hierarchy like which contains both bin/pypy-c and
+        # lib/pypy1.2/*
+        prefix = udir.join('pathtest').ensure(dir=1)
+        fake_exe = 'bin/pypy-c'
+        if sys.platform == 'win32':
+            fake_exe += '.exe'
+        fake_exe = prefix.join(fake_exe).ensure(file=1)
+        expected_path = [str(prefix.join(subdir).ensure(dir=1))
+                         for subdir in ('lib_pypy',
+                                        'lib-python/%s' % cpy_ver)]
+
+        self.w_goal_dir = self.space.wrap(goal_dir)
+        self.w_fake_exe = self.space.wrap(str(fake_exe))
+        self.w_expected_path = self.space.wrap(expected_path)
+        self.w_trunkdir = self.space.wrap(os.path.dirname(pypydir))
+
+        foo_py = prefix.join('foo.py').write("pass")
+        self.w_foo_py = self.space.wrap(str(foo_py))
+
+    def test_setup_bootstrap_path(self):
+        import sys
+        old_sys_path = sys.path[:]
+        sys.path.append(self.goal_dir)
+        try:
+            import app_main
+            app_main.setup_bootstrap_path('/tmp/pypy-c') # stdlib not found
+            assert sys.executable == ''
+            assert sys.path == old_sys_path + [self.goal_dir]
+
+            app_main.setup_bootstrap_path(self.fake_exe)
+            assert sys.executable == self.fake_exe
+            assert self.goal_dir not in sys.path
+
+            newpath = sys.path[:]
+            if newpath[0].endswith('__extensions__'):
+                newpath = newpath[1:]
+            # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
+            assert newpath[:len(self.expected_path)] == self.expected_path
+        finally:
+            sys.path[:] = old_sys_path
+
+    def test_trunk_can_be_prefix(self):
+        import sys
+        import os
+        old_sys_path = sys.path[:]
+        sys.path.append(self.goal_dir)
+        try:
+            import app_main
+            pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c')
+            app_main.setup_bootstrap_path(pypy_c)
+            newpath = sys.path[:]
+            # we get at least lib_pypy 
+            # lib-python/X.Y.Z, and maybe more (e.g. plat-linux2)
+            assert len(newpath) >= 2
+            for p in newpath:
+                assert p.startswith(self.trunkdir)
+        finally:
+            sys.path[:] = old_sys_path
+
+    def test_entry_point(self):
+        import sys
+        import os
+        old_sys_path = sys.path[:]
+        sys.path.append(self.goal_dir)
+        try:
+            import app_main
+            pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c')
+            app_main.entry_point(pypy_c, [self.foo_py])
+            # assert it did not crash
+        finally:
+            sys.path[:] = old_sys_path
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -0,0 +1,18 @@
+from pypy.goal.targetpypystandalone import get_entry_point, create_entry_point
+from pypy.config.pypyoption import get_pypy_config
+
+class TestTargetPyPy(object):
+    def test_run(self):
+        config = get_pypy_config(translating=False)
+        entry_point = get_entry_point(config)[0]
+        entry_point(['pypy-c' , '-S', '-c', 'print 3'])
+
+def test_exeucte_source(space):
+    _, execute_source = create_entry_point(space, None)
+    execute_source("import sys; sys.modules['xyz'] = 3")
+    x = space.int_w(space.getitem(space.getattr(space.builtin_modules['sys'],
+                                                space.wrap('modules')),
+                                                space.wrap('xyz')))
+    assert x == 3
+    execute_source("sys")
+    # did not crash - the same globals
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -485,3 +485,68 @@
         pckl   = pickle.dumps(pack.mod)
         result = pickle.loads(pckl)
         assert pack.mod is result
+
+
+class AppTestGeneratorCloning:
+
+    def setup_class(cls):
+        try:
+            cls.space.appexec([], """():
+                def f(): yield 42
+                f().__reduce__()
+            """)
+        except TypeError, e:
+            if 'pickle generator' not in str(e):
+                raise
+            py.test.skip("Frames can't be __reduce__()-ed")
+
+    def test_deepcopy_generator(self):
+        import copy
+
+        def f(n):
+            for i in range(n):
+                yield 42 + i
+        g = f(4)
+        g2 = copy.deepcopy(g)
+        res = g.next()
+        assert res == 42
+        res = g2.next()
+        assert res == 42
+        g3 = copy.deepcopy(g)
+        res = g.next()
+        assert res == 43
+        res = g2.next()
+        assert res == 43
+        res = g3.next()
+        assert res == 43
+
+    def test_shallowcopy_generator(self):
+        """Note: shallow copies of generators are often confusing.
+        To start with, 'for' loops have an iterator that will not
+        be copied, and so create tons of confusion.
+        """
+        import copy
+
+        def f(n):
+            while n > 0:
+                yield 42 + n
+                n -= 1
+        g = f(2)
+        g2 = copy.copy(g)
+        res = g.next()
+        assert res == 44
+        res = g2.next()
+        assert res == 44
+        g3 = copy.copy(g)
+        res = g.next()
+        assert res == 43
+        res = g2.next()
+        assert res == 43
+        res = g3.next()
+        assert res == 43
+        g4 = copy.copy(g2)
+        for i in range(2):
+            raises(StopIteration, g.next)
+            raises(StopIteration, g2.next)
+            raises(StopIteration, g3.next)
+            raises(StopIteration, g4.next)
diff --git a/pypy/interpreter/test2/__init__.py b/pypy/interpreter/test2/__init__.py
deleted file mode 100644
--- a/pypy/interpreter/test2/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-#empty
diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py
deleted file mode 100644
--- a/pypy/interpreter/test2/test_app_main.py
+++ /dev/null
@@ -1,965 +0,0 @@
-"""
-Tests for the entry point of pypy-c, app_main.py.
-"""
-from __future__ import with_statement
-import py
-import sys, os, re, runpy, subprocess
-from rpython.tool.udir import udir
-from contextlib import contextmanager
-from pypy.conftest import pypydir
-
-banner = sys.version.splitlines()[0]
-
-app_main = os.path.join(os.path.realpath(os.path.dirname(__file__)), os.pardir, 'app_main.py')
-app_main = os.path.abspath(app_main)
-
-_counter = 0
-def _get_next_path(ext='.py'):
-    global _counter
-    p = udir.join('demo_test_app_main_%d%s' % (_counter, ext))
-    _counter += 1
-    return p
-
-def getscript(source):
-    p = _get_next_path()
-    p.write(str(py.code.Source(source)))
-    return str(p)
-
-def getscript_pyc(space, source):
-    p = _get_next_path()
-    p.write(str(py.code.Source(source)))
-    w_dir = space.wrap(str(p.dirpath()))
-    w_modname = space.wrap(p.purebasename)
-    space.appexec([w_dir, w_modname], """(dir, modname):
-        import sys
-        d = sys.modules.copy()
-        sys.path.insert(0, dir)
-        __import__(modname)
-        sys.path.pop(0)
-        for key in sys.modules.keys():
-            if key not in d:
-                del sys.modules[key]
-    """)
-    p = str(p) + 'c'
-    assert os.path.isfile(p)   # the .pyc file should have been created above
-    return p
-
-def getscript_in_dir(source):
-    pdir = _get_next_path(ext='')
-    p = pdir.ensure(dir=1).join('__main__.py')
-    p.write(str(py.code.Source(source)))
-    # return relative path for testing purposes 
-    return py.path.local().bestrelpath(pdir)
-
-demo_script = getscript("""
-    print 'hello'
-    print 'Name:', __name__
-    print 'File:', __file__
-    import sys
-    print 'Exec:', sys.executable
-    print 'Argv:', sys.argv
-    print 'goodbye'
-    myvalue = 6*7
-    """)
-
-crashing_demo_script = getscript("""
-    print 'Hello2'
-    myvalue2 = 11
-    ooups
-    myvalue2 = 22
-    print 'Goodbye2'   # should not be reached
-    """)
-
-
-class TestParseCommandLine:
-    def check_options(self, options, sys_argv, **expected):
-        assert sys.argv == sys_argv
-        for key, value in expected.items():
-            assert options[key] == value
-        for key, value in options.items():
-            if key not in expected:
-                assert not value, (
-                    "option %r has unexpectedly the value %r" % (key, value))
-
-    def check(self, argv, env, **expected):
-        import StringIO
-        from pypy.interpreter import app_main
-        saved_env = os.environ.copy()
-        saved_sys_argv = sys.argv[:]
-        saved_sys_stdout = sys.stdout
-        saved_sys_stderr = sys.stdout
-        app_main.os = os
-        try:
-            os.environ.update(env)
-            sys.stdout = sys.stderr = StringIO.StringIO()
-            try:
-                options = app_main.parse_command_line(argv)
-            except SystemExit:
-                output = expected['output_contains']
-                assert output in sys.stdout.getvalue()
-            else:
-                self.check_options(options, **expected)
-        finally:
-            os.environ.clear()
-            os.environ.update(saved_env)
-            sys.argv[:] = saved_sys_argv
-            sys.stdout = saved_sys_stdout
-            sys.stderr = saved_sys_stderr
-
-    def test_all_combinations_I_can_think_of(self):
-        self.check([], {}, sys_argv=[''], run_stdin=True)
-        self.check(['-'], {}, sys_argv=['-'], run_stdin=True)
-        self.check(['-S'], {}, sys_argv=[''], run_stdin=True, no_site=1)
-        self.check(['-OO'], {}, sys_argv=[''], run_stdin=True, optimize=2)
-        self.check(['-O', '-O'], {}, sys_argv=[''], run_stdin=True, optimize=2)
-        self.check(['-Qnew'], {}, sys_argv=[''], run_stdin=True, division_new=1)
-        self.check(['-Qold'], {}, sys_argv=[''], run_stdin=True, division_new=0)
-        self.check(['-Qwarn'], {}, sys_argv=[''], run_stdin=True, division_warning=1)
-        self.check(['-Qwarnall'], {}, sys_argv=[''], run_stdin=True,
-                   division_warning=2)
-        self.check(['-Q', 'new'], {}, sys_argv=[''], run_stdin=True, division_new=1)
-        self.check(['-SOQnew'], {}, sys_argv=[''], run_stdin=True,
-                   no_site=1, optimize=1, division_new=1)
-        self.check(['-SOQ', 'new'], {}, sys_argv=[''], run_stdin=True,
-                   no_site=1, optimize=1, division_new=1)
-        self.check(['-i'], {}, sys_argv=[''], run_stdin=True,
-                   interactive=1, inspect=1)
-        self.check(['-?'], {}, output_contains='usage:')
-        self.check(['-h'], {}, output_contains='usage:')
-        self.check(['-S', '-tO', '-h'], {}, output_contains='usage:')
-        self.check(['-S', '-thO'], {}, output_contains='usage:')
-        self.check(['-S', '-tO', '--help'], {}, output_contains='usage:')
-        self.check(['-S', '-tO', '--info'], {}, output_contains='translation')
-        self.check(['-S', '-tO', '--version'], {}, output_contains='Python')
-        self.check(['-S', '-tOV'], {}, output_contains='Python')
-        self.check(['--jit', 'foobar', '-S'], {}, sys_argv=[''],
-                   run_stdin=True, no_site=1)
-        self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass')
-        self.check(['-cpass'], {}, sys_argv=['-c'], run_command='pass')

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:47 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix a comment
Message-ID: <20130604095847.2CE371C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64759:1b7d2a50f5f2
Date: 2013-05-10 17:26 +0400
http://bitbucket.org/pypy/pypy/changeset/1b7d2a50f5f2/

Log:	Fix a comment

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,7 +2,7 @@
 # App-level version of py.py.
 # See test/test_app_main.
 
-# Missing vs CPython: -d, -OO, -t, -v, -x, -3
+# Missing vs CPython: -d, -t, -v, -x, -3
 """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:50 2013
From: noreply at buildbot.pypy.org (waedt)
Date: Tue,  4 Jun 2013 11:58:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge pypy/pypy default
Message-ID: <20130604095850.7A5D81C016D@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade <wayedt at gmail.com>
Branch: 
Changeset: r64760:317c58dc4f89
Date: 2013-05-22 10:36 -0500
http://bitbucket.org/pypy/pypy/changeset/317c58dc4f89/

Log:	Merge pypy/pypy default

diff too long, truncating to 2000 out of 11846 lines

diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -1,30 +1,16 @@
-"""Provide access to Python's configuration information.  The specific
-configuration variables available depend heavily on the platform and
-configuration.  The values may be retrieved using
-get_config_var(name), and the list of variables is available via
-get_config_vars().keys().  Additional convenience functions are also
-available.
-
-Written by:   Fred L. Drake, Jr.
-Email:        <fdrake at acm.org>
-"""
-
-__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
-
-import sys
-
 
 # The content of this file is redirected from
 # sysconfig_cpython or sysconfig_pypy.
+# All underscore names are imported too, because
+# people like to use undocumented sysconfig._xxx
+# directly.
 
+import sys
 if '__pypy__' in sys.builtin_module_names:
-    from distutils.sysconfig_pypy import *
-    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
-    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+    from distutils import sysconfig_pypy as _sysconfig_module
 else:
-    from distutils.sysconfig_cpython import *
-    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
-    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
+    from distutils import sysconfig_cpython as _sysconfig_module
+globals().update(_sysconfig_module.__dict__)
 
 _USE_CLANG = None
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -9,7 +9,7 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
 import os
 import re
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -1,6 +1,15 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
 import imp
@@ -119,7 +128,7 @@
     optional C speedup components.
     """
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-fPIC', '-Wimplicit'])
+        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CFLAGS" in os.environ:
             cflags = os.environ["CFLAGS"].split()
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -324,7 +324,12 @@
                 if self._close:
                     self._sock.close()
                 else:
-                    self._sock._decref_socketios()
+                    try:
+                        self._sock._decref_socketios()
+                    except AttributeError:
+                        pass  # bah, someone built a _fileobject manually
+                              # with some unexpected replacement of the
+                              # _socketobject class
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_codecs.py b/lib-python/2.7/test/test_codecs.py
--- a/lib-python/2.7/test/test_codecs.py
+++ b/lib-python/2.7/test/test_codecs.py
@@ -2,7 +2,11 @@
 import unittest
 import codecs
 import locale
-import sys, StringIO, _testcapi
+import sys, StringIO
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 class Queue(object):
     """
@@ -1387,7 +1391,7 @@
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
-            if encoding not in broken_incremental_coders:
+            if encoding not in broken_incremental_coders and _testcapi:
                 # check incremental decoder/encoder (fetched via the Python
                 # and C API) and iterencode()/iterdecode()
                 try:
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -7,7 +7,8 @@
 import subprocess
 from copy import copy, deepcopy
 
-from test.test_support import run_unittest, TESTFN, unlink, get_attribute
+from test.test_support import (run_unittest, TESTFN, unlink, get_attribute,
+                               import_module)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -236,7 +237,10 @@
 
     def test_get_config_h_filename(self):
         config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
+        # import_module skips the test when the CPython C Extension API
+        # appears to not be supported
+        self.assertTrue(os.path.isfile(config_h) or
+                        not import_module('_testcapi'), config_h)
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -1,6 +1,9 @@
 """Test cases for traceback module"""
 
-from _testcapi import traceback_print
+try:
+    from _testcapi import traceback_print
+except ImportError:
+    traceback_print = None
 from StringIO import StringIO
 import sys
 import unittest
@@ -176,6 +179,8 @@
 class TracebackFormatTests(unittest.TestCase):
 
     def test_traceback_format(self):
+        if traceback_print is None:
+            raise unittest.SkipTest('Requires _testcapi')
         try:
             raise KeyError('blah')
         except KeyError:
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -1609,7 +1609,10 @@
         self.assertEqual("{}".format(u), '__unicode__ overridden')
 
     def test_encode_decimal(self):
-        from _testcapi import unicode_encodedecimal
+        try:
+            from _testcapi import unicode_encodedecimal
+        except ImportError:
+            raise unittest.SkipTest('Requires _testcapi')
         self.assertEqual(unicode_encodedecimal(u'123'),
                          b'123')
         self.assertEqual(unicode_encodedecimal(u'\u0663.\u0661\u0664'),
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -130,7 +130,7 @@
     RegrTest('test_bz2.py', usemodules='bz2'),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
-    RegrTest('test_capi.py'),
+    RegrTest('test_capi.py', usemodules='cpyext'),
     RegrTest('test_cd.py'),
     RegrTest('test_cfgparser.py'),
     RegrTest('test_cgi.py'),
@@ -177,7 +177,7 @@
     RegrTest('test_cprofile.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
     RegrTest('test_csv.py', usemodules='_csv'),
-    RegrTest('test_ctypes.py', usemodules="_rawffi thread"),
+    RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
     RegrTest('test_curses.py'),
     RegrTest('test_datetime.py', usemodules='binascii struct'),
     RegrTest('test_dbm.py'),
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -54,4 +54,9 @@
     fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
     imp.load_module('_testcapi', fp, filename, description)
 
-compile_shared()
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testcapi'")
+else:
+    compile_shared()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -32,10 +32,11 @@
      "rctime" , "select", "zipimport", "_lsprof",
      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
-     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
+     "thread", "itertools", "pyexpat", "_ssl", "array",
      "binascii", "_multiprocessing", '_warnings',
      "_collections", "_multibytecodec", "micronumpy", "_ffi",
-     "_continuation", "_cffi_backend", "_csv", "cppyy"]
+     "_continuation", "_cffi_backend", "_csv"] # "cpyext", "cppyy"]
+# disabled until problems are fixed
 ))
 
 translation_modules = default_modules.copy()
@@ -120,12 +121,10 @@
                     __import__(name)
             except (ImportError, CompilationError, py.test.skip.Exception), e:
                 errcls = e.__class__.__name__
-                config.add_warning(
+                raise Exception(
                     "The module %r is disabled\n" % (modname,) +
                     "because importing %s raised %s\n" % (name, errcls) +
                     str(e))
-                raise ConflictConfigError("--withmod-%s: %s" % (modname,
-                                                                errcls))
         return validator
     else:
         return None
@@ -364,6 +363,7 @@
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
+
     config.objspace.usemodules.suggest(**dict.fromkeys(modules, True))
 
 def enable_translationmodules(config):
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -339,8 +339,9 @@
 
 + methods and other class attributes do not change after startup
 + single inheritance is fully supported
-+ simple mixins work too, but the mixed in class needs a ``_mixin_ = True``
-  class attribute
++ simple mixins somewhat work too, but the mixed in class needs a
+  ``_mixin_ = True`` class attribute. isinstance checks against the
+  mixin type will fail when translated.
 
 + classes are first-class objects too
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.0'
+release = '2.0.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -28,7 +28,8 @@
 Layers
 ------
 
-PyPy has layers. Those layers help us keep the respective parts separated enough
+PyPy has layers. Just like Ogres or onions.
+Those layers help us keep the respective parts separated enough
 to be worked on independently and make the complexity manageable. This is,
 again, just a sanity requirement for such a complex project. For example writing
 a new optimization for the JIT usually does **not** involve touching a Python
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -22,7 +22,8 @@
   will capture the revision number of this change for the release;
   some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
-  necessary
+  necessary; also update the version number in pypy/doc/conf.py,
+  and in pypy/doc/index.rst
 * update pypy/doc/contributor.rst (and possibly LICENSE)
 * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
   and create a fresh whatsnew_head.rst after the release
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -40,7 +40,7 @@
 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.0`_: the latest official release
+* `Release 2.0.2`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
@@ -110,7 +110,7 @@
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.0`: http://pypy.org/download.html
+.. _`Release 2.0.2`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
diff --git a/pypy/doc/release-2.0.1.rst b/pypy/doc/release-2.0.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.1.rst
@@ -0,0 +1,46 @@
+==============================
+PyPy 2.0.1 - Bohr Sm&#248;rrebr&#248;d
+==============================
+
+We're pleased to announce PyPy 2.0.1.  This is a stable bugfix release
+over `2.0`_.  You can download it here:
+  
+    http://pypy.org/download.html
+
+The fixes are mainly about fatal errors or crashes in our stdlib.  See
+below for more details.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Support for ARM is progressing but not bug-free yet.
+
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
+
+Highlights
+==========
+
+- fix an occasional crash in the JIT that ends in `RPython Fatal error:
+  NotImplementedError`__.
+
+- `id(x)` is now always a positive number (except on int/float/long/complex).
+  This fixes an issue in ``_sqlite.py`` (mostly for 32-bit Linux).
+
+- fix crashes of callback-from-C-functions (with cffi) when used together
+  with Stackless features, on asmgcc (i.e. Linux only).  Now `gevent should
+  work better`__.
+
+- work around an eventlet issue with `socket._decref_socketios()`__.
+
+.. __: https://bugs.pypy.org/issue1482
+.. __: http://mail.python.org/pipermail/pypy-dev/2013-May/011362.html
+.. __: https://bugs.pypy.org/issue1468
+.. _2.0: release-2.0.0.html
+
+Cheers,
+arigo et. al. for the PyPy team
diff --git a/pypy/doc/release-2.0.2.rst b/pypy/doc/release-2.0.2.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.2.rst
@@ -0,0 +1,46 @@
+=========================
+PyPy 2.0.2 - Fermi Panini
+=========================
+
+We're pleased to announce PyPy 2.0.2.  This is a stable bugfix release
+over `2.0`_ and `2.0.1`_.  You can download it here:
+
+    http://pypy.org/download.html
+
+It fixes a crash in the JIT when calling external C functions (with
+ctypes/cffi) in a multithreaded context.
+
+.. _2.0: release-2.0.0.html
+.. _2.0.1: release-2.0.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Support for ARM is progressing but not bug-free yet.
+
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
+
+Highlights
+==========
+
+This release contains only the fix described above.  A crash (or wrong
+results) used to occur if all these conditions were true:
+
+- your program is multithreaded;
+
+- it runs on a single-core machine or a heavily-loaded multi-core one;
+
+- it uses ctypes or cffi to issue external calls to C functions.
+
+This was fixed in the branch `emit-call-x86`__ (see the example file
+``bug1.py``).
+
+.. __: https://bitbucket.org/pypy/pypy/commits/7c80121abbf4
+
+Cheers,
+arigo et. al. for the PyPy team
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -19,23 +19,28 @@
     branches.discard('default')
     return startrev, branches
 
-def get_merged_branches(path, startrev, endrev):
-    if getstatusoutput('hg root')[0]:
+def get_merged_branches(path, startrev, endrev, current_branch=None):
+    errcode, wc_branch = getstatusoutput('hg branch')
+    if errcode != 0:
         py.test.skip('no Mercurial repo')
+    if current_branch is None:
+        current_branch = wc_branch
 
     # X = take all the merges which are descendants of startrev and are on default
     # revset = all the parents of X which are not on default
     # ===>
     # revset contains all the branches which have been merged to default since
     # startrev
-    revset = 'parents(%s::%s and \
+    revset = "parents(%s::%s and \
                       merge() and \
-                      branch(default)) and \
-              not branch(default)' % (startrev, endrev)
+                      branch('%s')) and \
+              not branch('%s')" % (startrev, endrev,
+                                   current_branch, current_branch)
     cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset)
     out = getoutput(cmd)
     branches = set(map(str.strip, out.splitlines()))
-    return branches
+    branches.discard("default")
+    return branches, current_branch
 
 
 def test_parse_doc():
@@ -65,7 +70,8 @@
     assert branches == set(['foobar', 'hello'])
 
 def test_get_merged_branches():
-    branches = get_merged_branches(ROOT, 'f34f0c11299f', '79770e0c2f93')
+    branches, _ = get_merged_branches(ROOT, 'f34f0c11299f', '79770e0c2f93',
+                                      'default')
     assert branches == set(['numpy-indexing-by-arrays-bool',
                             'better-jit-hooks-2',
                             'numpypy-ufuncs'])
@@ -76,7 +82,9 @@
     whatsnew_list.sort()
     last_whatsnew = whatsnew_list[-1].read()
     startrev, documented = parse_doc(last_whatsnew)
-    merged = get_merged_branches(ROOT, startrev, '')
+    merged, branch = get_merged_branches(ROOT, startrev, '')
+    merged.discard('default')
+    merged.discard('')
     not_documented = merged.difference(documented)
     not_merged = documented.difference(merged)
     print 'Branches merged but not documented:'
@@ -85,4 +93,6 @@
     print 'Branches documented but not merged:'
     print '\n'.join(not_merged)
     print
-    assert not not_documented and not not_merged
+    assert not not_documented
+    if branch == 'default':
+        assert not not_merged
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,24 @@
 
 .. branch: numpy-pickle
 Pickling of numpy arrays and dtypes (including record dtypes)
+
+.. branch: remove-array-smm
+Remove multimethods in the arraymodule
+
+.. branch: callback-stacklet
+Fixed bug when switching stacklets from a C callback
+
+.. branch: remove-set-smm
+Remove multi-methods on sets
+
+.. branch: numpy-subarrays
+Implement subarrays for numpy
+
+.. branch: remove-dict-smm
+Remove multi-methods on dict
+
+.. branch: remove-list-smm-2
+Remove remaining multi-methods on list
+
+.. branch: arm-stacklet
+Stacklet support for ARM, enables _continuation support
diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py
--- a/pypy/goal/getnightly.py
+++ b/pypy/goal/getnightly.py
@@ -8,7 +8,9 @@
     arch = 'linux'
     cmd = 'wget "%s"'
     tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'"
-if sys.platform.startswith('darwin'):
+    if os.uname()[-1].startswith('arm'):
+        arch += '-armhf-raspbian'
+elif sys.platform.startswith('darwin'):
     arch = 'osx'
     cmd = 'curl -O "%s"'
     tar = "tar -x -v --strip-components=2 -f %s '*/bin/pypy'"
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -2,6 +2,7 @@
 
 import os, sys
 
+import pypy
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError
 from pypy.tool.ann_override import PyPyAnnotatorPolicy
@@ -9,6 +10,8 @@
 from rpython.config.config import ConflictConfigError
 from pypy.tool.option import make_objspace
 from pypy.conftest import pypydir
+from rpython.rlib import rthread
+from pypy.module.thread import os_thread
 
 thisdir = py.path.local(__file__).dirpath()
 
@@ -78,13 +81,58 @@
     # should be used as sparsely as possible, just to register callbacks
 
     from rpython.rlib.entrypoint import entrypoint
-    from rpython.rtyper.lltypesystem import rffi
+    from rpython.rtyper.lltypesystem import rffi, lltype
+
+    w_pathsetter = space.appexec([], """():
+    def f(path):
+        import sys
+        sys.path[:] = path
+    return f
+    """)
+
+    @entrypoint('main', [rffi.CCHARP, lltype.Signed], c_name='pypy_setup_home')
+    def pypy_setup_home(ll_home, verbose):
+        from pypy.module.sys.initpath import pypy_find_stdlib
+        if ll_home:
+            home = rffi.charp2str(ll_home)
+        else:
+            home = pypydir
+        w_path = pypy_find_stdlib(space, home)
+        if space.is_none(w_path):
+            if verbose:
+                debug("Failed to find library based on pypy_find_stdlib")
+            return 1
+        space.startup()
+        space.call_function(w_pathsetter, w_path)
+        # import site
+        try:
+            import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
+                                    space.wrap('__import__'))
+            space.call_function(import_, space.wrap('site'))
+            return 0
+        except OperationError, e:
+            if verbose:
+                debug("OperationError:")
+                debug(" operror-type: " + e.w_type.getname(space))
+                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            return 1
 
     @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
         source = rffi.charp2str(ll_source)
         return _pypy_execute_source(source)
 
+    @entrypoint('main', [], c_name='pypy_init_threads')
+    def pypy_init_threads():
+        if space.config.objspace.usemodules.thread:
+            os_thread.setup_threads(space)
+            rffi.aroundstate.before()
+
+    @entrypoint('main', [], c_name='pypy_thread_attach')
+    def pypy_thread_attach():
+        if space.config.objspace.usemodules.thread:
+            rthread.gc_thread_start()
+
     w_globals = space.newdict()
     space.setitem(w_globals, space.wrap('__builtins__'),
                   space.builtin_modules['__builtin__'])
@@ -101,7 +149,10 @@
             return 1
         return 0
 
-    return entry_point, _pypy_execute_source # for tests
+    return entry_point, {'pypy_execute_source': pypy_execute_source,
+                         'pypy_init_threads': pypy_init_threads,
+                         'pypy_thread_attach': pypy_thread_attach,
+                         'pypy_setup_home': pypy_setup_home}
 
 def call_finish(space):
     space.finish()
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -855,9 +855,10 @@
             self.emit_jump(ops.JUMP_IF_FALSE_OR_POP, cleanup, True)
             if i < (ops_count - 1):
                 comp.comparators[i].walkabout(self)
-        comp.comparators[-1].walkabout(self)
-        last_kind = compare_operations(comp.ops[-1])
-        self.emit_op_arg(ops.COMPARE_OP, last_kind)
+        last_op, last_comparator = comp.ops[-1], comp.comparators[-1]
+        if not self._optimize_comparator(last_op, last_comparator):
+            last_comparator.walkabout(self)
+        self.emit_op_arg(ops.COMPARE_OP, compare_operations(last_op))
         if ops_count > 1:
             end = self.new_block()
             self.emit_jump(ops.JUMP_FORWARD, end)
@@ -866,6 +867,37 @@
             self.emit_op(ops.POP_TOP)
             self.use_next_block(end)
 
+    def _optimize_comparator(self, op, node):
+        """Fold lists/sets of constants in the context of "in"/"not in".
+
+        lists are folded into tuples, sets into frozensets, otherwise
+        returns False
+        """
+        if op in (ast.In, ast.NotIn):
+            is_list = isinstance(node, ast.List)
+            if is_list or isinstance(node, ast.Set):
+                w_const = self._tuple_of_consts(node.elts)
+                if w_const is not None:
+                    if not is_list:
+                        from pypy.objspace.std.setobject import (
+                            W_FrozensetObject)
+                        w_const = W_FrozensetObject(self.space, w_const)
+                    self.load_const(w_const)
+                    return True
+        return False
+
+    def _tuple_of_consts(self, elts):
+        """Return a tuple of consts from elts if possible, or None"""
+        count = len(elts) if elts is not None else 0
+        consts_w = [None] * count
+        for i in range(count):
+            w_value = elts[i].as_constant()
+            if w_value is None:
+                # Not all constants
+                return None
+            consts_w[i] = w_value
+        return self.space.newtuple(consts_w)
+
     def visit_IfExp(self, ifexp):
         self.update_position(ifexp.lineno)
         end = self.new_block()
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1025,3 +1025,30 @@
         counts = self.count_instructions(source3)
         assert counts[ops.BUILD_LIST] == 1
         assert ops.BUILD_LIST_FROM_ARG not in counts
+
+    def test_folding_of_list_constants(self):
+        for source in (
+            # in/not in constants with BUILD_LIST should be folded to a tuple:
+            'a in [1,2,3]',
+            'a not in ["a","b","c"]',
+            'a in [None, 1, None]',
+            'a not in [(1, 2), 3, 4]',
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BUILD_LIST not in counts
+            assert ops.LOAD_CONST in counts
+
+    def test_folding_of_set_constants(self):
+        for source in (
+            # in/not in constants with BUILD_SET should be folded to a frozenset:
+            'a in {1,2,3}',
+            'a not in {"a","b","c"}',
+            'a in {None, 1, None}',
+            'a not in {(1, 2), 3, 4}',
+            'a in {1, 2, 3, 3, 2, 1}',
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BUILD_SET not in counts
+            assert ops.LOAD_CONST in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -242,6 +242,11 @@
     def __spacebind__(self, space):
         return self
 
+    def unwrap(self, space):
+        """NOT_RPYTHON"""
+        # _____ this code is here to support testing only _____
+        return self
+
 
 class W_InterpIterable(W_Root):
     def __init__(self, space, w_iterable):
@@ -666,7 +671,8 @@
     def id(self, w_obj):
         w_result = w_obj.immutable_unique_id(self)
         if w_result is None:
-            w_result = self.wrap(compute_unique_id(w_obj))
+            # in the common case, returns an unsigned value
+            w_result = self.wrap(r_uint(compute_unique_id(w_obj)))
         return w_result
 
     def hash_w(self, w_obj):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -803,7 +803,6 @@
     args = inspect.getargs(func.func_code)
     if args.varargs or args.keywords:
         raise TypeError("Varargs and keywords not supported in unwrap_spec")
-    assert not func.func_defaults
     argspec = ', '.join([arg for arg in args.args[1:]])
     func_code = py.code.Source("""
     def f(w_obj, %(args)s):
@@ -812,11 +811,13 @@
     d = {}
     exec func_code.compile() in d
     f = d['f']
+    f.func_defaults = unbound_meth.func_defaults
+    f.func_doc = unbound_meth.func_doc
     f.__module__ = func.__module__
     # necessary for unique identifiers for pickling
     f.func_name = func.func_name
     if unwrap_spec is None:
-        unwrap_spec = {}
+        unwrap_spec = getattr(unbound_meth, 'unwrap_spec', {})
     else:
         assert isinstance(unwrap_spec, dict)
         unwrap_spec = unwrap_spec.copy()
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -903,22 +903,33 @@
         expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
                                         'lib-python/%s' % cpy_ver)]
+        # an empty directory from where we can't find the stdlib
+        tmp_dir = str(udir.join('tmp').ensure(dir=1))
 
         self.w_goal_dir = self.space.wrap(goal_dir)
         self.w_fake_exe = self.space.wrap(str(fake_exe))
         self.w_expected_path = self.space.wrap(expected_path)
         self.w_trunkdir = self.space.wrap(os.path.dirname(pypydir))
 
+        self.w_tmp_dir = self.space.wrap(tmp_dir)
+
         foo_py = prefix.join('foo.py').write("pass")
         self.w_foo_py = self.space.wrap(str(foo_py))
 
     def test_setup_bootstrap_path(self):
-        import sys
+        # Check how sys.path is handled depending on if we can find a copy of
+        # the stdlib in setup_bootstrap_path.
+        import sys, os
         old_sys_path = sys.path[:]
+        old_cwd = os.getcwd()
+
         sys.path.append(self.goal_dir)
+        # make sure cwd does not contain a stdlib
+        os.chdir(self.tmp_dir)
+        tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
             import app_main
-            app_main.setup_bootstrap_path('/tmp/pypy-c') # stdlib not found
+            app_main.setup_bootstrap_path(tmp_pypy_c)  # stdlib not found
             assert sys.executable == ''
             assert sys.path == old_sys_path + [self.goal_dir]
 
@@ -933,6 +944,7 @@
             assert newpath[:len(self.expected_path)] == self.expected_path
         finally:
             sys.path[:] = old_sys_path
+            os.chdir(old_cwd)
 
     def test_trunk_can_be_prefix(self):
         import sys
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -936,6 +936,21 @@
         output = s.getvalue()
         assert "LOAD_GLOBAL" not in output
 
+    def test_folding_of_list_constants(self):
+        source = 'a in [1, 2, 3]'
+        co = compile(source, '', 'exec')
+        i = co.co_consts.index((1, 2, 3))
+        assert i > -1
+        assert isinstance(co.co_consts[i], tuple)
+
+    def test_folding_of_set_constants(self):
+        source = 'a in {1, 2, 3}'
+        co = compile(source, '', 'exec')
+        i = co.co_consts.index(set([1, 2, 3]))
+        assert i > -1
+        assert isinstance(co.co_consts[i], frozenset)
+
+
 class AppTestCallMethod(object):
     spaceconfig = {'objspace.opcodes.CALL_METHOD': True}
         
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -135,18 +135,39 @@
 
     def test_interpindirect2app(self):
         space = self.space
+
         class BaseA(W_Root):
             def method(self, space, x):
+                "This is a method"
+                pass
+
+            def method_with_default(self, space, x=5):
+                pass
+
+            @gateway.unwrap_spec(x=int)
+            def method_with_unwrap_spec(self, space, x):
                 pass
 
         class A(BaseA):
             def method(self, space, x):
                 return space.wrap(x + 2)
 
+            def method_with_default(self, space, x):
+                return space.wrap(x + 2)
+
+            def method_with_unwrap_spec(self, space, x):
+                return space.wrap(x + 2)
+
         class B(BaseA):
             def method(self, space, x):
                 return space.wrap(x + 1)
 
+            def method_with_default(self, space, x):
+                return space.wrap(x + 1)
+
+            def method_with_unwrap_spec(self, space, x):
+                return space.wrap(x + 1)
+
         class FakeTypeDef(object):
             rawdict = {}
             bases = {}
@@ -163,6 +184,23 @@
         assert space.int_w(space.call_function(w_c, w_a, space.wrap(1))) == 1 + 2
         assert space.int_w(space.call_function(w_c, w_b, space.wrap(-10))) == -10 + 1
 
+        doc = space.str_w(space.getattr(w_c, space.wrap('__doc__')))
+        assert doc == "This is a method"
+
+        meth_with_default = gateway.interpindirect2app(
+            BaseA.method_with_default, {'x': int})
+        w_d = space.wrap(meth_with_default)
+
+        assert space.int_w(space.call_function(w_d, w_a, space.wrap(4))) == 4 + 2
+        assert space.int_w(space.call_function(w_d, w_b, space.wrap(-10))) == -10 + 1
+        assert space.int_w(space.call_function(w_d, w_a)) == 5 + 2
+        assert space.int_w(space.call_function(w_d, w_b)) == 5 + 1
+
+        meth_with_unwrap_spec = gateway.interpindirect2app(
+            BaseA.method_with_unwrap_spec)
+        w_e = space.wrap(meth_with_unwrap_spec)
+        assert space.int_w(space.call_function(w_e, w_a, space.wrap(4))) == 4 + 2
+
     def test_interp2app_unwrap_spec(self):
         space = self.space
         w = space.wrap
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -1,5 +1,6 @@
 from pypy.goal.targetpypystandalone import get_entry_point, create_entry_point
 from pypy.config.pypyoption import get_pypy_config
+from rpython.rtyper.lltypesystem import rffi, lltype
 
 class TestTargetPyPy(object):
     def test_run(self):
@@ -8,11 +9,20 @@
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
 def test_exeucte_source(space):
-    _, execute_source = create_entry_point(space, None)
-    execute_source("import sys; sys.modules['xyz'] = 3")
+    _, d = create_entry_point(space, None)
+    execute_source = d['pypy_execute_source']
+    lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
+    execute_source(lls)
+    lltype.free(lls, flavor='raw')
     x = space.int_w(space.getitem(space.getattr(space.builtin_modules['sys'],
                                                 space.wrap('modules')),
                                                 space.wrap('xyz')))
     assert x == 3
-    execute_source("sys")
+    lls = rffi.str2charp("sys")
+    execute_source(lls)
+    lltype.free(lls, flavor='raw')
     # did not crash - the same globals
+    pypy_setup_home = d['pypy_setup_home']
+    lls = rffi.str2charp(__file__)
+    pypy_setup_home(lls, 1)
+    lltype.free(lls, flavor='raw')
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -55,6 +55,7 @@
         'validate_fd'               : 'interp_magic.validate_fd',
         'resizelist_hint'           : 'interp_magic.resizelist_hint',
         'newlist_hint'              : 'interp_magic.newlist_hint',
+        'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'dictstrategy'              : 'interp_dict.dictstrategy',
         'set_debug'                 : 'interp_magic.set_debug',
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,11 +1,10 @@
-from pypy.interpreter.baseobjspace import ObjSpace, W_Root
 from pypy.interpreter.error import OperationError, wrap_oserror
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.objspace.std.typeobject import MethodCache
 from pypy.objspace.std.mapdict import IndexCache
-from rpython.rlib import rposix
+from rpython.rlib import rposix, rgc
 
 
 def internal_repr(space, w_object):
@@ -56,7 +55,7 @@
     bltn = BuiltinFunction(func)
     return space.wrap(bltn)
 
- at unwrap_spec(ObjSpace, W_Root, str)
+ at unwrap_spec(meth=str)
 def lookup_special(space, w_obj, meth):
     """Lookup up a special method on an object."""
     if space.is_oldstyle_instance(w_obj):
@@ -108,3 +107,7 @@
     space.setitem(space.builtin.w_dict,
                   space.wrap('__debug__'),
                   space.wrap(debug))
+
+ at unwrap_spec(estimate=int)
+def add_memory_pressure(estimate):
+    rgc.add_memory_pressure(estimate)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -114,8 +114,11 @@
     ge = _make_comparison('ge')
 
     def hash(self):
-        h = (objectmodel.compute_identity_hash(self.ctype) ^
-             rffi.cast(lltype.Signed, self._cdata))
+        h = rffi.cast(lltype.Signed, self._cdata)
+        # To hash pointers in dictionaries.  Assumes that h shows some
+        # alignment (to 4, 8, maybe 16 bytes), so we use the following
+        # formula to avoid the trailing bits being always 0.
+        h = h ^ (h >> 4)
         return self.space.wrap(h)
 
     def getitem(self, w_index):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -365,8 +365,9 @@
     BInt = new_primitive_type("int")
     BFloat = new_primitive_type("float")
     for i in range(1, 20):
-        if (hash(cast(BChar, chr(i))) !=
-            hash(cast(BInt, i))):
+        x1 = cast(BChar, chr(i))
+        x2 = cast(BInt, i)
+        if hash(x1) != hash(x2):
             break
     else:
         raise AssertionError("hashes are equal")
@@ -2723,6 +2724,14 @@
     assert x.__name__ == '<cdata>'
     assert hasattr(x, '__doc__')
 
+def test_different_types_of_ptr_equality():
+    BVoidP = new_pointer_type(new_void_type())
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    x = cast(BVoidP, 12345)
+    assert x == cast(BIntP, 12345)
+    assert x != cast(BIntP, 12344)
+    assert hash(x) == hash(cast(BIntP, 12345))
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "0.6"
diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py
--- a/pypy/module/_ffi/test/test_funcptr.py
+++ b/pypy/module/_ffi/test/test_funcptr.py
@@ -74,9 +74,9 @@
         from _ffi import CDLL, types
         # this should return *all* loaded libs, dlopen(NULL)
         dll = CDLL(None)
-        # Assume CPython, or PyPy compiled with cpyext
-        res = dll.getfunc('Py_IsInitialized', [], types.slong)()
-        assert res == 1
+        # libm should be loaded
+        res = dll.getfunc('sqrt', [types.double], types.double)(1.0)
+        assert res == 1.0
 
     def test_callfunc(self):
         from _ffi import CDLL, types
@@ -139,7 +139,7 @@
 
     def test_pointer_args(self):
         """
-            extern int dummy; // defined in test_void_result 
+            extern int dummy; // defined in test_void_result
             DLLEXPORT int* get_dummy_ptr() { return &dummy; }
             DLLEXPORT void set_val_to_ptr(int* ptr, int val) { *ptr = val; }
         """
@@ -158,7 +158,7 @@
 
     def test_convert_pointer_args(self):
         """
-            extern int dummy; // defined in test_void_result 
+            extern int dummy; // defined in test_void_result
             DLLEXPORT int* get_dummy_ptr(); // defined in test_pointer_args
             DLLEXPORT void set_val_to_ptr(int* ptr, int val); // ditto
         """
@@ -170,7 +170,7 @@
             def _as_ffi_pointer_(self, ffitype):
                 assert ffitype is types.void_p
                 return self.value
-        
+
         libfoo = CDLL(self.libfoo_name)
         get_dummy = libfoo.getfunc('get_dummy', [], types.sint)
         get_dummy_ptr = libfoo.getfunc('get_dummy_ptr', [], types.void_p)
@@ -259,7 +259,7 @@
 
     def test_typed_pointer_args(self):
         """
-            extern int dummy; // defined in test_void_result 
+            extern int dummy; // defined in test_void_result
             DLLEXPORT int* get_dummy_ptr(); // defined in test_pointer_args
             DLLEXPORT void set_val_to_ptr(int* ptr, int val); // ditto
         """
@@ -551,7 +551,7 @@
         from _ffi import CDLL, types
         libfoo = CDLL(self.libfoo_name)
         raises(TypeError, "libfoo.getfunc('sum_xy', [types.void], types.sint)")
-        
+
     def test_OSError_loading(self):
         from _ffi import CDLL, types
         raises(OSError, "CDLL('I do not exist')")
@@ -606,7 +606,7 @@
         from _rawffi import FUNCFLAG_STDCALL
         libm = CDLL(self.libm_name)
         pow_addr = libm.getaddressindll('pow')
-        wrong_pow = FuncPtr.fromaddr(pow_addr, 'pow', 
+        wrong_pow = FuncPtr.fromaddr(pow_addr, 'pow',
                 [types.double, types.double], types.double, FUNCFLAG_STDCALL)
         try:
             wrong_pow(2, 3) == 8
@@ -622,7 +622,7 @@
         from _rawffi import FUNCFLAG_STDCALL
         kernel = WinDLL('Kernel32.dll')
         sleep_addr = kernel.getaddressindll('Sleep')
-        sleep = FuncPtr.fromaddr(sleep_addr, 'sleep', [types.uint], 
+        sleep = FuncPtr.fromaddr(sleep_addr, 'sleep', [types.uint],
                             types.void, FUNCFLAG_STDCALL)
         sleep(10)
 
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -232,9 +232,9 @@
         import _rawffi
         # this should return *all* loaded libs, dlopen(NULL)
         dll = _rawffi.CDLL(None)
-        # Assume CPython, or PyPy compiled with cpyext
-        res = dll.ptr('Py_IsInitialized', [], 'l')()
-        assert res[0] == 1
+        func = dll.ptr('rand', [], 'i')
+        res = func()
+        assert res[0] != 0
 
     def test_libc_load(self):
         import _rawffi
diff --git a/pypy/module/array/__init__.py b/pypy/module/array/__init__.py
--- a/pypy/module/array/__init__.py
+++ b/pypy/module/array/__init__.py
@@ -1,12 +1,5 @@
 from pypy.interpreter.mixedmodule import MixedModule
 
-from pypy.module.array.interp_array import types
-from pypy.objspace.std.model import registerimplementation
-
-for mytype in types.values():
-    registerimplementation(mytype.w_class)
-
-
 class Module(MixedModule):
     interpleveldefs = {
         'array': 'interp_array.W_ArrayBase',
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -2,17 +2,14 @@
 
 from pypy.interpreter.buffer import RWBuffer
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import GetSetProperty, make_weakref_descr
+from pypy.interpreter.gateway import interp2app, unwrap_spec, interpindirect2app
+from pypy.interpreter.typedef import GetSetProperty, make_weakref_descr, TypeDef
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.module._file.interp_file import W_File
-from pypy.objspace.std.model import W_Object
-from pypy.objspace.std.multimethod import FailedToImplement
-from pypy.objspace.std.stdtypedef import SMM, StdTypeDef
-from pypy.objspace.std.register_all import register_all
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import ovfcheck, widen
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.objectmodel import specialize, keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.lltypesystem import lltype, rffi
 
 
@@ -39,9 +36,9 @@
             if len(__args__.arguments_w) > 0:
                 w_initializer = __args__.arguments_w[0]
                 if space.type(w_initializer) is space.w_str:
-                    a.fromstring(space.str_w(w_initializer))
+                    a.descr_fromstring(space, space.str_w(w_initializer))
                 elif space.type(w_initializer) is space.w_list:
-                    a.fromlist(w_initializer)
+                    a.descr_fromlist(space, w_initializer)
                 else:
                     a.extend(w_initializer, True)
             break
@@ -52,31 +49,6 @@
     return a
 
 
-array_append = SMM('append', 2)
-array_extend = SMM('extend', 2)
-
-array_count = SMM('count', 2)
-array_index = SMM('index', 2)
-array_reverse = SMM('reverse', 1)
-array_remove = SMM('remove', 2)
-array_pop = SMM('pop', 2, defaults=(-1,))
-array_insert = SMM('insert', 3)
-
-array_tolist = SMM('tolist', 1)
-array_fromlist = SMM('fromlist', 2)
-array_tostring = SMM('tostring', 1)
-array_fromstring = SMM('fromstring', 2)
-array_tounicode = SMM('tounicode', 1)
-array_fromunicode = SMM('fromunicode', 2)
-array_tofile = SMM('tofile', 2)
-array_fromfile = SMM('fromfile', 3)
-
-array_buffer_info = SMM('buffer_info', 1)
-array_reduce = SMM('__reduce__', 1)
-array_copy = SMM('__copy__', 1)
-array_byteswap = SMM('byteswap', 1)
-
-
 def descr_itemsize(space, self):
     return space.wrap(self.itemsize)
 
@@ -84,28 +56,476 @@
 def descr_typecode(space, self):
     return space.wrap(self.typecode)
 
+arr_eq_driver = jit.JitDriver(greens = ['comp_func'], reds = 'auto')
+EQ, NE, LT, LE, GT, GE = range(6)
 
-class W_ArrayBase(W_Object):
-    @staticmethod
-    def register(typeorder):
-        typeorder[W_ArrayBase] = []
+def compare_arrays(space, arr1, arr2, comp_op, comp_func):
+    if (not isinstance(arr1, W_ArrayBase) or
+        not isinstance(arr2, W_ArrayBase)):
+        return space.w_NotImplemented
+    if comp_op == EQ and arr1.len != arr2.len:
+        return space.w_False
+    if comp_op == NE and arr1.len != arr2.len:
+        return space.w_True
+    lgt = min(arr1.len, arr2.len)
+    for i in range(lgt):
+        arr_eq_driver.jit_merge_point(comp_func=comp_func)
+        w_elem1 = arr1.w_getitem(space, i)
+        w_elem2 = arr2.w_getitem(space, i)
+        res = space.is_true(comp_func(w_elem1, w_elem2))
+        if comp_op == EQ:
+            if not res:
+                return space.w_False
+        elif comp_op == NE:
+            if res:
+                return space.w_True
+        elif comp_op == LT or comp_op == GT:
+            if res:
+                return space.w_True
+            elif not space.is_true(space.eq(w_elem1, w_elem2)):
+                return space.w_False
+        else:
+            if not res:
+                return space.w_False
+            elif not space.is_true(space.eq(w_elem1, w_elem2)):
+                return space.w_True
+    # we have some leftovers
+    if comp_op == EQ:
+        return space.w_True
+    elif comp_op == NE:
+        return space.w_False
+    if arr1.len == arr2.len:
+        if comp_op == LT or comp_op == GT:
+            return space.w_False
+        return space.w_True
+    if comp_op == LT or comp_op == LE:
+        if arr1.len < arr2.len:
+            return space.w_True
+        return space.w_False
+    if arr1.len > arr2.len:
+        return space.w_True
+    return space.w_False
 
-W_ArrayBase.typedef = StdTypeDef(
+UNICODE_ARRAY = lltype.Ptr(lltype.Array(lltype.UniChar,
+                                        hints={'nolength': True}))
+
+class W_ArrayBase(W_Root):
+    _attrs_ = ('space', 'len', 'allocated', '_lifeline_') # no buffer
+
+    def __init__(self, space):
+        self.space = space
+        self.len = 0
+        self.allocated = 0
+
+    def descr_append(self, space, w_x):
+        """ append(x)
+
+        Append new value x to the end of the array.
+        """
+        raise NotImplementedError
+
+    def descr_extend(self, space, w_x):
+        """ extend(array or iterable)
+
+        Append items to the end of the array.
+        """
+        self.extend(w_x)
+
+    def descr_count(self, space, w_val):
+        """ count(x)
+
+        Return number of occurrences of x in the array.
+        """
+        raise NotImplementedError
+
+    def descr_index(self, space, w_x):
+        """ index(x)
+
+        Return index of first occurrence of x in the array.
+        """
+        raise NotImplementedError
+
+    def descr_reverse(self, space):
+        """ reverse()
+
+        Reverse the order of the items in the array.
+        """
+        raise NotImplementedError
+
+    def descr_remove(self, space, w_val):
+        """ remove(x)
+
+        Remove the first occurrence of x in the array.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(i=int)
+    def descr_pop(self, space, i=-1):
+        """ pop([i])
+
+        Return the i-th element and delete it from the array. i defaults to -1.
+        """
+        raise NotImplementedError
+
+    @unwrap_spec(idx=int)
+    def descr_insert(self, space, idx, w_val):
+        """ insert(i,x)
+
+        Insert a new item x into the array before position i.
+        """
+        raise NotImplementedError
+
+    def descr_tolist(self, space):
+        """ tolist() -> list
+
+        Convert array to an ordinary list with the same items.
+        """
+        w_l = space.newlist([])
+        for i in range(self.len):
+            w_l.append(self.w_getitem(space, i))
+        return w_l
+
+    def descr_fromlist(self, space, w_lst):
+        """ fromlist(list)
+
+        Append items to array from list.
+        """
+        if not space.isinstance_w(w_lst, space.w_list):
+            raise OperationError(space.w_TypeError,
+                                 space.wrap("arg must be list"))
+        s = self.len
+        try:
+            self.fromsequence(w_lst)
+        except OperationError:
+            self.setlen(s)
+            raise
+
+    def descr_tostring(self, space):
+        """ tostring() -> string
+
+        Convert the array to an array of machine values and return the string
+        representation.
+        """
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * self.itemsize)
+        self._charbuf_stop()
+        return self.space.wrap(s)
+
+    @unwrap_spec(s=str)
+    def descr_fromstring(self, space, s):
+        """ fromstring(string)
+
+        Appends items from the string, interpreting it as an array of machine
+        values,as if it had been read from a file using the fromfile() method).
+        """
+        if len(s) % self.itemsize != 0:
+            msg = 'string length not a multiple of item size'
+            raise OperationError(self.space.w_ValueError, self.space.wrap(msg))
+        oldlen = self.len
+        new = len(s) / self.itemsize
+        self.setlen(oldlen + new)
+        cbuf = self._charbuf_start()
+        for i in range(len(s)):
+            cbuf[oldlen * self.itemsize + i] = s[i]
+        self._charbuf_stop()
+
+    @unwrap_spec(w_f=W_File, n=int)
+    def descr_fromfile(self, space, w_f, n):
+        """ fromfile(f, n)
+
+        Read n objects from the file object f and append them to the end of the
+        array.  Also called as read.
+        """
+        try:
+            size = ovfcheck(self.itemsize * n)
+        except OverflowError:
+            raise MemoryError
+        w_item = space.call_method(w_f, 'read', space.wrap(size))
+        item = space.str_w(w_item)
+        if len(item) < size:
+            n = len(item) % self.itemsize
+            elems = max(0, len(item) - (len(item) % self.itemsize))
+            if n != 0:
+                item = item[0:elems]
+            self.descr_fromstring(space, item)
+            msg = "not enough items in file"
+            raise OperationError(space.w_EOFError, space.wrap(msg))
+        self.descr_fromstring(space, item)
+
+    @unwrap_spec(w_f=W_File)
+    def descr_tofile(self, space, w_f):
+        """ tofile(f)
+
+        Write all items (as machine values) to the file object f.  Also called as
+        write.
+        """
+        w_s = self.descr_tostring(space)
+        space.call_method(w_f, 'write', w_s)
+
+    def descr_fromunicode(self, space, w_ustr):
+        """ fromunicode(ustr)
+
+        Extends this array with data from the unicode string ustr.
+        The array must be a type 'u' array; otherwise a ValueError
+        is raised.  Use array.fromstring(ustr.decode(...)) to
+        append Unicode data to an array of some other type.
+        """
+        # XXX the following probable bug is not emulated:
+        # CPython accepts a non-unicode string or a buffer, and then
+        # behaves just like fromstring(), except that it strangely truncate
+        # string arguments at multiples of the unicode byte size.
+        # Let's only accept unicode arguments for now.
+        if self.typecode == 'u':
+            self.fromsequence(w_ustr)
+        else:
+            msg = "fromunicode() may only be called on type 'u' arrays"
+            raise OperationError(space.w_ValueError, space.wrap(msg))
+
+    def descr_tounicode(self, space):
+        """ tounicode() -> unicode
+
+        Convert the array to a unicode string.  The array must be
+        a type 'u' array; otherwise a ValueError is raised.  Use
+        array.tostring().decode() to obtain a unicode string from
+        an array of some other type.
+        """
+        if self.typecode == 'u':
+            buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
+            return space.wrap(rffi.wcharpsize2unicode(buf, self.len))
+        else:
+            msg = "tounicode() may only be called on type 'u' arrays"
+            raise OperationError(space.w_ValueError, space.wrap(msg))
+
+    def descr_buffer_info(self, space):
+        """ buffer_info() -> (address, length)
+
+        Return a tuple (address, length) giving the current memory address and
+        the length in items of the buffer used to hold array's contents
+        The length should be multiplied by the itemsize attribute to calculate
+        the buffer length in bytes.
+        """
+        w_ptr = space.wrap(self._buffer_as_unsigned())
+        w_len = space.wrap(self.len)
+        return space.newtuple([w_ptr, w_len])
+
+    def descr_reduce(self, space):
+        """ Return state information for pickling.
+        """
+        if self.len > 0:
+            w_s = self.descr_tostring(space)
+            args = [space.wrap(self.typecode), w_s]
+        else:
+            args = [space.wrap(self.typecode)]
+        try:
+            dct = space.getattr(self, space.wrap('__dict__'))
+        except OperationError:
+            dct = space.w_None
+        return space.newtuple([space.type(self), space.newtuple(args), dct])
+
+    def descr_copy(self, space):
+        """ copy(array)
+
+        Return a copy of the array.
+        """
+        w_a = self.constructor(self.space)
+        w_a.setlen(self.len, overallocate=False)
+        rffi.c_memcpy(
+            rffi.cast(rffi.VOIDP, w_a._buffer_as_unsigned()),
+            rffi.cast(rffi.VOIDP, self._buffer_as_unsigned()),
+            self.len * self.itemsize
+        )
+        return w_a
+
+    def descr_byteswap(self, space):
+        """ byteswap()
+
+        Byteswap all items of the array.  If the items in the array are not 1, 2,
+        4, or 8 bytes in size, RuntimeError is raised.
+        """
+        if self.itemsize not in [1, 2, 4, 8]:
+            msg = "byteswap not supported for this array"
+            raise OperationError(space.w_RuntimeError, space.wrap(msg))
+        if self.len == 0:
+            return
+        bytes = self._charbuf_start()
+        tmp = [bytes[0]] * self.itemsize
+        for start in range(0, self.len * self.itemsize, self.itemsize):
+            stop = start + self.itemsize - 1
+            for i in range(self.itemsize):
+                tmp[i] = bytes[start + i]
+            for i in range(self.itemsize):
+                bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
+
+    def descr_len(self, space):
+        return space.wrap(self.len)
+
+    def descr_eq(self, space, w_arr2):
+        "x.__eq__(y) <==> x==y"
+        return compare_arrays(space, self, w_arr2, EQ, space.eq)
+
+    def descr_ne(self, space, w_arr2):
+        "x.__ne__(y) <==> x!=y"
+        return compare_arrays(space, self, w_arr2, NE, space.ne)
+
+    def descr_lt(self, space, w_arr2):
+        "x.__lt__(y) <==> x<y"
+        return compare_arrays(space, self, w_arr2, LT, space.lt)
+
+    def descr_le(self, space, w_arr2):
+        "x.__le__(y) <==> x<=y"
+        return compare_arrays(space, self, w_arr2, LE, space.le)
+
+    def descr_gt(self, space, w_arr2):
+        "x.__gt__(y) <==> x>y"
+        return compare_arrays(space, self, w_arr2, GT, space.gt)
+
+    def descr_ge(self, space, w_arr2):
+        "x.__ge__(y) <==> x>=y"
+        return compare_arrays(space, self, w_arr2, GE, space.ge)
+
+    # Basic get/set/append/extend methods
+
+    def descr_getitem(self, space, w_idx):
+        "x.__getitem__(y) <==> x[y]"
+        if not space.isinstance_w(w_idx, space.w_slice):
+            idx, stop, step = space.decode_index(w_idx, self.len)
+            assert step == 0
+            return self.w_getitem(space, idx)
+        else:
+            return self.getitem_slice(space, w_idx)
+
+    def descr_getslice(self, space, w_i, w_j):
+        return space.getitem(self, space.newslice(w_i, w_j, space.w_None))
+
+
+    def descr_setitem(self, space, w_idx, w_item):
+        "x.__setitem__(i, y) <==> x[i]=y"
+        if space.isinstance_w(w_idx, space.w_slice):
+            self.setitem_slice(space, w_idx, w_item)
+        else:
+            self.setitem(space, w_idx, w_item)
+
+    def descr_setslice(self, space, w_start, w_stop, w_item):
+        self.setitem_slice(space,
+                           space.newslice(w_start, w_stop, space.w_None),
+                           w_item)
+
+    def descr_delitem(self, space, w_idx):
+        start, stop, step, size = self.space.decode_index4(w_idx, self.len)
+        if step != 1:
+            # I don't care about efficiency of that so far
+            w_lst = self.descr_tolist(space)
+            space.delitem(w_lst, w_idx)
+            self.setlen(0)
+            self.fromsequence(w_lst)
+            return
+        return self.delitem(space, start, stop)
+
+    def descr_delslice(self, space, w_start, w_stop):
+        self.descr_delitem(space, space.newslice(w_start, w_stop, space.w_None))
+
+    def descr_add(self, space, w_other):
+        raise NotImplementedError
+
+    def descr_inplace_add(self, space, w_other):
+        raise NotImplementedError
+
+    def descr_mul(self, space, w_repeat):
+        raise NotImplementedError
+
+    def descr_inplace_mul(self, space, w_repeat):
+        raise NotImplementedError
+
+    def descr_radd(self, space, w_other):
+        return self.descr_add(space, w_other)
+
+    def descr_rmul(self, space, w_repeat):
+        return self.descr_mul(space, w_repeat)
+
+    # Misc methods
+
+    def descr_buffer(self, space):
+        return space.wrap(ArrayBuffer(self))
+
+    def descr_repr(self, space):
+        if self.len == 0:
+            return space.wrap("array('%s')" % self.typecode)
+        elif self.typecode == "c":
+            r = space.repr(self.descr_tostring(space))
+            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
+            return space.wrap(s)
+        elif self.typecode == "u":
+            r = space.repr(self.descr_tounicode(space))
+            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
+            return space.wrap(s)
+        else:
+            r = space.repr(self.descr_tolist(space))
+            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
+            return space.wrap(s)
+
+W_ArrayBase.typedef = TypeDef(
     'array',
     __new__ = interp2app(w_array),
     __module__   = 'array',
+
+    __len__ = interp2app(W_ArrayBase.descr_len),
+    __eq__ = interp2app(W_ArrayBase.descr_eq),
+    __ne__ = interp2app(W_ArrayBase.descr_ne),
+    __lt__ = interp2app(W_ArrayBase.descr_lt),
+    __le__ = interp2app(W_ArrayBase.descr_le),
+    __gt__ = interp2app(W_ArrayBase.descr_gt),
+    __ge__ = interp2app(W_ArrayBase.descr_ge),
+
+    __getitem__ = interp2app(W_ArrayBase.descr_getitem),
+    __getslice__ = interp2app(W_ArrayBase.descr_getslice),
+    __setitem__ = interp2app(W_ArrayBase.descr_setitem),
+    __setslice__ = interp2app(W_ArrayBase.descr_setslice),
+    __delitem__ = interp2app(W_ArrayBase.descr_delitem),
+    __delslice__ = interp2app(W_ArrayBase.descr_delslice),
+
+    __add__ = interpindirect2app(W_ArrayBase.descr_add),
+    __iadd__ = interpindirect2app(W_ArrayBase.descr_inplace_add),
+    __mul__ = interpindirect2app(W_ArrayBase.descr_mul),
+    __imul__ = interpindirect2app(W_ArrayBase.descr_inplace_mul),
+    __radd__ = interp2app(W_ArrayBase.descr_radd),
+    __rmul__ = interp2app(W_ArrayBase.descr_rmul),
+
+    __buffer__ = interp2app(W_ArrayBase.descr_buffer),
+    __repr__ = interp2app(W_ArrayBase.descr_repr),
+
     itemsize = GetSetProperty(descr_itemsize),
     typecode = GetSetProperty(descr_typecode),
     __weakref__ = make_weakref_descr(W_ArrayBase),
+    append = interpindirect2app(W_ArrayBase.descr_append),
+    extend = interp2app(W_ArrayBase.descr_extend),
+    count = interpindirect2app(W_ArrayBase.descr_count),
+    index = interpindirect2app(W_ArrayBase.descr_index),
+    reverse = interpindirect2app(W_ArrayBase.descr_reverse),
+    remove = interpindirect2app(W_ArrayBase.descr_remove),
+    pop = interpindirect2app(W_ArrayBase.descr_pop),
+    insert = interpindirect2app(W_ArrayBase.descr_insert),
+
+    tolist = interp2app(W_ArrayBase.descr_tolist),
+    fromlist = interp2app(W_ArrayBase.descr_fromlist),
+    tostring = interp2app(W_ArrayBase.descr_tostring),
+    fromstring = interp2app(W_ArrayBase.descr_fromstring),
+    tofile = interp2app(W_ArrayBase.descr_tofile),
+    fromfile = interp2app(W_ArrayBase.descr_fromfile),
+    fromunicode = interp2app(W_ArrayBase.descr_fromunicode),
+    tounicode = interp2app(W_ArrayBase.descr_tounicode),
+
+    buffer_info = interp2app(W_ArrayBase.descr_buffer_info),
+    __copy__ = interp2app(W_ArrayBase.descr_copy),
+    __reduce__ = interp2app(W_ArrayBase.descr_reduce),
+    byteswap = interp2app(W_ArrayBase.descr_byteswap),
 )
-W_ArrayBase.typedef.registermethods(globals())
 
 
 class TypeCode(object):
     def __init__(self, itemtype, unwrap, canoverflow=False, signed=False):
         self.itemtype = itemtype
         self.bytes = rffi.sizeof(itemtype)
-        #self.arraytype = lltype.GcArray(itemtype)
         self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
         self.unwrap = unwrap
         self.signed = signed
@@ -175,14 +595,10 @@
         itemsize = mytype.bytes
         typecode = mytype.typecode
 
-        @staticmethod
-        def register(typeorder):
-            typeorder[W_Array] = [(W_ArrayBase, None)]
+        _attrs_ = ('space', 'len', 'allocated', '_lifeline_', 'buffer')
 
         def __init__(self, space):
-            self.space = space
-            self.len = 0
-            self.allocated = 0
+            W_ArrayBase.__init__(self, space)
             self.buffer = lltype.nullptr(mytype.arraytype)
 
         def item_w(self, w_item):
@@ -289,26 +705,6 @@
                 raise
             self.setlen(oldlen + i)
 
-        def fromstring(self, s):
-            if len(s) % self.itemsize != 0:
-                msg = 'string length not a multiple of item size'
-                raise OperationError(self.space.w_ValueError, self.space.wrap(msg))
-            oldlen = self.len
-            new = len(s) / mytype.bytes
-            self.setlen(oldlen + new)
-            cbuf = self._charbuf_start()
-            for i in range(len(s)):
-                cbuf[oldlen * mytype.bytes + i] = s[i]
-            self._charbuf_stop()
-
-        def fromlist(self, w_lst):
-            s = self.len
-            try:
-                self.fromsequence(w_lst)
-            except OperationError:
-                self.setlen(s)
-                raise
-
         def extend(self, w_iterable, accept_different_array=False):
             space = self.space
             if isinstance(w_iterable, W_Array):
@@ -332,6 +728,9 @@
         def _charbuf_start(self):
             return rffi.cast(rffi.CCHARP, self.buffer)
 
+        def _buffer_as_unsigned(self):
+            return rffi.cast(lltype.Unsigned, self.buffer)
+
         def _charbuf_stop(self):
             keepalive_until_here(self)
 
@@ -343,202 +742,180 @@
                 item = float(item)
             return space.wrap(item)
 
-    # Basic get/set/append/extend methods
+        # interface
 
-    def len__Array(space, self):
-        return space.wrap(self.len)
+        def descr_append(self, space, w_x):
+            x = self.item_w(w_x)
+            self.setlen(self.len + 1)
+            self.buffer[self.len - 1] = x
 
-    def getitem__Array_ANY(space, self, w_idx):
-        idx, stop, step = space.decode_index(w_idx, self.len)
-        assert step == 0
-        return self.w_getitem(space, idx)
+        # List interface
+        def descr_count(self, space, w_val):
+            cnt = 0
+            for i in range(self.len):
+                # XXX jitdriver
+                w_item = self.w_getitem(space, i)
+                if space.is_true(space.eq(w_item, w_val)):
+                    cnt += 1
+            return space.wrap(cnt)
 
-    def getitem__Array_Slice(space, self, w_slice):
-        start, stop, step, size = space.decode_index4(w_slice, self.len)
-        w_a = mytype.w_class(self.space)
-        w_a.setlen(size, overallocate=False)
-        assert step != 0
-        j = 0
-        for i in range(start, stop, step):
-            w_a.buffer[j] = self.buffer[i]
-            j += 1
-        return w_a
+        def descr_index(self, space, w_val):
+            for i in range(self.len):
+                w_item = self.w_getitem(space, i)
+                if space.is_true(space.eq(w_item, w_val)):
+                    return space.wrap(i)
+            msg = 'array.index(x): x not in list'
+            raise OperationError(space.w_ValueError, space.wrap(msg))
 
-    def getslice__Array_ANY_ANY(space, self, w_i, w_j):
-        return space.getitem(self, space.newslice(w_i, w_j, space.w_None))
+        def descr_reverse(self, space):
+            b = self.buffer
+            for i in range(self.len / 2):
+                b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i]
 
-    def setitem__Array_ANY_ANY(space, self, w_idx, w_item):
-        idx, stop, step = space.decode_index(w_idx, self.len)
-        if step != 0:
-            msg = 'can only assign array to array slice'
-            raise OperationError(self.space.w_TypeError, self.space.wrap(msg))
-        item = self.item_w(w_item)
-        self.buffer[idx] = item
+        def descr_pop(self, space, i):
+            if i < 0:
+                i += self.len
+            if i < 0 or i >= self.len:
+                msg = 'pop index out of range'
+                raise OperationError(space.w_IndexError, space.wrap(msg))
+            w_val = self.w_getitem(space, i)
+            while i < self.len - 1:
+                self.buffer[i] = self.buffer[i + 1]
+                i += 1
+            self.setlen(self.len - 1)
+            return w_val
 
-    def setitem__Array_Slice_Array(space, self, w_idx, w_item):
-        start, stop, step, size = self.space.decode_index4(w_idx, self.len)
-        assert step != 0
-        if w_item.len != size or self is w_item:
-            # XXX this is a giant slow hack
-            w_lst = array_tolist__Array(space, self)
-            w_item = space.call_method(w_item, 'tolist')
-            space.setitem(w_lst, w_idx, w_item)
-            self.setlen(0)
-            self.fromsequence(w_lst)
-        else:
+        def descr_remove(self, space, w_val):
+            w_idx = self.descr_index(space, w_val)
+            self.descr_pop(space, space.int_w(w_idx))
+
+        def descr_insert(self, space, idx, w_val):
+            if idx < 0:
+                idx += self.len
+            if idx < 0:
+                idx = 0
+            if idx > self.len:
+                idx = self.len
+
+            val = self.item_w(w_val)
+            self.setlen(self.len + 1)
+            i = self.len - 1
+            while i > idx:
+                self.buffer[i] = self.buffer[i - 1]
+                i -= 1
+            self.buffer[i] = val
+
+        def getitem_slice(self, space, w_idx):
+            start, stop, step, size = space.decode_index4(w_idx, self.len)
+            w_a = mytype.w_class(self.space)
+            w_a.setlen(size, overallocate=False)
+            assert step != 0
             j = 0
             for i in range(start, stop, step):
-                self.buffer[i] = w_item.buffer[j]
+                w_a.buffer[j] = self.buffer[i]
                 j += 1
+            return w_a
 
-    def setslice__Array_ANY_ANY_ANY(space, self, w_i, w_j, w_x):
-        space.setitem(self, space.newslice(w_i, w_j, space.w_None), w_x)
+        def setitem(self, space, w_idx, w_item):
+            idx, stop, step = space.decode_index(w_idx, self.len)
+            if step != 0:
+                msg = 'can only assign array to array slice'
+                raise OperationError(self.space.w_TypeError,
+                                     self.space.wrap(msg))
+            item = self.item_w(w_item)
+            self.buffer[idx] = item
 
-    def array_append__Array_ANY(space, self, w_x):
-        x = self.item_w(w_x)
-        self.setlen(self.len + 1)
-        self.buffer[self.len - 1] = x
+        def setitem_slice(self, space, w_idx, w_item):
+            if not isinstance(w_item, W_Array):
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "can only assign to a slice array"))
+            start, stop, step, size = self.space.decode_index4(w_idx, self.len)
+            assert step != 0
+            if w_item.len != size or self is w_item:
+                # XXX this is a giant slow hack
+                w_lst = self.descr_tolist(space)
+                w_item = space.call_method(w_item, 'tolist')
+                space.setitem(w_lst, w_idx, w_item)
+                self.setlen(0)
+                self.fromsequence(w_lst)
+            else:
+                j = 0
+                for i in range(start, stop, step):
+                    self.buffer[i] = w_item.buffer[j]
+                    j += 1
 
-    def array_extend__Array_ANY(space, self, w_iterable):
-        self.extend(w_iterable)
+        # We can't look into this function until ptradd works with things (in the
+        # JIT) other than rffi.CCHARP
+        @jit.dont_look_inside
+        def delitem(self, space, i, j):
+            if i < 0:
+                i += self.len
+            if i < 0:
+                i = 0
+            if j < 0:
+                j += self.len
+            if j < 0:
+                j = 0
+            if j > self.len:
+                j = self.len
+            if i >= j:
+                return None
+            oldbuffer = self.buffer
+            self.buffer = lltype.malloc(mytype.arraytype,
+                          max(self.len - (j - i), 0), flavor='raw',
+                          add_memory_pressure=True)
+            if i:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, self.buffer),
+                    rffi.cast(rffi.VOIDP, oldbuffer),
+                    i * mytype.bytes
+                )
+            if j < self.len:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, i)),
+                    rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer, j)),
+                    (self.len - j) * mytype.bytes
+                )
+            self.len -= j - i
+            self.allocated = self.len
+            if oldbuffer:
+                lltype.free(oldbuffer, flavor='raw')
 
-    # List interface
-    def array_count__Array_ANY(space, self, w_val):
-        cnt = 0
-        for i in range(self.len):
-            w_item = self.w_getitem(space, i)
-            if space.is_true(space.eq(w_item, w_val)):
-                cnt += 1
-        return space.wrap(cnt)
+        # Add and mul methods
 
-    def array_index__Array_ANY(space, self, w_val):
-        for i in range(self.len):
-            w_item = self.w_getitem(space, i)
-            if space.is_true(space.eq(w_item, w_val)):
-                return space.wrap(i)
-        msg = 'array.index(x): x not in list'
-        raise OperationError(space.w_ValueError, space.wrap(msg))
+        def descr_add(self, space, w_other):
+            if not isinstance(w_other, W_Array):
+                return space.w_NotImplemented
+            a = mytype.w_class(space)
+            a.setlen(self.len + w_other.len, overallocate=False)
+            for i in range(self.len):
+                a.buffer[i] = self.buffer[i]
+            for i in range(w_other.len):
+                a.buffer[i + self.len] = w_other.buffer[i]
+            return a
 
-    def array_reverse__Array(space, self):
-        b = self.buffer
-        for i in range(self.len / 2):
-            b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i]
+        def descr_inplace_add(self, space, w_other):
+            if not isinstance(w_other, W_Array):
+                return space.w_NotImplemented
+            oldlen = self.len
+            otherlen = w_other.len
+            self.setlen(oldlen + otherlen)
+            for i in range(otherlen):
+                self.buffer[oldlen + i] = w_other.buffer[i]
+            return self
 
-    def array_pop__Array_ANY(space, self, w_idx):
-        i = space.int_w(w_idx)
-        if i < 0:
-            i += self.len
-        if i < 0 or i >= self.len:
-            msg = 'pop index out of range'
-            raise OperationError(space.w_IndexError, space.wrap(msg))
-        w_val = self.w_getitem(space, i)
-        while i < self.len - 1:

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Jun 2013 11:58:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a comment that -O in pyinteractive
 is -OO in CPython.
Message-ID: <20130604095851.AA3371C016D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64761:4c73a79fd2d3
Date: 2013-06-04 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/4c73a79fd2d3/

Log:	Add a comment that -O in pyinteractive is -OO in CPython.

diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "skip assert statements and remove docstrings when importing modules",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Jun 2013 11:58:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Revert this change,
 unsure why it was there (maybe some merge mistake?)
Message-ID: <20130604095854.8CE3C1C016D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64762:155e0c2706e2
Date: 2013-06-04 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/155e0c2706e2/

Log:	Revert this change, unsure why it was there (maybe some merge
	mistake?)

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -3,7 +3,7 @@
 # See test/test_app_main.
 
 # Missing vs CPython: -d, -t, -v, -x, -3
-"""\
+USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 -c cmd : program passed in as string (terminates option list)
@@ -28,7 +28,6 @@
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
 """
-USAGE1 = __doc__
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
 Other environment variables:
@@ -157,12 +156,13 @@
     raise SystemExit
 
 def print_help(*args):
-    print 'usage: %s [options] [-c cmd|-m mod|file.py|-] [arg...]' % (
+    import os
+    print 'usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % (
         sys.executable,)
-    print __doc__.rstrip()
+    print USAGE1,
     if 'pypyjit' in sys.builtin_module_names:
-        print "  --jit OPTIONS  advanced JIT options: try 'off' or 'help'"
-    print
+        print "--jit options: advanced JIT options: try 'off' or 'help'"
+    print (USAGE2 % (os.pathsep,)),
     raise SystemExit
 
 def _print_jit_help():

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Jun 2013 11:58:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make really JUMP_IF_NOT_DEBUG's target
	relative.
Message-ID: <20130604095856.044641C016D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64763:ea2d3e68529d
Date: 2013-06-04 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/ea2d3e68529d/

Log:	Make really JUMP_IF_NOT_DEBUG's target relative.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -373,7 +373,7 @@
     def visit_Assert(self, asrt):
         self.update_position(asrt.lineno)
         end = self.new_block()
-        self.emit_jump(ops.JUMP_IF_NOT_DEBUG, end, True)
+        self.emit_jump(ops.JUMP_IF_NOT_DEBUG, end)
         asrt.test.accept_jump_if(self, True, end)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "AssertionError")
         if asrt.msg:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -876,9 +876,9 @@
         self.popvalue()
         return next_instr
 
-    def JUMP_IF_NOT_DEBUG(self, target, next_instr):
+    def JUMP_IF_NOT_DEBUG(self, jumpby, next_instr):
         if not self.space.sys.debug:
-            return target
+            next_instr += jumpby
         return next_instr
 
     def GET_ITER(self, oparg, next_instr):

From noreply at buildbot.pypy.org  Tue Jun  4 11:58:57 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Jun 2013 11:58:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge the pull request #149,
	after a few extra fixes:
Message-ID: <20130604095857.606911C016D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64764:e953dfbc7f0a
Date: 2013-06-04 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/e953dfbc7f0a/

Log:	merge the pull request #149, after a few extra fixes:

	 implement the -O and -OO flags, by Tyler Wade

	Most importantly, this is done without generating .pyo files, but by
	checking at runtime if we should run each assert or not. Similarly,
	-OO is done by removing the docstrings from memory after we load the
	.pyc files.

diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Distance to target address
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,7 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,8 +2,8 @@
 # App-level version of py.py.
 # See test/test_app_main.
 
-# Missing vs CPython: -d, -OO, -t, -v, -x, -3
-"""\
+# Missing vs CPython: -d, -t, -v, -x, -3
+USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 -c cmd : program passed in as string (terminates option list)
@@ -12,7 +12,8 @@
 -i     : inspect interactively after running script; forces a prompt even
          if stdin does not appear to be a terminal; also PYTHONINSPECT=x
 -m mod : run library module as a script (terminates option list)
--O     : dummy optimization flag for compatibility with CPython
+-O     : skip assert statements
+-OO    : remove docstrings when importing modules in addition to -O
 -R     : ignored (see http://bugs.python.org/issue14621)
 -Q arg : division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
@@ -27,7 +28,6 @@
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
 """
-USAGE1 = __doc__
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
 Other environment variables:
@@ -470,6 +470,10 @@
         sys.py3kwarning = bool(sys.flags.py3k_warning)
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
+        if sys.flags.optimize >= 1:
+            import __pypy__
+            __pypy__.set_debug(False)
+
         if sys.py3kwarning:
             print >> sys.stderr, (
                 "Warning: pypy does not implement py3k warnings")
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -245,6 +245,8 @@
         if w_len is None:
             w_len = space.len(self.w_consts)
             space.setitem(self.w_consts, w_key, w_len)
+        if space.int_w(w_len) == 0:
+            self.scope.doc_removable = False
         return space.int_w(w_len)
 
     def _make_key(self, obj):
@@ -632,6 +634,7 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+    ops.JUMP_IF_NOT_DEBUG : 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -254,6 +254,7 @@
                 start = 1
                 doc_expr.walkabout(self)
                 self.name_op("__doc__", ast.Store)
+                self.scope.doc_removable = True
             for i in range(start, len(body)):
                 body[i].walkabout(self)
             return True
@@ -371,6 +372,7 @@
     def visit_Assert(self, asrt):
         self.update_position(asrt.lineno)
         end = self.new_block()
+        self.emit_jump(ops.JUMP_IF_NOT_DEBUG, end)
         asrt.test.accept_jump_if(self, True, end)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "AssertionError")
         if asrt.msg:
@@ -1207,7 +1209,10 @@
         tree.walkabout(self)
 
     def _get_code_flags(self):
-        return 0
+        flags = 0
+        if self.scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
+        return flags
 
 
 class AbstractFunctionCodeGenerator(PythonCodeGenerator):
@@ -1234,6 +1239,8 @@
             flags |= consts.CO_VARARGS
         if scope.has_keywords_arg:
             flags |= consts.CO_VARKEYWORDS
+        if scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
         if not self.cell_vars and not self.free_vars:
             flags |= consts.CO_NOFREE
         return PythonCodeGenerator._get_code_flags(self) | flags
@@ -1250,6 +1257,7 @@
             doc_expr = None
         if doc_expr is not None:
             self.add_const(doc_expr.s)
+            self.scope.doc_removable = True
             start = 1
         else:
             self.add_const(self.space.w_None)
@@ -1312,3 +1320,9 @@
         self._handle_body(cls.body)
         self.emit_op(ops.LOAD_LOCALS)
         self.emit_op(ops.RETURN_VALUE)
+
+    def _get_code_flags(self):
+        flags = 0
+        if self.scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
+        return PythonCodeGenerator._get_code_flags(self) | flags
diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -15,6 +15,8 @@
 CO_FUTURE_WITH_STATEMENT = 0x8000
 CO_FUTURE_PRINT_FUNCTION = 0x10000
 CO_FUTURE_UNICODE_LITERALS = 0x20000
+#pypy specific:
+CO_KILL_DOCSTRING = 0x100000
 
 PyCF_SOURCE_IS_UTF8 = 0x0100
 PyCF_DONT_IMPLY_DEDENT = 0x0200
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -42,6 +42,7 @@
         self.has_free = False
         self.child_has_free = False
         self.nested = False
+        self.doc_removable = False
 
     def lookup(self, name):
         """Find the scope of identifier 'name'."""
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -812,6 +812,58 @@
         """
         self.simple_test(source, 'ok', 1)
 
+    def test_remove_docstring(self):
+        source = '"module_docstring"\n' + """if 1:
+        def f1():
+            'docstring'
+        def f2():
+            'docstring'
+            return 'docstring'
+        def f3():
+            'foo'
+            return 'bar'
+        class C1():
+            'docstring'
+        class C2():
+            __doc__ = 'docstring'
+        class C3():
+            field = 'not docstring'
+        class C4():
+            'docstring'
+            field = 'docstring'
+        """
+        code_w = compile_with_astcompiler(source, 'exec', self.space)
+        code_w.remove_docstrings(self.space)
+        dict_w = self.space.newdict();
+        code_w.exec_code(self.space, dict_w, dict_w)
+
+        yield self.check, dict_w, "f1.__doc__", None
+        yield self.check, dict_w, "f2.__doc__", 'docstring'
+        yield self.check, dict_w, "f2()", 'docstring'
+        yield self.check, dict_w, "f3.__doc__", None
+        yield self.check, dict_w, "f3()", 'bar'
+        yield self.check, dict_w, "C1.__doc__", None
+        yield self.check, dict_w, "C2.__doc__", 'docstring'
+        yield self.check, dict_w, "C3.field", 'not docstring'
+        yield self.check, dict_w, "C4.field", 'docstring'
+        yield self.check, dict_w, "C4.__doc__", 'docstring'
+        yield self.check, dict_w, "C4.__doc__", 'docstring'
+        yield self.check, dict_w, "__doc__", None
+
+    def test_assert_skipping(self):
+        space = self.space
+        mod = space.getbuiltinmodule('__pypy__')
+        w_set_debug = space.getattr(mod, space.wrap('set_debug'))
+        space.call_function(w_set_debug, space.w_False)
+
+        source = """if 1:
+        assert False
+        """
+        try:
+            self.run(source)
+        finally:
+            space.call_function(w_set_debug, space.w_True)
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -12,7 +12,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
-    CO_GENERATOR)
+    CO_GENERATOR, CO_KILL_DOCSTRING)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import compute_hash
@@ -218,6 +218,13 @@
                 return w_first
         return space.w_None
 
+    def remove_docstrings(self, space):
+        if self.co_flags & CO_KILL_DOCSTRING:
+            self.co_consts_w[0] = space.w_None
+        for w_co in self.co_consts_w:
+            if isinstance(w_co, PyCode):
+                w_co.remove_docstrings(space)
+
     def _to_code(self):
         """For debugging only."""
         consts = [None] * len(self.co_consts_w)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -876,6 +876,11 @@
         self.popvalue()
         return next_instr
 
+    def JUMP_IF_NOT_DEBUG(self, jumpby, next_instr):
+        if not self.space.sys.debug:
+            next_instr += jumpby
+        return next_instr
+
     def GET_ITER(self, oparg, next_instr):
         w_iterable = self.popvalue()
         w_iterator = self.space.iter(w_iterable)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -58,6 +58,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'dictstrategy'              : 'interp_dict.dictstrategy',
+        'set_debug'                 : 'interp_magic.set_debug',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -101,6 +101,13 @@
 def newlist_hint(space, sizehint):
     return space.newlist_hint(sizehint)
 
+ at unwrap_spec(debug=bool)
+def set_debug(space, debug):
+    space.sys.debug = debug
+    space.setitem(space.builtin.w_dict,
+                  space.wrap('__debug__'),
+                  space.wrap(debug))
+
 @unwrap_spec(estimate=int)
 def add_memory_pressure(estimate):
     rgc.add_memory_pressure(estimate)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -915,6 +915,13 @@
             if not space.is_true(space.sys.get('dont_write_bytecode')):
                 write_compiled_module(space, code_w, cpathname, mode, mtime)
 
+    try:
+        optimize = space.sys.get_flag('optimize')
+    except Exception:
+        optimize = 0
+    if optimize >= 2:
+        code_w.remove_docstrings(space)
+
     update_code_filenames(space, code_w, pathname)
     exec_code_module(space, w_mod, code_w)
 
@@ -1009,6 +1016,13 @@
                               "Bad magic number in %s", cpathname)
     #print "loading pyc file:", cpathname
     code_w = read_compiled_module(space, cpathname, source)
+    try:
+        optimize = space.sys.get_flag('optimize')
+    except Exception:
+        optimize = 0
+    if optimize >= 2:
+        code_w.remove_docstrings(space)
+
     exec_code_module(space, w_mod, code_w)
 
     return w_mod
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -7,7 +7,7 @@
 
 class Module(MixedModule):
     """Sys Builtin Module. """
-    _immutable_fields_ = ["defaultencoding?"]
+    _immutable_fields_ = ["defaultencoding?", "debug?"]
 
     def __init__(self, space, w_name):
         """NOT_RPYTHON""" # because parent __init__ isn't
@@ -18,6 +18,7 @@
         self.w_default_encoder = None
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
+        self.debug = True
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))', 
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -800,6 +800,9 @@
         self.popvalue()
         return next_instr
 
+    def JUMP_IF_NOT_DEBUG(self, target, next_instr):
+        return next_instr
+
     def GET_ITER(self, oparg, next_instr):
         w_iterable = self.popvalue()
         w_iterator = self.space.iter(w_iterable)

From noreply at buildbot.pypy.org  Tue Jun  4 12:13:22 2013
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Tue,  4 Jun 2013 12:13:22 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.0.x: test and fix for getting
	ctypes.byref contents
Message-ID: <20130604101322.20DF01C016D@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin <kostia.lopuhin at gmail.com>
Branch: release-2.0.x
Changeset: r64765:cb8c14632483
Date: 2013-05-20 00:23 +0400
http://bitbucket.org/pypy/pypy/changeset/cb8c14632483/

Log:	test and fix for getting ctypes.byref contents

diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
@@ -108,6 +108,13 @@
 
             py.test.raises(TypeError, delitem, p, 0)
 
+    def test_byref(self):
+        for ct, pt in zip(ctype_types, python_types):
+            i = ct(42)
+            p = byref(i)
+            assert type(p._obj) is ct
+            assert p._obj.value == 42
+
     def test_pointer_to_pointer(self):
         x = c_int(32)
         y = c_int(42)

From noreply at buildbot.pypy.org  Tue Jun  4 13:17:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Jun 2013 13:17:00 +0200 (CEST)
Subject: [pypy-commit] cffi default: Propagate the original OSError,
 which contains information returned by
Message-ID: <20130604111700.3F7141C016D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1267:28f10889b5aa
Date: 2013-06-04 13:16 +0200
http://bitbucket.org/cffi/cffi/changeset/28f10889b5aa/

Log:	Propagate the original OSError, which contains information returned
	by dlerror().

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #

From noreply at buildbot.pypy.org  Tue Jun  4 17:06:51 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue,  4 Jun 2013 17:06:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test for pickling SliceArrays
Message-ID: <20130604150651.D74FE1C016D@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r64766:dd61debfb36a
Date: 2013-06-04 17:04 +0200
http://bitbucket.org/pypy/pypy/changeset/dd61debfb36a/

Log:	Add a test for pickling SliceArrays

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1801,6 +1801,13 @@
         pickled_data = dumps(a)
         assert (loads(pickled_data) == a).all()
 
+    def test_pickle_slice(self):
+        from cPickle import loads, dumps
+        import numpypy as numpy
+
+        a = numpy.arange(10.)[::2]
+        assert (loads(dumps(a)) == a).all()
+
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import numpypy

From noreply at buildbot.pypy.org  Tue Jun  4 17:41:26 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:26 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: a branch where to implement a fast
 json module in rpython
Message-ID: <20130604154126.CE1781C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64767:9b78f614a0f3
Date: 2013-06-04 14:39 +0200
http://bitbucket.org/pypy/pypy/changeset/9b78f614a0f3/

Log:	a branch where to implement a fast json module in rpython


From noreply at buildbot.pypy.org  Tue Jun  4 17:41:28 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:28 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: start to implement the json decoder:
 decode only strings so far
Message-ID: <20130604154128.166E51C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64768:6ec983152d16
Date: 2013-06-04 15:28 +0200
http://bitbucket.org/pypy/pypy/changeset/6ec983152d16/

Log:	start to implement the json decoder: decode only strings so far

diff --git a/pypy/module/_fastjson/__init__.py b/pypy/module/_fastjson/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_fastjson/__init__.py
@@ -0,0 +1,10 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    """fast json implementation"""
+
+    appleveldefs = {}
+
+    interpleveldefs = {
+        'loads' : 'interp_decoder.loads',
+        }
diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -0,0 +1,65 @@
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter import unicodehelper
+
+def is_whitespace(ch):
+    return ch == ' ' or ch == '\t' or ch == '\r' or ch == '\n'
+
+TYPE_INVALID = 0
+TYPE_STRING = 0
+
+class JSONDecoder(object):
+    def __init__(self, space, s):
+        self.space = space
+        self.s = s
+        self.i = 0
+        self.last_type = TYPE_INVALID
+
+    def eof(self):
+        return self.i == len(self.s)
+
+    def peek(self):
+        return self.s[self.i]
+
+    def next(self):
+        ch = self.peek()
+        self.i += 1
+        return ch
+
+    def skip_whitespace(self):
+        while not self.eof():
+            ch = self.peek()
+            if is_whitespace(ch):
+                self.next()
+            else:
+                break
+
+    def decode_any(self):
+        self.skip_whitespace()
+        ch = self.peek()
+        if ch == '"':
+            return self.decode_string()
+        else:
+            assert False, 'Unkown char: %s' % ch
+
+    def decode_string(self):
+        self.next()
+        start = self.i
+        while True:
+            ch = self.next()
+            if ch == '"':
+                end = self.i-1
+                assert end > 0
+                content = self.s[start:end]
+                self.last_type = TYPE_STRING
+                return self.space.wrap(unicodehelper.decode_utf8(self.space, content))
+            elif ch == '\\':
+                raise Exception("escaped strings not supported yet")
+
+
+ at unwrap_spec(s=str)
+def loads(space, s):
+    decoder = JSONDecoder(space, s)
+    return decoder.decode_any()
+
+
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+import py
+from pypy.module._fastjson.interp_decoder import JSONDecoder
+
+def test_skip_whitespace():
+    dec = JSONDecoder('fake space', '   hello   ')
+    assert dec.i == 0
+    dec.skip_whitespace()
+    assert dec.next() == 'h'
+    assert dec.next() == 'e'
+    assert dec.next() == 'l'
+    assert dec.next() == 'l'
+    assert dec.next() == 'o'
+    dec.skip_whitespace()
+    assert dec.eof()
+
+    
+
+class AppTest(object):
+    spaceconfig = {"objspace.usemodules._fastjson": True}
+
+    def test_load_string(self):
+        import _fastjson
+        res = _fastjson.loads('"hello"')
+        assert res == u'hello'
+        assert type(res) is unicode
+
+    def test_load_string_utf8(self):
+        import _fastjson
+        s = u'&#224;&#232;&#236;&#242;&#249;'
+        res = _fastjson.loads('"%s"' % s.encode('utf-8'))
+        assert res == s

From noreply at buildbot.pypy.org  Tue Jun  4 17:41:29 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:29 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: skip the whitespace at the end and
 complain if there are extra chars
Message-ID: <20130604154129.667011C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64769:51df42a6fd1c
Date: 2013-06-04 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/51df42a6fd1c/

Log:	skip the whitespace at the end and complain if there are extra chars

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import unicodehelper
 
@@ -60,6 +60,10 @@
 @unwrap_spec(s=str)
 def loads(space, s):
     decoder = JSONDecoder(space, s)
-    return decoder.decode_any()
-
-
+    w_res = decoder.decode_any()
+    decoder.skip_whitespace()
+    if not decoder.eof():
+        start = decoder.i
+        end = len(decoder.s)
+        raise operationerrfmt(space.w_ValueError, "Extra data: char %d - %d", start, end)
+    return w_res
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -19,14 +19,21 @@
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._fastjson": True}
 
-    def test_load_string(self):
+    def test_decode_string(self):
         import _fastjson
         res = _fastjson.loads('"hello"')
         assert res == u'hello'
         assert type(res) is unicode
 
-    def test_load_string_utf8(self):
+    def test_decode_string_utf8(self):
         import _fastjson
         s = u'&#224;&#232;&#236;&#242;&#249;'
         res = _fastjson.loads('"%s"' % s.encode('utf-8'))
         assert res == s
+
+    def test_skip_whitespace(self):
+        import _fastjson
+        s = '   "hello"   '
+        assert _fastjson.loads(s) == u'hello'
+        s = '   "hello"   extra'
+        raises(ValueError, "_fastjson.loads(s)")

From noreply at buildbot.pypy.org  Tue Jun  4 17:41:30 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:30 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add support for escape sequences and
 complain if the string is non terminated
Message-ID: <20130604154130.9FFAE1C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64770:9e8497c271b6
Date: 2013-06-04 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/9e8497c271b6/

Log:	add support for escape sequences and complain if the string is non
	terminated

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -1,3 +1,4 @@
+from rpython.rlib.rstring import StringBuilder
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import unicodehelper
@@ -26,6 +27,9 @@
         self.i += 1
         return ch
 
+    def unget(self):
+        self.i -= 1
+
     def skip_whitespace(self):
         while not self.eof():
             ch = self.peek()
@@ -42,21 +46,65 @@
         else:
             assert False, 'Unkown char: %s' % ch
 
+    def getslice(self, start, end):
+        assert end > 0
+        return self.s[start:end]
+
     def decode_string(self):
         self.next()
         start = self.i
-        while True:
+        while not self.eof():
+            # this loop is a fast path for strings which do not contain escape
+            # characters
             ch = self.next()
             if ch == '"':
-                end = self.i-1
-                assert end > 0
-                content = self.s[start:end]
+                content_utf8 = self.getslice(start, self.i-1)
+                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
                 self.last_type = TYPE_STRING
-                return self.space.wrap(unicodehelper.decode_utf8(self.space, content))
+                return self.space.wrap(content_unicode)
             elif ch == '\\':
-                raise Exception("escaped strings not supported yet")
+                content_so_far = self.getslice(start, self.i-1)
+                self.unget()
+                return self.decode_string_escaped(start, content_so_far)
+        raise operationerrfmt(self.space.w_ValueError,
+                              "Unterminated string starting at char %d", start)
 
 
+    def decode_string_escaped(self, start, content_so_far):
+        builder = StringBuilder(len(content_so_far)*2) # just an estimate
+        builder.append(content_so_far)
+        while not self.eof():
+            ch = self.next()
+            if ch == '"':
+                content_utf8 = builder.build()
+                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
+                self.last_type = TYPE_STRING
+                return self.space.wrap(content_unicode)
+            elif ch == '\\':
+                newchar = self.decode_escape_sequence()
+                builder.append_multiple_char(newchar, 1) # we should implement append_char
+            else:
+                builder.append_multiple_char(newchar, 1)
+            
+        raise operationerrfmt(self.space.w_ValueError,
+                              "Unterminated string starting at char %d", start)
+
+    def decode_escape_sequence(self):
+        ch = self.next()
+        if ch == '\\':  return '\\'
+        elif ch == '"': return '"'
+        elif ch == '/': return '/'
+        elif ch == 'b': return '\b'
+        elif ch == 'f': return '\f'
+        elif ch == 'n': return '\n'
+        elif ch == 'r': return '\r'
+        elif ch == 't': return '\t'
+        elif ch == 'u':
+            assert False, 'not implemented yet'
+        else:
+            raise operationerrfmt(self.space.w_ValueError,
+                                  "Invalid \\escape: %s (char %d)", ch, self.i-1)
+
 @unwrap_spec(s=str)
 def loads(space, s):
     decoder = JSONDecoder(space, s)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -37,3 +37,19 @@
         assert _fastjson.loads(s) == u'hello'
         s = '   "hello"   extra'
         raises(ValueError, "_fastjson.loads(s)")
+
+    def test_unterminated_string(self):
+        import _fastjson
+        s = '"hello' # missing the trailing "
+        raises(ValueError, "_fastjson.loads(s)")
+
+    def test_escape_sequence(self):
+        import _fastjson
+        assert _fastjson.loads(r'"\\"') == u'\\'
+        assert _fastjson.loads(r'"\""') == u'"'
+        assert _fastjson.loads(r'"\/"') == u'/'       
+        assert _fastjson.loads(r'"\b"') == u'\b'
+        assert _fastjson.loads(r'"\f"') == u'\f'
+        assert _fastjson.loads(r'"\n"') == u'\n'
+        assert _fastjson.loads(r'"\r"') == u'\r'
+        assert _fastjson.loads(r'"\t"') == u'\t'

From noreply at buildbot.pypy.org  Tue Jun  4 17:41:31 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:31 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: what is not tested is broken: test&fix
Message-ID: <20130604154131.E4EF31C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64771:d04ad613ac6b
Date: 2013-06-04 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/d04ad613ac6b/

Log:	what is not tested is broken: test&fix

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -81,10 +81,10 @@
                 self.last_type = TYPE_STRING
                 return self.space.wrap(content_unicode)
             elif ch == '\\':
-                newchar = self.decode_escape_sequence()
-                builder.append_multiple_char(newchar, 1) # we should implement append_char
+                ch = self.decode_escape_sequence()
+                builder.append_multiple_char(ch, 1) # we should implement append_char
             else:
-                builder.append_multiple_char(newchar, 1)
+                builder.append_multiple_char(ch, 1)
             
         raise operationerrfmt(self.space.w_ValueError,
                               "Unterminated string starting at char %d", start)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -53,3 +53,8 @@
         assert _fastjson.loads(r'"\n"') == u'\n'
         assert _fastjson.loads(r'"\r"') == u'\r'
         assert _fastjson.loads(r'"\t"') == u'\t'
+
+    def test_escape_sequence_in_the_middle(self):
+        import _fastjson
+        s = r'"hello\nworld"'
+        assert _fastjson.loads(s) == "hello\nworld"

From noreply at buildbot.pypy.org  Tue Jun  4 17:41:33 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:33 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: a passing test
Message-ID: <20130604154133.324811C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64772:7fddf7d2a349
Date: 2013-06-04 16:26 +0200
http://bitbucket.org/pypy/pypy/changeset/7fddf7d2a349/

Log:	a passing test

diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -58,3 +58,9 @@
         import _fastjson
         s = r'"hello\nworld"'
         assert _fastjson.loads(s) == "hello\nworld"
+
+    def test_unterminated_string_after_escape_sequence(self):
+        import _fastjson
+        s = r'"hello\nworld' # missing the trailing "
+        raises(ValueError, "_fastjson.loads(s)")
+        

From noreply at buildbot.pypy.org  Tue Jun  4 17:41:34 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:34 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: implement parsing of \uXXXX escapes
Message-ID: <20130604154134.8CE8C1C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64773:752e9547c85c
Date: 2013-06-04 17:15 +0200
http://bitbucket.org/pypy/pypy/changeset/752e9547c85c/

Log:	implement parsing of \uXXXX escapes

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -81,30 +81,45 @@
                 self.last_type = TYPE_STRING
                 return self.space.wrap(content_unicode)
             elif ch == '\\':
-                ch = self.decode_escape_sequence()
+                self.decode_escape_sequence(builder)
+            else:
                 builder.append_multiple_char(ch, 1) # we should implement append_char
-            else:
-                builder.append_multiple_char(ch, 1)
             
         raise operationerrfmt(self.space.w_ValueError,
                               "Unterminated string starting at char %d", start)
 
-    def decode_escape_sequence(self):
+    def decode_escape_sequence(self, builder):
+        put = builder.append_multiple_char
         ch = self.next()
-        if ch == '\\':  return '\\'
-        elif ch == '"': return '"'
-        elif ch == '/': return '/'
-        elif ch == 'b': return '\b'
-        elif ch == 'f': return '\f'
-        elif ch == 'n': return '\n'
-        elif ch == 'r': return '\r'
-        elif ch == 't': return '\t'
+        if ch == '\\':  put('\\', 1)
+        elif ch == '"': put('"' , 1)
+        elif ch == '/': put('/' , 1)
+        elif ch == 'b': put('\b', 1)
+        elif ch == 'f': put('\f', 1)
+        elif ch == 'n': put('\n', 1)
+        elif ch == 'r': put('\r', 1)
+        elif ch == 't': put('\t', 1)
         elif ch == 'u':
-            assert False, 'not implemented yet'
+            return self.decode_escape_sequence_unicode(builder)
         else:
             raise operationerrfmt(self.space.w_ValueError,
                                   "Invalid \\escape: %s (char %d)", ch, self.i-1)
 
+    def decode_escape_sequence_unicode(self, builder):
+        # at this point we are just after the 'u' of the \u1234 sequence.
+        hexdigits = self.getslice(self.i, self.i+4)
+        self.i += 4
+        try:
+            uchr = unichr(int(hexdigits, 16))
+        except ValueError:
+            raise operationerrfmt(self.space.w_ValueError,
+                                  "Invalid \uXXXX escape (char %d)", self.i-1)
+        #
+        utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
+        builder.append(utf8_ch)
+
+
+
 @unwrap_spec(s=str)
 def loads(space, s):
     decoder = JSONDecoder(space, s)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -64,3 +64,7 @@
         s = r'"hello\nworld' # missing the trailing "
         raises(ValueError, "_fastjson.loads(s)")
         
+    def test_escape_sequence_unicode(self):
+        import _fastjson
+        s = r'"\u1234"'
+        assert _fastjson.loads(s) == u'\u1234'

From noreply at buildbot.pypy.org  Tue Jun  4 17:41:35 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:41:35 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: implement decoding of objects
Message-ID: <20130604154135.DB8001C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64774:0c2d7c11a454
Date: 2013-06-04 17:40 +0200
http://bitbucket.org/pypy/pypy/changeset/0c2d7c11a454/

Log:	implement decoding of objects

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -6,15 +6,15 @@
 def is_whitespace(ch):
     return ch == ' ' or ch == '\t' or ch == '\r' or ch == '\n'
 
-TYPE_INVALID = 0
-TYPE_STRING = 0
+TYPE_UNKNOWN = 0
+TYPE_STRING = 1
 
 class JSONDecoder(object):
     def __init__(self, space, s):
         self.space = space
         self.s = s
         self.i = 0
-        self.last_type = TYPE_INVALID
+        self.last_type = TYPE_UNKNOWN
 
     def eof(self):
         return self.i == len(self.s)
@@ -30,6 +30,10 @@
     def unget(self):
         self.i -= 1
 
+    def getslice(self, start, end):
+        assert end > 0
+        return self.s[start:end]
+
     def skip_whitespace(self):
         while not self.eof():
             ch = self.peek()
@@ -37,21 +41,56 @@
                 self.next()
             else:
                 break
+            
+    def _raise(self, msg, *args):
+        raise operationerrfmt(self.w_ValueError, msg, *args)
 
     def decode_any(self):
         self.skip_whitespace()
-        ch = self.peek()
+        ch = self.next()
         if ch == '"':
             return self.decode_string()
+        elif ch == '{':
+            return self.decode_object()
         else:
             assert False, 'Unkown char: %s' % ch
 
-    def getslice(self, start, end):
-        assert end > 0
-        return self.s[start:end]
+    def decode_object(self):
+        start = self.i
+        w_dict = self.space.newdict()
+        while not self.eof():
+            ch = self.peek()
+            if ch == '}':
+                self.next()
+                return w_dict
+            #
+            # parse a key: value
+            self.last_type = TYPE_UNKNOWN
+            w_name = self.decode_any()
+            if self.last_type != TYPE_STRING:
+                self._raise("Key name must be string")
+            self.skip_whitespace()
+            ch = self.next()
+            if ch != ':':
+                self._raise("No ':' found at char %d", self.i)
+            self.skip_whitespace()
+            #
+            w_value = self.decode_any()
+            self.space.setitem(w_dict, w_name, w_value)
+            self.skip_whitespace()
+            ch = self.next()
+            if ch == '}':
+                return w_dict
+            elif ch == ',':
+                pass
+            else:
+                self._raise("Unexpected '%s' when decoding object (char %d)",
+                            ch, self.i)
+        self._raise("Unterminated object starting at char %d", start)
+
+
 
     def decode_string(self):
-        self.next()
         start = self.i
         while not self.eof():
             # this loop is a fast path for strings which do not contain escape
@@ -66,8 +105,7 @@
                 content_so_far = self.getslice(start, self.i-1)
                 self.unget()
                 return self.decode_string_escaped(start, content_so_far)
-        raise operationerrfmt(self.space.w_ValueError,
-                              "Unterminated string starting at char %d", start)
+        self._raise("Unterminated string starting at char %d", start)
 
 
     def decode_string_escaped(self, start, content_so_far):
@@ -84,9 +122,8 @@
                 self.decode_escape_sequence(builder)
             else:
                 builder.append_multiple_char(ch, 1) # we should implement append_char
-            
-        raise operationerrfmt(self.space.w_ValueError,
-                              "Unterminated string starting at char %d", start)
+        #
+        self._raise("Unterminated string starting at char %d", start)
 
     def decode_escape_sequence(self, builder):
         put = builder.append_multiple_char
@@ -102,8 +139,7 @@
         elif ch == 'u':
             return self.decode_escape_sequence_unicode(builder)
         else:
-            raise operationerrfmt(self.space.w_ValueError,
-                                  "Invalid \\escape: %s (char %d)", ch, self.i-1)
+            self._raise("Invalid \\escape: %s (char %d)", ch, self.i-1)
 
     def decode_escape_sequence_unicode(self, builder):
         # at this point we are just after the 'u' of the \u1234 sequence.
@@ -112,8 +148,7 @@
         try:
             uchr = unichr(int(hexdigits, 16))
         except ValueError:
-            raise operationerrfmt(self.space.w_ValueError,
-                                  "Invalid \uXXXX escape (char %d)", self.i-1)
+            self._raise("Invalid \uXXXX escape (char %d)", self.i-1)
         #
         utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
         builder.append(utf8_ch)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -68,3 +68,15 @@
         import _fastjson
         s = r'"\u1234"'
         assert _fastjson.loads(s) == u'\u1234'
+
+    def test_decode_object(self):
+        import _fastjson
+        assert _fastjson.loads('{}') == {}
+        #
+        s = '{"hello": "world", "aaa": "bbb"}'
+        assert _fastjson.loads(s) == {'hello': 'world',
+                                      'aaa': 'bbb'}
+
+    def test_decode_object_nonstring_key(self):
+        pass # write me when we have numbers
+    

From noreply at buildbot.pypy.org  Tue Jun  4 17:42:05 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 17:42:05 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: enable _fastjson by default
Message-ID: <20130604154205.1F2131C145C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64775:7cc8032c9144
Date: 2013-06-04 17:41 +0200
http://bitbucket.org/pypy/pypy/changeset/7cc8032c9144/

Log:	enable _fastjson by default

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -35,7 +35,7 @@
      "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
      "binascii", "_multiprocessing", '_warnings',
      "_collections", "_multibytecodec", "micronumpy", "_ffi",
-     "_continuation", "_cffi_backend", "_csv", "cppyy"]
+     "_continuation", "_cffi_backend", "_csv", "cppyy", "_fastjson"]
 ))
 
 translation_modules = default_modules.copy()

From noreply at buildbot.pypy.org  Tue Jun  4 19:40:28 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 19:40:28 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: rpython fix and typo
Message-ID: <20130604174028.3B11E1C0651@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64776:08d1da984920
Date: 2013-06-04 17:57 +0200
http://bitbucket.org/pypy/pypy/changeset/08d1da984920/

Log:	rpython fix and typo

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -28,7 +28,9 @@
         return ch
 
     def unget(self):
-        self.i -= 1
+        i2 = self.i - 1
+        assert i2 > 0 # so that we can use self.i as slice start
+        self.i = i2
 
     def getslice(self, start, end):
         assert end > 0
@@ -43,7 +45,7 @@
                 break
             
     def _raise(self, msg, *args):
-        raise operationerrfmt(self.w_ValueError, msg, *args)
+        raise operationerrfmt(self.space.w_ValueError, msg, *args)
 
     def decode_any(self):
         self.skip_whitespace()

From noreply at buildbot.pypy.org  Tue Jun  4 19:40:29 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 19:40:29 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: another rpython fix
Message-ID: <20130604174029.7202C1C0651@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64777:ced1fbce0894
Date: 2013-06-04 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/ced1fbce0894/

Log:	another rpython fix

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import unicodehelper
@@ -43,7 +44,8 @@
                 self.next()
             else:
                 break
-            
+
+    @specialize.arg(1)
     def _raise(self, msg, *args):
         raise operationerrfmt(self.space.w_ValueError, msg, *args)
 

From noreply at buildbot.pypy.org  Tue Jun  4 19:40:30 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue,  4 Jun 2013 19:40:30 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: another rpython fix
Message-ID: <20130604174030.D77871C0651@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64778:9777a0934c56
Date: 2013-06-04 18:11 +0200
http://bitbucket.org/pypy/pypy/changeset/9777a0934c56/

Log:	another rpython fix

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -72,7 +72,7 @@
             self.last_type = TYPE_UNKNOWN
             w_name = self.decode_any()
             if self.last_type != TYPE_STRING:
-                self._raise("Key name must be string")
+                self._raise("Key name must be string for object starting at char %d", start)
             self.skip_whitespace()
             ch = self.next()
             if ch != ':':

From noreply at buildbot.pypy.org  Tue Jun  4 20:44:29 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue,  4 Jun 2013 20:44:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update test
Message-ID: <20130604184429.30FB01C016D@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r64779:b11b29c91555
Date: 2013-06-04 17:50 +0200
http://bitbucket.org/pypy/pypy/changeset/b11b29c91555/

Log:	Update test

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1805,7 +1805,7 @@
         from cPickle import loads, dumps
         import numpypy as numpy
 
-        a = numpy.arange(10.)[::2]
+        a = numpy.arange(10.).reshape((5, 2))[::2]
         assert (loads(dumps(a)) == a).all()
 
 class AppTestMultiDim(BaseNumpyAppTest):

From noreply at buildbot.pypy.org  Tue Jun  4 20:44:30 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue,  4 Jun 2013 20:44:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Implement pickling for slices
Message-ID: <20130604184430.924391C145C@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r64780:d8b79f042e37
Date: 2013-06-04 20:43 +0200
http://bitbucket.org/pypy/pypy/changeset/d8b79f042e37/

Log:	Implement pickling for slices

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -13,6 +13,9 @@
 from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage
 from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.rstring import StringBuilder
+
 
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
@@ -42,7 +45,23 @@
     return func_with_new_name(new, name + "_box_new"), staticmethod(_get_dtype), func_with_new_name(descr_reduce, "descr_reduce")
 
 
-class PrimitiveBox(object):
+class Box(object):
+    _mixin_ = True
+
+    def reduce(self, space):
+        from rpython.rlib.rstring import StringBuilder
+        from rpython.rtyper.lltypesystem import rffi, lltype
+
+        numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(numpypy, MixedModule)
+        multiarray = numpypy.get("multiarray")
+        assert isinstance(multiarray, MixedModule)
+        scalar = multiarray.get("scalar")
+
+        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
+        return ret
+
+class PrimitiveBox(Box):
     _mixin_ = True
 
     def __init__(self, value):
@@ -54,27 +73,19 @@
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self.value)
 
-    def reduce(self, space):
-        from rpython.rlib.rstring import StringBuilder
-        from rpython.rtyper.lltypesystem import rffi, lltype
-
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
-        assert isinstance(multiarray, MixedModule)
-        scalar = multiarray.get("scalar")
-
+    def raw_str(self):
         value = lltype.malloc(rffi.CArray(lltype.typeOf(self.value)), 1, flavor="raw")
         value[0] = self.value
 
         builder = StringBuilder()
         builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.value)))
+        ret = builder.build()
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(builder.build())])])
         lltype.free(value, flavor="raw")
+
         return ret
 
-class ComplexBox(object):
+class ComplexBox(Box):
     _mixin_ = True
 
     def __init__(self, real, imag=0.):
@@ -90,25 +101,17 @@
     def convert_imag_to(self, dtype):
         return dtype.box(self.imag)
 
-    def reduce(self, space):
-        from rpython.rlib.rstring import StringBuilder
-        from rpython.rtyper.lltypesystem import rffi, lltype
-
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
-        assert isinstance(multiarray, MixedModule)
-        scalar = multiarray.get("scalar")
-
+    def raw_str(self):
         value = lltype.malloc(rffi.CArray(lltype.typeOf(self.real)), 2, flavor="raw")
         value[0] = self.real
         value[1] = self.imag
 
         builder = StringBuilder()
         builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.real)) * 2)
+        ret = builder.build()
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(builder.build())])])
         lltype.free(value, flavor="raw")
+
         return ret
 
 class W_GenericBox(W_Root):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -786,6 +786,7 @@
         from rpython.rtyper.lltypesystem import rffi
         from rpython.rlib.rstring import StringBuilder
         from pypy.interpreter.mixedmodule import MixedModule
+        from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
 
         numpypy = space.getbuiltinmodule("_numpypy")
         assert isinstance(numpypy, MixedModule)
@@ -796,7 +797,14 @@
         parameters = space.newtuple([space.gettypefor(W_NDimArray), space.newtuple([space.wrap(0)]), space.wrap("b")])
 
         builder = StringBuilder()
-        builder.append_charpsize(self.implementation.get_storage(), self.implementation.get_storage_size())
+        if isinstance(self.implementation, SliceArray):
+            iter = self.implementation.create_iter()
+            while not iter.done():
+                box = iter.getitem()
+                builder.append(box.raw_str())
+                iter.next()
+        else:
+            builder.append_charpsize(self.implementation.get_storage(), self.implementation.get_storage_size())
 
         state = space.newtuple([
                 space.wrap(1),      # version

From noreply at buildbot.pypy.org  Tue Jun  4 22:28:21 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Tue,  4 Jun 2013 22:28:21 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: Implement swapcase() method.
Message-ID: <20130604202821.05A761C1401@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64783:6a4af8ab46ec
Date: 2013-06-04 21:17 +0200
http://bitbucket.org/pypy/pypy/changeset/6a4af8ab46ec/

Log:	Implement swapcase() method.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -96,6 +96,27 @@
 
     _builder = StringBuilder
 
+    def _isupper(self, ch):
+        return ch.isupper()
+
+    def _islower(self, ch):
+        return ch.islower()
+
+    def _istitle(self, ch):
+        return ch.istitle()
+
+    def _isspace(self, ch):
+        return ch.isspace()
+
+    def _isalpha(self, ch):
+        return ch.isalpha()
+
+    def _isalnum(self, ch):
+        return ch.isalnum()
+
+    def _isdigit(self, ch):
+        return ch.isdigit()
+
     def _upper(self, ch):
         if ch.islower():
             o = ord(ch) - 32
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -9,12 +9,6 @@
 from rpython.rlib.rstring import split
 
 
-_isspace = lambda c: c.isspace()
-_isdigit = lambda c: c.isdigit()
-_isalpha = lambda c: c.isalpha()
-_isalnum = lambda c: c.isalnum()
-
-
 class StringMethods(object):
     _mixin_ = True
 
@@ -296,13 +290,13 @@
         return space.w_True
 
     def descr_isalnum(self, space):
-        return self._is_generic(space, _isalnum)
+        return self._is_generic(space, self._isalnum)
 
     def descr_isalpha(self, space):
-        return self._is_generic(space, _isalpha)
+        return self._is_generic(space, self._isalpha)
 
     def descr_isdigit(self, space):
-        return self._is_generic(space, _isdigit)
+        return self._is_generic(space, self._isdigit)
 
     def descr_islower(self, space):
         v = self._value
@@ -318,7 +312,7 @@
         return space.newbool(cased)
 
     def descr_isspace(self, space):
-        return self._is_generic(space, _isspace)
+        return self._is_generic(space, self._isspace)
 
     def descr_istitle(self, space):
         input = self._value
@@ -769,19 +763,14 @@
         return self._strip(space, w_chars, left=0, right=1)
 
     def descr_swapcase(self, space):
-        # XXX just to pass the test
-        return space.wrap(self._val().swapcase())
-
-        selfvalue = self._value
+        selfvalue = self._val()
         builder = self._builder(len(selfvalue))
         for i in range(len(selfvalue)):
             ch = selfvalue[i]
-            if ch.isupper():
-                o = ord(ch) + 32
-                builder.append(chr(o))
-            elif ch.islower():
-                o = ord(ch) - 32
-                builder.append(chr(o))
+            if self._isupper(ch):
+                builder.append(self._lower(ch))
+            elif self._islower(ch):
+                builder.append(self._upper(ch))
             else:
                 builder.append(ch)
         return space.wrap(builder.build())
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -89,6 +89,27 @@
 
     _builder = UnicodeBuilder
 
+    def _isupper(self, ch):
+        return ch.isupper()
+
+    def _islower(self, ch):
+        return ch.islower()
+
+    def _istitle(self, ch):
+        return ch.istitle()
+
+    def _isspace(self, ch):
+        return ch.isspace()
+
+    def _isalpha(self, ch):
+        return ch.isalpha()
+
+    def _isalnum(self, ch):
+        return ch.isalnum()
+
+    def _isdigit(self, ch):
+        return ch.isdigit()
+
     def _upper(self, ch):
         return unichr(unicodedb.toupper(ord(ch)))
 

From noreply at buildbot.pypy.org  Tue Jun  4 22:28:19 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Tue,  4 Jun 2013 22:28:19 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: Implement capitalize()
	method.
Message-ID: <20130604202819.A89551C12FE@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64782:cfd9859a607d
Date: 2013-06-04 21:03 +0200
http://bitbucket.org/pypy/pypy/changeset/cfd9859a607d/

Log:	Implement capitalize() method.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -96,6 +96,20 @@
 
     _builder = StringBuilder
 
+    def _upper(self, ch):
+        if ch.islower():
+            o = ord(ch) - 32
+            return chr(o)
+        else:
+            return ch
+
+    def _lower(self, ch):
+        if ch.isupper():
+            o = ord(ch) + 32
+            return chr(o)
+        else:
+            return ch
+
     @staticmethod
     @unwrap_spec(w_object = WrappedDefault(""))
     def descr_new(space, w_stringtype, w_object):
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -27,20 +27,6 @@
     def _val(self):
         raise NotImplementedError
 
-    def _upper(self, ch):
-        if ch.islower():
-            o = ord(ch) - 32
-            return chr(o)
-        else:
-            return ch
-
-    def _lower(self, ch):
-        if ch.isupper():
-            o = ord(ch) + 32
-            return chr(o)
-        else:
-            return ch
-
     def _sliced(self, space, s, start, stop, orig_obj):
         assert start >= 0
         assert stop >= 0
@@ -174,27 +160,14 @@
             return self._sliced(space, selfvalue, start, stop, self)
 
     def descr_capitalize(self, space):
-        # XXX just to pass the test
-        return self._new(self._val().capitalize())
+        value = self._val()
+        if len(value) == 0:
+            return self.EMPTY
 
-        input = self._value
-        builder = self._builder(len(input))
-        if len(input) > 0:
-            ch = input[0]
-            if ch.islower():
-                o = ord(ch) - 32
-                builder.append(chr(o))
-            else:
-                builder.append(ch)
-
-            for i in range(1, len(input)):
-                ch = input[i]
-                if ch.isupper():
-                    o = ord(ch) + 32
-                    builder.append(chr(o))
-                else:
-                    builder.append(ch)
-
+        builder = self._builder(len(value))
+        builder.append(self._upper(value[0]))
+        for i in range(1, len(value)):
+            builder.append(self._lower(value[i]))
         return space.wrap(builder.build())
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -89,6 +89,12 @@
 
     _builder = UnicodeBuilder
 
+    def _upper(self, ch):
+        return unichr(unicodedb.toupper(ord(ch)))
+
+    def _lower(self, ch):
+        return unichr(unicodedb.tolower(ord(ch)))
+
     def descr_repr(self, space):
         chars = self._value
         size = len(chars)

From noreply at buildbot.pypy.org  Tue Jun  4 22:28:18 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Tue,  4 Jun 2013 22:28:18 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: hg merge default
Message-ID: <20130604202818.488471C016D@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64781:08c45f23c86d
Date: 2013-06-04 20:41 +0200
http://bitbucket.org/pypy/pypy/changeset/08c45f23c86d/

Log:	hg merge default

diff too long, truncating to 2000 out of 7083 lines

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -58,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -80,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -151,6 +151,8 @@
     'DEBUG': DEBUG,
     'NOTSET': NOTSET,
 }
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Distance to target address
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,16 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    try:
-                        self._sock._decref_socketios()
-                    except AttributeError:
-                        pass  # bah, someone built a _fileobject manually
-                              # with some unexpected replacement of the
-                              # _socketobject class
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_pypy_testcapi.py
copy from lib_pypy/_testcapi.py
copy to lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,14 +1,20 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
     """
     thisdir = os.path.dirname(__file__)
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -19,13 +25,13 @@
         ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
+    res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
                            extra_preargs=ccflags)
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
+    output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
@@ -37,7 +43,7 @@
             library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
+                        '/EXPORT:init' + modulename]
     else:
         libraries = []
         extra_ldargs = []
@@ -49,14 +55,7 @@
         libraries=libraries,
         extra_preargs=extra_ldargs)
 
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-try:
-    import cpyext
-except ImportError:
-    raise ImportError("No module named '_testcapi'")
-else:
-    compile_shared()
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,62 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
 try:
     import cpyext
 except ImportError:
     raise ImportError("No module named '_testcapi'")
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -156,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -701,7 +704,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
     # ----------
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -74,6 +74,9 @@
         class FFILibrary(types.ModuleType):
             _cffi_generic_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir
         library = FFILibrary("")
         #
         # finally, call the loaded_gen_xxx() functions.  This will set
@@ -168,21 +171,22 @@
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
-            if any(isinstance(type, model.StructOrUnion) for type in tp.args):
+            if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
-                for i, type in enumerate(tp.args):
-                    if isinstance(type, model.StructOrUnion):
-                        type = model.PointerType(type)
-                        indirections.append((i, type))
-                    indirect_args.append(type)
+                for i, typ in enumerate(tp.args):
+                    if isinstance(typ, model.StructOrUnion):
+                        typ = model.PointerType(typ)
+                        indirections.append((i, typ))
+                    indirect_args.append(typ)
                 tp = model.FunctionPtrType(tuple(indirect_args),
                                            tp.result, tp.ellipsis)
             BFunc = self.ffi._get_cached_btype(tp)
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
-            for i, type in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, type)
+            for i, typ in indirections:
+                newfunction = self._make_struct_wrapper(newfunction, i, typ)
         setattr(library, name, newfunction)
+        type(library)._cffi_dir.append(name)
 
     def _make_struct_wrapper(self, oldfunc, i, tp):
         backend = self.ffi._backend
@@ -390,6 +394,7 @@
         is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
         value = self._load_constant(is_int, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # enums
@@ -437,6 +442,7 @@
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
             setattr(library, enumerator, enumvalue)
+            type(library)._cffi_dir.append(enumerator)
 
     # ----------
     # macros: for now only for integers
@@ -450,6 +456,7 @@
     def _loaded_gen_macro(self, tp, name, module, library):
         value = self._load_constant(True, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # global variables
@@ -475,6 +482,7 @@
                 BArray = self.ffi._get_cached_btype(tp)
                 value = self.ffi.cast(BArray, value)
             setattr(library, name, value)
+            type(library)._cffi_dir.append(name)
             return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
@@ -486,7 +494,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
 cffimod_header = r'''
 #include <stdio.h>
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
new file mode 100644
--- /dev/null
+++ b/lib_pypy/greenlet.egg-info
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: greenlet
+Version: 0.4.0
+Summary: Lightweight in-process concurrent programming
+Home-page: https://github.com/python-greenlet/greenlet
+Author: Ralf Schmitt (for CPython), PyPy team
+Author-email: pypy-dev at python.org
+License: MIT License
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -32,11 +32,10 @@
      "rctime" , "select", "zipimport", "_lsprof",
      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
-     "thread", "itertools", "pyexpat", "_ssl", "array",
+     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
      "binascii", "_multiprocessing", '_warnings',
      "_collections", "_multibytecodec", "micronumpy", "_ffi",
-     "_continuation", "_cffi_backend", "_csv"] # "cpyext", "cppyy"]
-# disabled until problems are fixed
+     "_continuation", "_cffi_backend", "_csv", "cppyy"]
 ))
 
 translation_modules = default_modules.copy()
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,7 @@
+
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,7 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/doc/rffi.rst b/pypy/doc/rffi.rst
--- a/pypy/doc/rffi.rst
+++ b/pypy/doc/rffi.rst
@@ -5,7 +5,7 @@
 Purpose
 -------
 
-This document describes an FFI for RPython language, concentrating
+This document describes an FFI for the RPython language, concentrating
 on low-level backends like C. It describes
 how to declare and call low-level (C) functions from RPython level.
 
@@ -50,7 +50,7 @@
 ------
 
 In rffi_ there are various declared types for C-structures, like CCHARP
-(char*), SIZE_T (size_t) and others. refer to file for details. 
+(char*), SIZE_T (size_t) and others. Refer to file for details. 
 Instances of non-primitive types must be alloced by hand, with call 
 to lltype.malloc, and freed by lltype.free both with keyword argument 
 flavor='raw'. There are several helpers like string -> char*
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,19 @@
 
 .. branch: remove-iter-smm
 Remove multi-methods on iterators
+
+.. branch: emit-call-x86
+.. branch: emit-call-arm
+
+.. branch: on-abort-resops
+Added list of resops to the pypyjit on_abort hook.
+
+.. branch: logging-perf
+Speeds up the stdlib logging module
+
+.. branch: operrfmt-NT
+Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -2,7 +2,7 @@
 PyPy on Windows
 ===============
 
-Pypy is supported on Windows platforms, starting with Windows 2000.
+PyPy is supported on Windows platforms, starting with Windows 2000.
 The following text gives some hints about how to translate the PyPy
 interpreter.
 
@@ -199,9 +199,9 @@
 
 or such, depending on your mingw64 download.
 
-hacking on Pypy with the mingw compiler
+hacking on PyPy with the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since hacking on Pypy means running tests, you will need a way to specify
+Since hacking on PyPy means running tests, you will need a way to specify
 the mingw compiler when hacking (as opposed to translating). As of
 March 2012, --cc is not a valid option for pytest.py. However if you set an
 environment variable CC to the compliter exe, testing will use it.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,8 +2,8 @@
 # App-level version of py.py.
 # See test/test_app_main.
 
-# Missing vs CPython: -d, -OO, -t, -v, -x, -3
-"""\
+# Missing vs CPython: -d, -t, -v, -x, -3
+USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 -c cmd : program passed in as string (terminates option list)
@@ -12,7 +12,8 @@
 -i     : inspect interactively after running script; forces a prompt even
          if stdin does not appear to be a terminal; also PYTHONINSPECT=x
 -m mod : run library module as a script (terminates option list)
--O     : dummy optimization flag for compatibility with CPython
+-O     : skip assert statements
+-OO    : remove docstrings when importing modules in addition to -O
 -R     : ignored (see http://bugs.python.org/issue14621)
 -Q arg : division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
@@ -27,7 +28,6 @@
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
 """
-USAGE1 = __doc__
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
 Other environment variables:
@@ -470,6 +470,10 @@
         sys.py3kwarning = bool(sys.flags.py3k_warning)
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
+        if sys.flags.optimize >= 1:
+            import __pypy__
+            __pypy__.set_debug(False)
+
         if sys.py3kwarning:
             print >> sys.stderr, (
                 "Warning: pypy does not implement py3k warnings")
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -86,12 +86,9 @@
             args_w = space.fixedview(w_stararg)
         except OperationError, e:
             if e.match(space, space.w_TypeError):
-                w_type = space.type(w_stararg)
-                typename = w_type.getname(space)
-                raise OperationError(
+                raise operationerrfmt(
                     space.w_TypeError,
-                    space.wrap("argument after * must be "
-                               "a sequence, not %s" % (typename,)))
+                    "argument after * must be a sequence, not %T", w_stararg)
             raise
         self.arguments_w = self.arguments_w + args_w
 
@@ -116,12 +113,10 @@
                 w_keys = space.call_method(w_starstararg, "keys")
             except OperationError, e:
                 if e.match(space, space.w_AttributeError):
-                    w_type = space.type(w_starstararg)
-                    typename = w_type.getname(space)
-                    raise OperationError(
+                    raise operationerrfmt(
                         space.w_TypeError,
-                        space.wrap("argument after ** must be "
-                                   "a mapping, not %s" % (typename,)))
+                        "argument after ** must be a mapping, not %T",
+                        w_starstararg)
                 raise
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -245,6 +245,8 @@
         if w_len is None:
             w_len = space.len(self.w_consts)
             space.setitem(self.w_consts, w_key, w_len)
+        if space.int_w(w_len) == 0:
+            self.scope.doc_removable = False
         return space.int_w(w_len)
 
     def _make_key(self, obj):
@@ -632,6 +634,7 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+    ops.JUMP_IF_NOT_DEBUG : 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2793,8 +2793,7 @@
 
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2835,8 +2834,7 @@
 
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2881,8 +2879,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -2925,8 +2922,7 @@
 
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2971,8 +2967,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -2993,8 +2988,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3024,8 +3018,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3046,8 +3039,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3064,8 +3056,7 @@
 
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3081,8 +3072,7 @@
 
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3131,8 +3121,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3149,8 +3138,7 @@
 
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
             list_w = []
@@ -3166,8 +3154,7 @@
 
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3183,8 +3170,7 @@
 
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3234,8 +3220,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3278,8 +3263,7 @@
 
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3320,8 +3304,7 @@
 
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3341,8 +3324,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3391,8 +3373,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3415,8 +3396,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3439,8 +3419,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3489,8 +3468,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3509,8 +3487,7 @@
 
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -3530,8 +3507,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3579,8 +3555,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3603,8 +3578,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3623,8 +3597,7 @@
 
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3640,8 +3613,7 @@
 
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3690,8 +3662,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3710,8 +3681,7 @@
 
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3727,8 +3697,7 @@
 
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3776,8 +3745,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3796,8 +3764,7 @@
 
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3813,8 +3780,7 @@
 
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3862,8 +3828,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -3886,8 +3851,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -3906,8 +3870,7 @@
 
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3954,8 +3917,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -3978,8 +3940,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4002,8 +3963,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4048,8 +4008,7 @@
 
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -4065,8 +4024,7 @@
 
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
             list_w = []
@@ -4082,8 +4040,7 @@
 
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -4128,8 +4085,7 @@
 
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -4145,8 +4101,7 @@
 
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
             list_w = []
@@ -4193,8 +4148,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4217,8 +4171,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4262,8 +4215,7 @@
 
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4308,8 +4260,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4329,8 +4280,7 @@
 
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4350,8 +4300,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4399,8 +4348,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4423,8 +4371,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4447,8 +4394,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4493,8 +4439,7 @@
 
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4539,8 +4484,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4638,8 +4582,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4660,8 +4603,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4691,8 +4633,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4711,8 +4652,7 @@
 
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -4758,8 +4698,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4782,8 +4721,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4806,8 +4744,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4856,8 +4793,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4880,8 +4816,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -4929,8 +4864,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -4951,8 +4885,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5000,8 +4933,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5024,8 +4956,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5048,8 +4979,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5094,8 +5024,7 @@
 
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
             list_w = []
@@ -5111,8 +5040,7 @@
 
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -5155,8 +5083,7 @@
 
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
             list_w = []
@@ -5201,8 +5128,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5221,8 +5147,7 @@
 
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5268,8 +5193,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5288,8 +5212,7 @@
 
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5335,8 +5258,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5359,8 +5281,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5379,8 +5300,7 @@
 
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5427,8 +5347,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5447,8 +5366,7 @@
 
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5494,8 +5412,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5542,8 +5459,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5562,8 +5478,7 @@
 
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
             list_w = []
@@ -5579,8 +5494,7 @@
 
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
             list_w = []
@@ -5628,8 +5542,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5648,8 +5561,7 @@
 
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
             list_w = []
@@ -5665,8 +5577,7 @@
 
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
             list_w = []
@@ -5686,8 +5597,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5710,8 +5620,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 64:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5764,8 +5673,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5812,8 +5720,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5858,8 +5765,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5904,8 +5810,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -5928,8 +5833,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -5950,8 +5854,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6000,8 +5903,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6024,8 +5926,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6048,8 +5949,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6098,8 +5998,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6120,8 +6019,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6165,8 +6063,7 @@
 
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
             list_w = []
@@ -6186,8 +6083,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6232,8 +6128,7 @@
 
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
             list_w = []
@@ -6253,8 +6148,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6303,8 +6197,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6422,8 +6315,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6446,8 +6338,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6470,8 +6361,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6516,8 +6406,7 @@
 
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
             list_w = []
@@ -6562,8 +6451,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6834,8 +6722,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6858,8 +6745,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6878,8 +6764,7 @@
 
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
             list_w = []
@@ -6926,8 +6811,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 

From noreply at buildbot.pypy.org  Tue Jun  4 22:28:22 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Tue,  4 Jun 2013 22:28:22 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: Implement title() method.
Message-ID: <20130604202822.6C72A1C145C@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64784:8c06f1b5cde7
Date: 2013-06-04 21:34 +0200
http://bitbucket.org/pypy/pypy/changeset/8c06f1b5cde7/

Log:	Implement title() method.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -117,6 +117,8 @@
     def _isdigit(self, ch):
         return ch.isdigit()
 
+    _iscased = _isalpha
+
     def _upper(self, ch):
         if ch.islower():
             o = ord(ch) - 32
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -776,22 +776,19 @@
         return space.wrap(builder.build())
 
     def descr_title(self, space):
-        # XXX just to pass the test
-        return space.wrap(self._val().title())
-
         selfval = self._val()
         if len(selfval) == 0:
             return self
 
         builder = self._builder(len(selfval))
-        prev_letter = ' '
-        for pos in range(len(input)):
-            ch = input[pos]
-            if not prev_letter.isalpha():
+        previous_is_cased = False
+        for pos in range(len(selfval)):
+            ch = selfval[pos]
+            if not previous_is_cased:
                 builder.append(self._upper(ch))
             else:
                 builder.append(self._lower(ch))
-            prev_letter = ch
+            previous_is_cased = self._iscased(ch)
         return space.wrap(builder.build())
 
     DEFAULT_NOOP_TABLE = ''.join([chr(i) for i in range(256)])
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -110,6 +110,9 @@
     def _isdigit(self, ch):
         return ch.isdigit()
 
+    def _iscased(self, ch):
+        return unicodedb.iscased(ord(ch))
+
     def _upper(self, ch):
         return unichr(unicodedb.toupper(ord(ch)))
 

From noreply at buildbot.pypy.org  Tue Jun  4 22:28:26 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Tue,  4 Jun 2013 22:28:26 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: Fix.
Message-ID: <20130604202826.F06181C016D@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64785:1fa825c8c78d
Date: 2013-06-04 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/1fa825c8c78d/

Log:	Fix.

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -12,7 +12,7 @@
 from pypy.module.cpyext.stringobject import PyString_Check
 from pypy.module.sys.interp_encoding import setdefaultencoding
 from pypy.module._codecs.interp_codecs import CodecState
-from pypy.objspace.std import unicodeobject, bytesobject
+from pypy.objspace.std import unicodeobject
 from rpython.rlib import runicode
 from rpython.tool.sourcetools import func_renamer
 import sys
@@ -684,9 +684,12 @@
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
     if rffi.cast(lltype.Signed, direction) <= 0:
-        return bytesobject.stringstartswith(str, substr, start, end)
+        w_ret = space.call_method(w_str, "startswith", w_substr,
+                                  space.wrap(start), space.wrap(end))
     else:
-        return bytesobject.stringendswith(str, substr, start, end)
+        w_ret = space.call_method(w_str, "endswith", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_ret)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_Count(space, w_str, w_substr, start, end):
diff --git a/pypy/module/micronumpy/stdobjspace.py b/pypy/module/micronumpy/stdobjspace.py
--- a/pypy/module/micronumpy/stdobjspace.py
+++ b/pypy/module/micronumpy/stdobjspace.py
@@ -7,5 +7,5 @@
 
 def register_delegates(typeorder):
     typeorder[interp_boxes.W_StringBox] = [
-        (bytesobject.W_StringObject, delegate_stringbox2stringobj),
+        (bytesobject.W_BytesObject, delegate_stringbox2stringobj),
     ]
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -434,8 +434,11 @@
         return space.wrap(value)
 
     def descr_lower(self, space):
-        self = self._value
-        return space.wrap(self.lower())
+        value = self._val()
+        builder = self._builder(len(value))
+        for i in range(len(value)):
+            builder.append(self._lower(value[i]))
+        return self._new(builder.build())
 
     def descr_partition(self, space, w_sub):
         value = self._val()
@@ -822,8 +825,11 @@
         return self._new(buf.build())
 
     def descr_upper(self, space):
-        self = self._value
-        return space.wrap(self.upper())
+        value = self._val()
+        builder = self._builder(len(value))
+        for i in range(len(value)):
+            builder.append(self._upper(value[i]))
+        return self._new(builder.build())
 
     @unwrap_spec(width=int)
     def descr_zfill(self, space, width):

From noreply at buildbot.pypy.org  Wed Jun  5 03:32:53 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed,  5 Jun 2013 03:32:53 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: properly decode SyntaxError source text
Message-ID: <20130605013253.76EA51C1527@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64786:e7eccdeaa126
Date: 2013-06-04 18:31 -0700
http://bitbucket.org/pypy/pypy/changeset/e7eccdeaa126/

Log:	properly decode SyntaxError source text

diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -12,15 +12,19 @@
         self.lastlineno = lastlineno
 
     def wrap_info(self, space):
-        if self.filename is None:
-            w_filename = space.w_None
-        else:
+        w_text = w_filename = space.w_None
+        if self.text is not None:
+            from rpython.rlib.runicode import str_decode_utf_8
+            # self.text may not be UTF-8 in case of decoding errors
+            w_text = space.wrap(str_decode_utf_8(self.text, len(self.text),
+                                                 'replace')[0])
+        if self.filename is not None:
             w_filename = space.fsdecode(space.wrapbytes(self.filename))
         return space.newtuple([space.wrap(self.msg),
                                space.newtuple([w_filename,
                                                space.wrap(self.lineno),
                                                space.wrap(self.offset),
-                                               space.wrap(self.text),
+                                               space.wrap(w_text),
                                                space.wrap(self.lastlineno)])])
 
     def __str__(self):
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -853,7 +853,6 @@
         """
 
     def test_cpython_issue2301(self):
-        skip('XXX')
         try:
             compile(b"# coding: utf7\nprint '+XnQ-'", "dummy", "exec")
         except SyntaxError as v:

From noreply at buildbot.pypy.org  Wed Jun  5 10:13:47 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed,  5 Jun 2013 10:13:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: add runner tests for cast_int_to_float
 and cast_float_to_int
Message-ID: <20130605081347.F29361C01CD@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64787:ca5c756f0e50
Date: 2013-06-05 03:10 -0500
http://bitbucket.org/pypy/pypy/changeset/ca5c756f0e50/

Log:	add runner tests for cast_int_to_float and cast_float_to_int

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1911,6 +1911,29 @@
                                      [BoxPtr(x)], 'int').value
         assert res == -19
 
+    def test_cast_int_to_float(self):
+        if not self.cpu.supports_floats:
+            py.test.skip("requires floats")
+        for x in [-10, -1, 0, 3, 42, sys.maxint-1]:
+            res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
+                                         [BoxInt(x)],  'float').value
+            assert longlong.getrealfloat(res) == float(x)
+            res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
+                                         [ConstInt(x)],  'float').value
+            assert longlong.getrealfloat(res) == float(x)
+
+    def test_cast_float_to_int(self):
+        if not self.cpu.supports_floats:
+            py.test.skip("requires floats")
+        for x in [-24.23, -5.3, 0.0, 3.1234, 11.1, 0.1]:
+            v = longlong.getfloatstorage(x)
+            res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
+                                         [BoxFloat(v)],  'int').value
+            assert res == int(x)
+            res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
+                                         [ConstFloat(v)],  'int').value
+            assert res == int(x)
+
     def test_convert_float_bytes(self):
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")

From noreply at buildbot.pypy.org  Wed Jun  5 11:50:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 11:50:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document merged branch
Message-ID: <20130605095043.4870B1C1007@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64788:5e587e197d4f
Date: 2013-06-05 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/5e587e197d4f/

Log:	Document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -50,3 +50,5 @@
 .. branch: win32-fixes3
 Skip and fix some non-translated (own) tests for win32 builds
 
+.. branch: ctypes-byref
+Add the '_obj' attribute on ctypes pointer() and byref() objects

From noreply at buildbot.pypy.org  Wed Jun  5 12:34:18 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 12:34:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix various tests failing because of
	e953dfbc7f0a.
Message-ID: <20130605103418.AF0B01C12FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64789:d0ffe15789d8
Date: 2013-06-05 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d0ffe15789d8/

Log:	Fix various tests failing because of e953dfbc7f0a.

diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,6 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
-jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Distance to target address
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_intbound.py b/pypy/module/pypyjit/test_pypy_c/test_intbound.py
--- a/pypy/module/pypyjit/test_pypy_c/test_intbound.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_intbound.py
@@ -231,6 +231,7 @@
         assert loop.match("""
             i8 = int_lt(i6, 300)
             guard_true(i8, descr=...)
+            guard_not_invalidated?
             i10 = int_lshift(i6, 1)
             i12 = int_add_ovf(i5, 1)
             guard_no_overflow(descr=...)
@@ -253,6 +254,7 @@
         assert loop.match("""
             i8 = int_lt(i6, 300)
             guard_true(i8, descr=...)
+            guard_not_invalidated?
             i10 = int_add_ovf(i5, 8)
             guard_no_overflow(descr=...)
             i12 = int_add(i6, 1)

From noreply at buildbot.pypy.org  Wed Jun  5 14:15:27 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 14:15:27 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add a fast path for decoding
 ascii-only, it gives ~10% speedup on certain benchmarks
Message-ID: <20130605121527.8541A1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64790:73de5d85c1b3
Date: 2013-06-05 11:32 +0200
http://bitbucket.org/pypy/pypy/changeset/73de5d85c1b3/

Log:	add a fast path for decoding ascii-only, it gives ~10% speedup on
	certain benchmarks

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -96,13 +96,20 @@
 
     def decode_string(self):
         start = self.i
+        bits = 0
         while not self.eof():
             # this loop is a fast path for strings which do not contain escape
             # characters
             ch = self.next()
+            bits |= ord(ch)
             if ch == '"':
                 content_utf8 = self.getslice(start, self.i-1)
-                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
+                if bits & 0x80:
+                    # the 8th bit is set, it's an utf8 strnig
+                    content_unicode = content_utf8.decode('utf-8')
+                else:
+                    # ascii only, faster to decode
+                    content_unicode = content_utf8.decode('ascii')
                 self.last_type = TYPE_STRING
                 return self.space.wrap(content_unicode)
             elif ch == '\\':

From noreply at buildbot.pypy.org  Wed Jun  5 14:15:28 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 14:15:28 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add parsing of integers
Message-ID: <20130605121528.D69CA1C12FE@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64791:65996c45a142
Date: 2013-06-05 12:32 +0200
http://bitbucket.org/pypy/pypy/changeset/65996c45a142/

Log:	add parsing of integers

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -51,14 +51,36 @@
 
     def decode_any(self):
         self.skip_whitespace()
-        ch = self.next()
+        ch = self.peek()
         if ch == '"':
+            self.next()
             return self.decode_string()
+        elif ch.isdigit() or ch == '-':
+            return self.decode_numeric(ch)
         elif ch == '{':
+            self.next()
             return self.decode_object()
         else:
             assert False, 'Unkown char: %s' % ch
 
+    def decode_numeric(self, ch):
+        intval = 0
+        sign = 1
+        if ch == '-':
+            sign = -1
+            self.next()
+
+        while not self.eof():
+            ch = self.peek()
+            if ch.isdigit():
+                intval = intval*10 + ord(ch)-ord('0')
+                self.next()
+            else:
+                break
+        #
+        intval = intval*sign
+        return self.space.wrap(intval)
+
     def decode_object(self):
         start = self.i
         w_dict = self.space.newdict()
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -69,6 +69,12 @@
         s = r'"\u1234"'
         assert _fastjson.loads(s) == u'\u1234'
 
+    def test_decode_numeric(self):
+        import _fastjson
+        assert _fastjson.loads('42') == 42
+        assert _fastjson.loads('-42') == -42
+        raises(ValueError, "_fastjson.loads('42 abc')")
+
     def test_decode_object(self):
         import _fastjson
         assert _fastjson.loads('{}') == {}

From noreply at buildbot.pypy.org  Wed Jun  5 14:15:30 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 14:15:30 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add support for parsing float values
Message-ID: <20130605121530.1F15C1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64792:535479329816
Date: 2013-06-05 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/535479329816/

Log:	add support for parsing float values

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -7,6 +7,15 @@
 def is_whitespace(ch):
     return ch == ' ' or ch == '\t' or ch == '\r' or ch == '\n'
 
+# precomputing negative powers of 10 is MUCH faster than using e.g. math.pow
+# at runtime
+NEG_POW_10 = [10**-i for i in range(16)]
+def neg_pow_10(x, exp):
+    if exp >= len(NEG_POW_10):
+        return 0.0
+    return x * NEG_POW_10[exp]
+
+
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 
@@ -23,6 +32,12 @@
     def peek(self):
         return self.s[self.i]
 
+    def peek_maybe(self):
+        if self.eof():
+            return '\0'
+        else:
+            return self.peek()
+
     def next(self):
         ch = self.peek()
         self.i += 1
@@ -56,30 +71,69 @@
             self.next()
             return self.decode_string()
         elif ch.isdigit() or ch == '-':
-            return self.decode_numeric(ch)
+            return self.decode_numeric()
         elif ch == '{':
             self.next()
             return self.decode_object()
         else:
-            assert False, 'Unkown char: %s' % ch
+            self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
+                        ch, self.i)
 
-    def decode_numeric(self, ch):
-        intval = 0
+    def decode_numeric(self):
+        intval = self.parse_integer()
+        #
+        is_float = False
+        exp = 0
+        frcval = 0.0
+        frccount = 0
+        #
+        # check for the optional fractional part
+        ch = self.peek_maybe()
+        if ch == '.':
+            is_float = True
+            self.next()
+            frcval, frccount = self.parse_digits()
+            frcval = neg_pow_10(frcval, frccount)
+            ch = self.peek_maybe()
+        # check for the optional exponent part
+        if ch == 'E' or ch == 'e':
+            is_float = True
+            self.next()
+            exp = self.parse_integer()
+        #
+        if is_float:
+            # build the float
+            floatval = intval + frcval
+            floatval = floatval * 10**exp
+            return self.space.wrap(floatval)
+        else:
+            return self.space.wrap(intval)
+
+    def parse_integer(self):
+        "Parse a decimal number with an optional minus sign"
         sign = 1
-        if ch == '-':
+        if self.peek_maybe() == '-':
             sign = -1
             self.next()
+        intval, _ = self.parse_digits()
+        return sign * intval
 
+    def parse_digits(self):
+        "Parse a sequence of digits as a decimal number. No sign allowed"
+        intval = 0
+        count = 0
         while not self.eof():
             ch = self.peek()
             if ch.isdigit():
                 intval = intval*10 + ord(ch)-ord('0')
+                count += 1
                 self.next()
             else:
                 break
-        #
-        intval = intval*sign
-        return self.space.wrap(intval)
+        if count == 0:
+            self._raise("Expected digit at char %d", self.i)
+        return intval, count
+        
 
     def decode_object(self):
         start = self.i
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -71,9 +71,31 @@
 
     def test_decode_numeric(self):
         import _fastjson
-        assert _fastjson.loads('42') == 42
-        assert _fastjson.loads('-42') == -42
-        raises(ValueError, "_fastjson.loads('42 abc')")
+        def check(s, val):
+            res = _fastjson.loads(s)
+            assert type(res) is type(val)
+            assert res == val
+        #
+        check('42', 42)
+        check('-42', -42)
+        check('42.123', 42.123)
+        check('42E0', 42.0)
+        check('42E3', 42000.0)
+        check('42E-1', 4.2)
+        check('42.123E3', 42123.0)
+
+    def test_decode_numeric_invalid(self):
+        import _fastjson
+        def error(s):
+            raises(ValueError, _fastjson.loads, s)
+        #
+        error('  42   abc')
+        error('.123')
+        error('12.')
+        error('12.-3')
+        error('12E')
+        error('12E-')
+
 
     def test_decode_object(self):
         import _fastjson

From noreply at buildbot.pypy.org  Wed Jun  5 14:53:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 14:53:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: Partial revert of 661d7f7624dc: don't
 let the JIT look inside rbigint.
Message-ID: <20130605125316.008271C12FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64793:236f4d2c1567
Date: 2013-06-05 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/236f4d2c1567/

Log:	Partial revert of 661d7f7624dc: don't let the JIT look inside
	rbigint. It causes issues. Instead, fix the test more directly by
	adding some @jit.elidable that were needed for consistency.

diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -118,7 +118,7 @@
     def look_inside_function(self, func):
         mod = func.__module__ or '?'
 
-        if mod == 'rpython.rlib.rlocale' or mod == 'rpython.rlib.rsocket':
+        if mod == 'rpython.rlib.rbigint' or mod == 'rpython.rlib.rlocale' or mod == 'rpython.rlib.rsocket':
             return False
         if mod.startswith('pypy.interpreter.astcompiler.'):
             return False
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -8,7 +8,12 @@
 
 def test_bigint():
     from rpython.rlib.rbigint import rbigint
-    assert pypypolicy.look_inside_function(rbigint.lt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.eq.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.ne.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.lt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.le.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.gt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.ge.im_func)
 
 def test_rlocale():
     from rpython.rlib.rlocale import setlocale
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -448,6 +448,7 @@
             i += 1
         return True
 
+    @jit.elidable
     def ne(self, other):
         return not self.eq(other)
 
@@ -486,12 +487,15 @@
             i -= 1
         return False
 
+    @jit.elidable
     def le(self, other):
         return not other.lt(self)
 
+    @jit.elidable
     def gt(self, other):
         return other.lt(self)
 
+    @jit.elidable
     def ge(self, other):
         return not self.lt(other)
 

From noreply at buildbot.pypy.org  Wed Jun  5 14:53:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 14:53:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: More @jit.elidable. The new test in
 test_misc is still not passing.
Message-ID: <20130605125317.4251A1C12FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64794:e4d4c3e4eafa
Date: 2013-06-05 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/e4d4c3e4eafa/

Log:	More @jit.elidable. The new test in test_misc is still not passing.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -407,5 +407,5 @@
 
         log = self.run(main, [300])
         loop, = log.loops_by_id("long_op")
-        assert log.match("""
+        assert loop.match("""
         """)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -83,6 +83,13 @@
     func._jit_look_inside_ = False
     return func
 
+def look_inside(func):
+    """ Make sure the JIT traces inside decorated function, even
+    if the rest of the module is not visible to the JIT
+    """
+    func._jit_look_inside_ = True
+    return func
+
 def unroll_safe(func):
     """ JIT can safely unroll loops in this function and this will
     not lead to code explosion
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -190,9 +190,9 @@
             
         
     @staticmethod
+    @jit.elidable
     def frombool(b):
-        # This function is marked as pure, so you must not call it and
-        # then modify the result.
+        # You must not call this function and then modify the result.
         if b:
             return ONERBIGINT
         return NULLRBIGINT
@@ -251,6 +251,7 @@
         return _decimalstr_to_bigint(s)
 
     @staticmethod
+    @jit.elidable
     def frombytes(s, byteorder, signed):
         if byteorder not in ('big', 'little'):
             raise InvalidEndiannessError()
@@ -383,9 +384,11 @@
     def tolonglong(self):
         return _AsLongLong(self)
 
+    @jit.look_inside
     def tobool(self):
         return self.sign != 0
 
+    @jit.elidable
     def touint(self):
         if self.sign == -1:
             raise ValueError("cannot convert negative integer to unsigned int")
@@ -410,13 +413,16 @@
             raise ValueError("cannot convert negative integer to unsigned int")
         return _AsULonglong_ignore_sign(self)
 
+    @jit.elidable
     def uintmask(self):
         return _AsUInt_mask(self)
 
+    @jit.elidable
     def ulonglongmask(self):
         """Return r_ulonglong(self), truncating."""
         return _AsULonglong_mask(self)
 
+    @jit.elidable
     def tofloat(self):
         return _AsDouble(self)
 
@@ -448,7 +454,7 @@
             i += 1
         return True
 
-    @jit.elidable
+    @jit.look_inside
     def ne(self, other):
         return not self.eq(other)
 
@@ -487,15 +493,15 @@
             i -= 1
         return False
 
-    @jit.elidable
+    @jit.look_inside
     def le(self, other):
         return not other.lt(self)
 
-    @jit.elidable
+    @jit.look_inside
     def gt(self, other):
         return other.lt(self)
 
-    @jit.elidable
+    @jit.look_inside
     def ge(self, other):
         return not self.lt(other)
 
@@ -596,6 +602,7 @@
             
         return div
 
+    @jit.look_inside
     def div(self, other):
         return self.floordiv(other)
 
@@ -796,14 +803,17 @@
             z = z.sub(c)
         return z
 
+    @jit.elidable
     def neg(self):
         return rbigint(self._digits, -self.sign, self.size)
 
+    @jit.elidable
     def abs(self):
         if self.sign != -1:
             return self
         return rbigint(self._digits, 1, self.size)
 
+    @jit.elidable
     def invert(self): #Implement ~x as -(x + 1)
         if self.sign == 0:
             return ONENEGATIVERBIGINT
@@ -913,12 +923,14 @@
     def or_(self, other):
         return _bitwise(self, '|', other)
 
+    @jit.elidable
     def oct(self):
         if self.sign == 0:
             return '0L'
         else:
             return _format(self, BASE8, '0', 'L')
 
+    @jit.elidable
     def hex(self):
         return _format(self, BASE16, '0x', 'L')
 

From noreply at buildbot.pypy.org  Wed Jun  5 15:01:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 15:01:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: A test and fix for jit.look_inside.
Message-ID: <20130605130137.C45EA1C1401@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64795:6ee99f717da5
Date: 2013-06-05 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/6ee99f717da5/

Log:	A test and fix for jit.look_inside.

diff --git a/rpython/jit/codewriter/policy.py b/rpython/jit/codewriter/policy.py
--- a/rpython/jit/codewriter/policy.py
+++ b/rpython/jit/codewriter/policy.py
@@ -39,8 +39,6 @@
         return True # look into everything by default
 
     def _reject_function(self, func):
-        if hasattr(func, '_jit_look_inside_'):
-            return not func._jit_look_inside_
         # explicitly elidable functions are always opaque
         if getattr(func, '_elidable_function_', False):
             return True
@@ -58,8 +56,11 @@
         except AttributeError:
             see_function = True
         else:
-            see_function = (self.look_inside_function(func) and not
-                            self._reject_function(func))
+            if hasattr(func, '_jit_look_inside_'):
+                see_function = func._jit_look_inside_   # override guessing
+            else:
+                see_function = (self.look_inside_function(func) and not
+                                self._reject_function(func))
             contains_loop = contains_loop and not getattr(
                     func, '_jit_unroll_safe_', False)
 
diff --git a/rpython/jit/codewriter/test/test_policy.py b/rpython/jit/codewriter/test/test_policy.py
--- a/rpython/jit/codewriter/test/test_policy.py
+++ b/rpython/jit/codewriter/test/test_policy.py
@@ -65,6 +65,20 @@
     graph = support.getgraph(h, [5])
     assert not JitPolicy().look_inside_graph(graph)
 
+def test_look_inside():
+    def h1(x):
+        return x + 1
+    @jit.look_inside    # force True, even if look_inside_function() thinks not
+    def h2(x):
+        return x + 2
+    class MyPolicy(JitPolicy):
+        def look_inside_function(self, func):
+            return False
+    graph1 = support.getgraph(h1, [5])
+    graph2 = support.getgraph(h2, [5])
+    assert not MyPolicy().look_inside_graph(graph1)
+    assert MyPolicy().look_inside_graph(graph2)
+
 def test_loops():
     def g(x):
         i = 0

From noreply at buildbot.pypy.org  Wed Jun  5 15:05:54 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 15:05:54 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: rpython fixes
Message-ID: <20130605130554.EB2FC1C1401@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64796:6b66c9b61731
Date: 2013-06-05 15:05 +0200
http://bitbucket.org/pypy/pypy/changeset/6b66c9b61731/

Log:	rpython fixes

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -1,3 +1,4 @@
+import math
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -9,13 +10,12 @@
 
 # precomputing negative powers of 10 is MUCH faster than using e.g. math.pow
 # at runtime
-NEG_POW_10 = [10**-i for i in range(16)]
+NEG_POW_10 = [10.0**-i for i in range(16)]
 def neg_pow_10(x, exp):
     if exp >= len(NEG_POW_10):
         return 0.0
     return x * NEG_POW_10[exp]
 
-
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 
@@ -104,7 +104,7 @@
         if is_float:
             # build the float
             floatval = intval + frcval
-            floatval = floatval * 10**exp
+            floatval = floatval * math.pow(10, exp)
             return self.space.wrap(floatval)
         else:
             return self.space.wrap(intval)
@@ -236,6 +236,8 @@
             uchr = unichr(int(hexdigits, 16))
         except ValueError:
             self._raise("Invalid \uXXXX escape (char %d)", self.i-1)
+            return # help the annotator to know that we'll never go beyond
+                   # this point
         #
         utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
         builder.append(utf8_ch)

From noreply at buildbot.pypy.org  Wed Jun  5 15:57:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 15:57:37 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130605135737.74E371C3333@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64:87d10f56bb42
Date: 2013-06-05 12:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/87d10f56bb42/

Log:	in-progress

diff --git a/.hgignore b/.hgignore
new file mode 100644
--- /dev/null
+++ b/.hgignore
@@ -0,0 +1,3 @@
+syntax: glob
+*.pyc
+*~
diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -18,21 +18,18 @@
    die if we start more than 0x7fff threads. */
 static revision_t next_locked_value = (LOCKED + 1) | 1;
 
-/* a negative odd number that uniquely identifies the currently running
-   transaction (but the number in aborted transactions is reused).
-   Because we don't know yet the value of 'global_cur_time' that we'll
-   be assigned when we commit, we use the (negative of) the value of
-   'global_cur_time' when we committed the previous transaction. */
-__thread revision_t stm_local_revision;
+/* a negative odd number that identifies the currently running
+   transaction within the thread. */
+__thread revision_t stm_private_rev_num;
 
 
 revision_t stm_global_cur_time(void)  /* for tests */
 {
   return global_cur_time;
 }
-revision_t stm_local_rev(void)        /* for tests */
+revision_t get_private_rev_num(void)        /* for tests */
 {
-  return stm_local_revision;
+  return stm_private_rev_num;
 }
 struct tx_descriptor *stm_thread_descriptor(void)  /* for tests */
 {
@@ -70,6 +67,8 @@
 
 static gcptr HeadOfRevisionChainList(struct tx_descriptor *d, gcptr G)
 {
+  abort();
+#if 0
   gcptr R = G;
   revision_t v;
 
@@ -135,10 +134,13 @@
       goto retry;                      // restart searching from R
     }
   return R;
+#endif
 }
 
 static inline gcptr AddInReadSet(struct tx_descriptor *d, gcptr R)
 {
+  abort();
+#if 0
   fprintf(stderr, "AddInReadSet(%p)\n", R);
   d->count_reads++;
   if (!fxcache_add(&d->recent_reads_cache, R)) {
@@ -153,10 +155,13 @@
       //      return Localize(d, R);
       //  }
   return R;
+#endif
 }
 
 gcptr stm_DirectReadBarrier(gcptr G)
 {
+  abort();
+#if 0
   gcptr R;
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
@@ -174,19 +179,13 @@
       G2L_FIND(d->public_to_private, R, entry, goto not_found);
       L = entry->val;
       assert(L->h_revision == stm_local_revision);
-#if 0
-      if (R_Container && !(R_Container->h_tid & GCFLAG_GLOBAL))
-        {    /* R_Container is a local object */
-          gcptr *ref = (gcptr *)(((char *)R_Container) + offset);
-          *ref = L;   /* fix in-place */
-        }
-#endif
       return L;
 
     not_found:;
     }
   R = AddInReadSet(d, R);
   return R;
+#endif
 }
 
 static gcptr _latest_gcptr(gcptr R)
@@ -312,6 +311,8 @@
 
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
+  abort();
+#if 0
   if (R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
     {
       wlog_t *entry;
@@ -332,6 +333,7 @@
   AddInReadSet(d, R);
   /*mark*/
   return L;
+#endif
 }
 
 gcptr stm_WriteBarrier(gcptr P)
@@ -376,6 +378,8 @@
 static _Bool ValidateDuringTransaction(struct tx_descriptor *d,
                                        _Bool during_commit)
 {
+  abort();
+#if 0
   long i, size = d->list_of_read_objects.size;
   gcptr *items = d->list_of_read_objects.items;
 
@@ -416,6 +420,7 @@
         }
     }
   return 1;
+#endif
 }
 
 static void ValidateNow(struct tx_descriptor *d)
@@ -638,8 +643,8 @@
         goto retry;
 
       gcptr L = item->val;
-      assert(L->h_revision == stm_local_revision);
-      assert(v != stm_local_revision);
+      assert(L->h_revision == stm_private_rev_num);
+      assert(v != stm_private_rev_num);
       L->h_revision = v;   /* store temporarily this value here */
 
     } G2L_LOOP_END;
@@ -647,6 +652,8 @@
 
 static void CancelLocks(struct tx_descriptor *d)
 {
+  abort();
+#if 0
   revision_t my_lock = d->my_lock;
   wlog_t *item;
 
@@ -672,6 +679,7 @@
       ACCESS_ONCE(R->h_revision) = v;
 
     } G2L_LOOP_END;
+#endif
 }
 
 static pthread_mutex_t mutex_prebuilt_gcroots = PTHREAD_MUTEX_INITIALIZER;
@@ -820,15 +828,15 @@
           "*************************************\n",
           (long)cur_time);
 
-  revision_t localrev = stm_local_revision;
+  revision_t localrev = stm_private_rev_num;
   UpdateProtectedChainHeads(d, cur_time, localrev);
   smp_wmb();
 
   revision_t newrev = -(cur_time + 1);
   assert(newrev & 1);
-  ACCESS_ONCE(stm_local_revision) = newrev;
+  ACCESS_ONCE(stm_private_rev_num) = newrev;
   fprintf(stderr, "%p: stm_local_revision = %ld\n", d, (long)newrev);
-  assert(d->local_revision_ref = &stm_local_revision);
+  assert(d->private_revision_ref = &stm_private_rev_num);
 
   UpdateChainHeads(d, cur_time, localrev);
 
@@ -1027,8 +1035,8 @@
         }
       assert(d->my_lock & 1);
       assert(d->my_lock > LOCKED);
-      stm_local_revision = -d->my_lock;   /* a unique negative odd value */
-      d->local_revision_ref = &stm_local_revision;
+      stm_private_rev_num = -1;
+      d->private_revision_ref = &stm_private_rev_num;
       d->max_aborts = -1;
       thread_descriptor = d;
 
diff --git a/c3/et.h b/c3/et.h
--- a/c3/et.h
+++ b/c3/et.h
@@ -1,7 +1,7 @@
 /*** Extendable Timestamps
  *
  * Documentation:
- * https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/stm2012/stmimpl.rst
+ * doc-*.txt
  *
  * This is very indirectly based on rstm_r5/stm/et.hpp.
  * See http://www.cs.rochester.edu/research/synchronization/rstm/api.shtml
@@ -20,35 +20,28 @@
  * collection: "young" objects are the ones in the nursery (plus a few big
  * ones outside) and will be collected by the following minor collection.
  *
- * Additionally, objects are either "public", "protected" or "private".  The
- * private objects have h_revision == stm_local_revision and are invisible
- * to other threads.  They become non-private when the transaction commits.
- *
- *                          non-private | private
- *           +------------------------------------------------------------
- *           |
- *       old |         public objects   |   old private objects
- *  ---------|
- *           |
- *     young |     [ protected objects  |  private objects  (--> grows) ]
- *  (nursery)|
+ * Additionally, objects are either "public", "protected" or "private".
  *
  * GCFLAG_OLD is set on old objects.
  *
  * GCFLAG_VISITED is used temporarily during major collections.
  *
+ * GCFLAG_PUBLIC is set on public objects.
+ *
+ * GCFLAG_BACKUP_COPY means the object is a (protected) backup copy.
+ *
  * GCFLAG_PUBLIC_TO_PRIVATE is added to a *public* object that has got a
  * *private* copy.  It is sticky, reset only at the next major collection.
  *
  * GCFLAG_PREBUILT_ORIGINAL is only set on the original version of
  * prebuilt objects.
  *
- * GCFLAG_WRITE_BARRIER is set on *old* *private* objects to track old-to-
- * young pointers.  It may be left set on *public* objects but is ignored
- * there, because the write barrier will trigger anyway on any non-private
- * object.  On an old private object, it is removed once a write occurs
- * and the object is recorded in 'private_old_pointing_to_young'; it is
- * set again at the next minor collection.
+ * GCFLAG_WRITE_BARRIER is set on *old* objects to track old-to- young
+ * pointers.  It may be left set on *public* objects but is ignored
+ * there, because public objects are read-only.  The flag is removed
+ * once a write occurs and the object is recorded in the list
+ * 'old_pointing_to_young'; it is set again at the next minor
+ * collection.
  *
  * GCFLAG_NURSERY_MOVED is used temporarily during minor collections.
  *
@@ -56,26 +49,31 @@
  * have been stolen.
  *
  * GCFLAG_STUB is used for debugging: it's set on stub objects made by
- * create_yo_stubs()
+ * stealing or by major collections.
  */
 #define GCFLAG_OLD               (STM_FIRST_GCFLAG << 0)
 #define GCFLAG_VISITED           (STM_FIRST_GCFLAG << 1)
-#define GCFLAG_PUBLIC_TO_PRIVATE (STM_FIRST_GCFLAG << 2)
+#define GCFLAG_PUBLIC            (STM_FIRST_GCFLAG << 2)
 #define GCFLAG_PREBUILT_ORIGINAL (STM_FIRST_GCFLAG << 3)
-#define GCFLAG_WRITE_BARRIER     (STM_FIRST_GCFLAG << 4)
-#define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 5)
-#define GCFLAG_STOLEN            (STM_FIRST_GCFLAG << 6)
-#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 7)   /* debugging */
+#define GCFLAG_BACKUP_COPY       (STM_FIRST_GCFLAG << 4)
+#define GCFLAG_PUBLIC_TO_PRIVATE (STM_FIRST_GCFLAG << 5)
+#define GCFLAG_WRITE_BARRIER     (STM_FIRST_GCFLAG << 6)
+#define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 7)
+#define GCFLAG_STOLEN            (STM_FIRST_GCFLAG << 8)
+#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 9)   /* debugging */
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
                           GCFLAG_PREBUILT_ORIGINAL | \
-                          GCFLAG_OLD)
+                          GCFLAG_OLD               | \
+                          GCFLAG_PUBLIC)
 
 #define GC_FLAG_NAMES  { "OLD",               \
                          "VISITED",           \
+                         "PUBLIC",            \
+                         "PREBUILT_ORIGINAL", \
+                         "BACKUP_COPY",       \
                          "PUBLIC_TO_PRIVATE", \
-                         "PREBUILT_ORIGINAL", \
                          "WRITE_BARRIER",     \
                          "NURSERY_MOVED",     \
                          "STOLEN",            \
@@ -121,12 +119,12 @@
   char *longest_abort_info;
   long long longest_abort_info_time;
   struct FXCache recent_reads_cache;
-  revision_t *local_revision_ref;
+  revision_t *private_revision_ref;
   struct tx_descriptor *tx_next, *tx_prev;   /* a doubly linked list */
 };
 
 extern __thread struct tx_descriptor *thread_descriptor;
-extern __thread revision_t stm_local_revision;
+extern __thread revision_t stm_private_rev_num;
 
 /************************************************************/
 
diff --git a/c3/gcpage.c b/c3/gcpage.c
--- a/c3/gcpage.c
+++ b/c3/gcpage.c
@@ -423,7 +423,7 @@
         assert(stmgc_classify(item->addr) == K_PUBLIC);
         /*..rt(stmgc_classify(item->val)  == K_PRIVATE); but in the
             other thread, which becomes: */
-        assert(item->val->h_revision == *d->local_revision_ref);
+        assert(item->val->h_revision == *d->private_revision_ref);
 
         item->addr->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
@@ -559,8 +559,8 @@
 {
     struct tx_descriptor *d;
     struct tx_descriptor *saved = thread_descriptor;
-    revision_t saved_local_rev = stm_local_revision;
-    assert(saved_local_rev == *saved->local_revision_ref);
+    revision_t saved_private_rev = stm_private_rev_num;
+    assert(saved_private_rev == *saved->private_revision_ref);
 
     for (d = tx_head; d; d = d->tx_next) {
         /* Force a minor collection to run in the thread 'd'.
@@ -572,12 +572,12 @@
             /* Hack: temporarily pretend that we "are" the other thread...
              */
             thread_descriptor = d;
-            stm_local_revision = *d->local_revision_ref;
+            stm_private_rev_num = *d->private_revision_ref;
             assert(stmgc_nursery_hiding(d, 0));
             stmgc_minor_collect_no_abort();
             assert(stmgc_nursery_hiding(d, 1));
             thread_descriptor = saved;
-            stm_local_revision = saved_local_rev;
+            stm_private_rev_num = saved_private_rev;
         }
     }
 }
diff --git a/c3/lists.c b/c3/lists.c
--- a/c3/lists.c
+++ b/c3/lists.c
@@ -223,15 +223,10 @@
 
 /************************************************************/
 
-void fxcache_clear(struct FXCache *fxcache)
+void _fxcache_reset(struct FXCache *fxcache)
 {
-  fxcache->shift += 4;
-  /* FX_ENTRIES+1 entries are needed */
-  if (fxcache->shift + FX_ENTRIES + 1 > FX_TOTAL) {
+    fxcache->shift = 0;
     memset(fxcache->cache, 0, sizeof(fxcache->cache));
-    fxcache->shift = 0;
-  }
-  fxcache->cache_start = (char *)(fxcache->cache + fxcache->shift);
 }
 
 /************************************************************/
diff --git a/c3/lists.h b/c3/lists.h
--- a/c3/lists.h
+++ b/c3/lists.h
@@ -168,16 +168,12 @@
 
 /* The fxcache_xx functions implement a fixed-size set of gcptr's.
    Moreover the gcptr's in the set are mapped to small integers.  In case
-   of collisions, old items are discarded.  The cache uses 3-way caching,
-   stored in 3 consecutive entries, but the 3 entries are in "cache lines"
-   that are only aligned to a multiple of 2.  This means that among the 3
-   items, the item 0 overlaps with the item 2 of the previous cache line,
-   and the item 2 overlaps with the item 0 of the following cache line.
-   The item 1 can only be seen by the current cache line.
+   of collisions, old items are discarded.  The cache doesn't use
+   multi-way caching for now.
 
-   The cache itself uses a total of FX_ENTRIES+1 entries in the 'cache'
-   array below, starting at 'cache_start'.  The reason it is bigger than
-   necessary is that fxcache_clear() simply shifts 'cache_start', making
+   The cache itself uses a total of FX_ENTRIES entries in the 'cache'
+   array below, starting at 'shift'.  The reason it is bigger than
+   necessary is that fxcache_clear() simply increments 'shift', making
    any previous entries invalid by not being in the correct position any
    more.
 */
@@ -187,41 +183,18 @@
 
 struct FXCache {
     char *cache_start;
-    revision_t nextadd;
     revision_t shift;
     revision_t cache[FX_TOTAL];
 };
 
-void fxcache_clear(struct FXCache *fxcache);
+void _fxcache_reset(struct FXCache *fxcache);
 
-static inline int fxcache_add(struct FXCache *fxcache, gcptr item) {
-    /* If 'item' is not in the cache, add it and returns 0.
-       If it is already, return 1.
-       */
-    revision_t uitem = (revision_t)item;
-    /* 'entry' points to 'cache_start[mask of uitem, even-valued]' */
-    revision_t *entry = (revision_t *)
-      (fxcache->cache_start + (uitem & ((FX_ENTRIES-2) * sizeof(revision_t))));
-    revision_t current;
-
-    current = entry[1];   /* first look here, the cache-private entry */
-    if (current == uitem)
-        return 1;
-
-    if (entry[0] == uitem) {
-        entry[0] = current;    /* move from this collidable entry to */
-        entry[1] = uitem;      /*    the cache-private entry         */
-        return 1;
-    }
-    if (entry[2] == uitem) {
-        entry[2] = current;    /* move from this collidable entry to */
-        entry[1] = uitem;      /*    the cache-private entry         */
-        return 1;
-    }
-
-    entry[fxcache->nextadd] = uitem;
-    fxcache->nextadd ^= 2;
-    return 0;
+static inline void fxcache_clear(struct FXCache *fxcache)
+{
+    fxcache->shift++;
+    if (fxcache->shift > FX_TOTAL - FX_ENTRIES)
+        _fxcache_reset(fxcache);
+    fxcache->cache_start = (char *)(fxcache->cache + fxcache->shift);
 }
 
 /************************************************************/
diff --git a/c3/nursery.c b/c3/nursery.c
--- a/c3/nursery.c
+++ b/c3/nursery.c
@@ -29,7 +29,7 @@
 enum protection_class_t stmgc_classify(gcptr obj)
 {
     /* note that this function never returns K_OLD_PRIVATE. */
-    if (obj->h_revision == stm_local_revision)
+    if (obj->h_revision == stm_private_rev_num)
         return K_PRIVATE;
     if (is_young(obj))
         return K_PROTECTED;
@@ -42,7 +42,7 @@
     /* for assertions only; moreover this function returns K_PRIVATE
        only for young private objects, and K_OLD_PRIVATE for old ones. */
     struct tx_descriptor *d = thread_descriptor;
-    int private = (obj->h_revision == stm_local_revision);
+    int private = (obj->h_revision == stm_private_rev_num);
     enum protection_class_t e;
 
     if (is_in_nursery(d, obj)) {
@@ -128,7 +128,7 @@
 {
     gcptr p = stmgcpage_malloc(size);
     memset(p, 0, size);
-    p->h_revision = stm_local_revision;
+    p->h_revision = stm_private_rev_num;
     p->h_tid = GCFLAG_OLD;
     return p;
 }
@@ -146,7 +146,7 @@
     }
     stm_dbgmem_used_again(cur, size, 1);
     gcptr p = (gcptr)cur;
-    p->h_revision = stm_local_revision;
+    p->h_revision = stm_private_rev_num;
     return p;
 }
 
@@ -185,7 +185,7 @@
                          GCFLAG_PREBUILT_ORIGINAL |
                          GCFLAG_WRITE_BARRIER     |
                          GCFLAG_OLD);
-    localobj->h_revision = stm_local_revision;
+    localobj->h_revision = stm_private_rev_num;
     return localobj;
 }
 
@@ -413,7 +413,7 @@
     /* nb. don't use stmgc_classify() here, because some objects trigger
        an assert at this point: young non-nursery objects which just
        grew the flag GCFLAG_OLD */
-    assert(obj->h_revision != stm_local_revision);  /* not a private object */
+    assert(obj->h_revision != stm_private_rev_num);  /* not a private object */
     PATCH_ROOT_WITH(obj);
     goto retry;
 }
@@ -490,7 +490,7 @@
 
         /* then we record the dependency in the dictionary
            'public_to_private' */
-        assert(L->h_revision == stm_local_revision);
+        assert(L->h_revision == stm_private_rev_num);
         g2l_insert(&d->public_to_private, R, L);
         /*mark*/
     }
@@ -956,6 +956,8 @@
 static gcptr extract_from_foreign_nursery(struct tx_descriptor *source_d,
                                           gcptr R)
 {
+    abort();
+#if 0
     /* "Stealing": this function follows a chain of protected objects in
        the foreign nursery of the thread 'source_d'.  It copies the last
        one outside the nursery, and return it. */
@@ -1011,6 +1013,7 @@
                            source_d->stolen_objects.size - 1);
 
     return N;
+#endif
 }
 
 void stmgc_public_to_foreign_protected(gcptr P)
@@ -1112,7 +1115,7 @@
 
             /* we re-insert L as a private copy of the public object N */
             N->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
-            assert(L->h_revision == stm_local_revision);
+            assert(L->h_revision == stm_private_rev_num);
             g2l_insert(&d->public_to_private, N, L);
             gcptrlist_insert(&d->public_to_young, N);
         }
diff --git a/c3/stmgc.h b/c3/stmgc.h
--- a/c3/stmgc.h
+++ b/c3/stmgc.h
@@ -21,7 +21,7 @@
 #define STM_SIZE_OF_USER_TID       (sizeof(revision_t) / 2)    /* in bytes */
 #define STM_FIRST_GCFLAG           (1L << (8 * STM_SIZE_OF_USER_TID))
 #define STM_USER_TID_MASK          (STM_FIRST_GCFLAG - 1)
-#define PREBUILT_FLAGS             (STM_FIRST_GCFLAG * (1 + 2 + 8))
+#define PREBUILT_FLAGS             (STM_FIRST_GCFLAG * (1 + 2 + 4 + 8))
 #define PREBUILT_REVISION          1
 
 
diff --git a/c3/stmsync.c b/c3/stmsync.c
--- a/c3/stmsync.c
+++ b/c3/stmsync.c
@@ -78,8 +78,11 @@
 gcptr stm_read_barrier(gcptr obj)
 {
     /* XXX inline in the caller */
+    abort();
+#if 0
     if (UNLIKELY(obj->h_revision != stm_local_revision))
         obj = stm_DirectReadBarrier(obj);
+#endif
     return obj;
 }
 
@@ -87,7 +90,7 @@
 {
     /* XXX inline in the caller */
     if (UNLIKELY(((obj->h_tid & GCFLAG_WRITE_BARRIER) != 0) |
-                 (obj->h_revision != stm_local_revision)))
+                 (obj->h_revision != stm_private_rev_num)))
         obj = stm_WriteBarrier(obj);
     return obj;
 }
diff --git a/c3/test/support.py b/c3/test/support.py
--- a/c3/test/support.py
+++ b/c3/test/support.py
@@ -80,7 +80,7 @@
     void rawsetlong(gcptr, long, long);
 
     gcptr pseudoprebuilt(size_t size, int tid);
-    revision_t get_local_revision(void);
+    revision_t get_private_rev_num(void);
     revision_t get_start_time(void);
 
     gcptr *addr_of_thread_local(void);
@@ -112,6 +112,7 @@
     extern revision_t stm_global_cur_time(void);
     extern void stmgcpage_add_prebuilt_root(gcptr);
     extern void stm_clear_between_tests(void);
+    extern revision_t get_private_rev_num(void);
 
     int gettid(gcptr obj)
     {
@@ -194,11 +195,6 @@
         return x;
     }
 
-    revision_t get_local_revision(void)
-    {
-        return stm_local_revision;
-    }
-
     revision_t get_start_time(void)
     {
         return thread_descriptor->start_time;
@@ -421,7 +417,8 @@
 def nalloc(size):
     "Allocate a fresh object from the nursery"
     p = lib.stm_allocate(size, 42 + size)
-    assert p.h_revision == lib.get_local_revision()
+    assert p.h_tid == 42 + size     # no GC flags
+    assert p.h_revision == lib.get_private_rev_num()
     return p
 
 def nalloc_refs(nrefs):
@@ -442,6 +439,7 @@
     p = lib.pseudoprebuilt(HDR + WORD * nrefs, 421 + nrefs)
     return p
 
+gettid = lib.gettid
 setptr = lib.setptr
 getptr = lib.getptr
 rawsetptr = lib.rawsetptr
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
new file mode 100644
--- /dev/null
+++ b/c3/test/test_et.py
@@ -0,0 +1,64 @@
+import py
+from support import *
+
+
+def setup_function(f):
+    lib.stm_clear_between_tests()
+    lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
+
+def teardown_function(_):
+    lib.stm_finalize()
+
+
+def test_freshly_created():
+    p = nalloc(HDR)
+    r = lib.get_private_rev_num()
+    assert r < 0 and r % 2 == 1
+    assert p.h_revision == r
+    assert p.h_tid == lib.gettid(p) | 0    # no GC flags
+
+def test_write_barrier_private():
+    p = nalloc(HDR)
+    assert lib.stm_write_barrier(p) == p
+    assert p.h_revision == lib.get_private_rev_num()
+    assert p.h_tid == lib.gettid(p) | 0    # no GC flags
+
+def test_protected_no_backup():
+    p = nalloc(HDR)
+    r = lib.get_private_rev_num()
+    assert p.h_revision == r
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    r2 = lib.get_private_rev_num()
+    assert r2 < 0 and r2 % 2 == 1
+    assert r != r2
+    assert p.h_revision == r
+    assert p.h_tid == lib.gettid(p) | 0    # no GC flags
+
+def test_private_with_backup():
+    p = nalloc(HDR)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    r2 = lib.get_private_rev_num()
+    assert p.h_revision != r2
+    p2 = lib.stm_write_barrier(p)
+    assert p2 == p       # does not move
+    assert p.h_revision == r2
+
+def test_get_backup_copy():
+    p = nalloc(HDR + WORD)
+    lib.setlong(p, 0, 78927812)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    org_r = p.h_revision
+    lib.setlong(p, 0, 927122)
+    assert p.h_revision == lib.get_private_rev_num()
+    pback = lib.stm_get_backup_copy(p)
+    assert pback and pback != p
+    assert pback.h_revision == org_r
+    assert pback.h_tid == p.h_tid | GCFLAG_BACKUP_COPY
+    assert lib.rawgetlong(pback, 0) == 78927812
+    assert lib.rawgetlong(p, 0) == 927122
+
+def test_protected_with_backup():
+    xxx

From noreply at buildbot.pypy.org  Wed Jun  5 15:57:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 15:57:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130605135738.A62651C3333@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65:7526350df472
Date: 2013-06-05 15:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/7526350df472/

Log:	in-progress

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -286,27 +286,33 @@
 #endif
 }
 
-static gcptr LocalizeProtected(struct tx_descriptor *d, gcptr R)
+static gcptr LocalizeProtected(struct tx_descriptor *d, gcptr P)
 {
-  assert(!(R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
+  gcptr B;
 
-  /* duplicate, and save the original R->h_revision into an extra
-     word allocated just after L */
-  assert(R->h_revision & 1);
-  gcptr L = stmgc_duplicate(R, R->h_revision);
+  assert(P->h_revision != stm_private_rev_num);
+  assert(!(P->h_tid & GCFLAG_PUBLIC));
+  assert(!(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
+  assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
+  assert(!(P->h_tid & GCFLAG_STUB));
 
-  /* cross-thread memory barrier: make sure the local object is correct
-     and has h_revision == stm_local_revision, and the extra word is
-     written as well; when it is done, and only then, then we change
-     R->h_revision */
-  smp_wmb();
-
-  R->h_revision = (revision_t)L;
-
-  gcptrlist_insert(&d->protected_with_private_copy, R);
-  AddInReadSet(d, R);
-  /*mark*/
-  return L;
+  if (P->h_revision & 1)
+    {
+      /* does not have a backup yet */
+      B = stmgc_duplicate(P, 0);
+      B->h_tid |= GCFLAG_BACKUP_COPY;
+    }
+  else
+    {
+      size_t size = stmcb_size(P);
+      B = (gcptr)P->h_revision;
+      assert(B->h_tid & GCFLAG_BACKUP_COPY);
+      memcpy(B + 1, P + 1, size - sizeof(*B));
+    }
+  assert(B->h_tid & GCFLAG_BACKUP_COPY);
+  g2l_insert(&d->private_to_backup, P, B);
+  P->h_revision = stm_private_rev_num;
+  return P;
 }
 
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
@@ -338,35 +344,25 @@
 
 gcptr stm_WriteBarrier(gcptr P)
 {
-  gcptr R, W;
+  gcptr W;
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
-  /* must normalize the situation now, otherwise we risk that
-     LocalizePublic creates a new private version of a public
-     object that has got one, attached to the equivalent stolen
-     protected object */
-  if (gcptrlist_size(&d->stolen_objects) > 0)
-    stmgc_normalize_stolen_objects();
-
-  /* XXX optimize me based on common patterns */
-  R = HeadOfRevisionChainList(d, P);
-
-  switch (stmgc_classify(R)) {
-  case K_PRIVATE:   W = R;                       break;
-  case K_PROTECTED: W = LocalizeProtected(d, R); break;
-  case K_PUBLIC:    W = LocalizePublic(d, R);    break;
-  default: abort();
-  }
-
-  if (W->h_tid & GCFLAG_WRITE_BARRIER)
-    stmgc_write_barrier(W);
+  W = LocalizeProtected(d, P);
 
   fprintf(stderr, "write_barrier: %p -> %p\n", P, W);
 
   return W;
 }
 
+gcptr stm_get_backup_copy(gcptr P)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  wlog_t *entry;
+  G2L_FIND(d->private_to_backup, P, entry, return NULL);
+  return entry->val;
+}
+
 /************************************************************/
 
 static revision_t GetGlobalCurTime(struct tx_descriptor *d)
@@ -527,6 +523,7 @@
   }
 
   gcptrlist_clear(&d->list_of_read_objects);
+  g2l_clear(&d->private_to_backup);
   stmgc_abort_transaction(d);
 
   fprintf(stderr,
@@ -590,6 +587,7 @@
     d->start_real_time.tv_nsec = -1;
   }
   assert(d->list_of_read_objects.size == 0);
+  assert(!g2l_any_entry(&d->private_to_backup));
   stmgc_start_transaction(d);
 
   d->count_reads = 1;
@@ -775,6 +773,23 @@
     }
 }
 
+void TurnPrivateWithBackupToProtected(struct tx_descriptor *d,
+                                      revision_t cur_time)
+{
+  wlog_t *item;
+  G2L_LOOP_FORWARD(d->private_to_backup, item)
+    {
+      gcptr P = item->addr;
+      gcptr B = item->val;
+      assert(P->h_revision == stm_private_rev_num);
+      assert(B->h_tid & GCFLAG_BACKUP_COPY);
+      B->h_revision = cur_time;
+      P->h_revision = (revision_t)B;
+
+    } G2L_LOOP_END;
+  g2l_clear(&d->private_to_backup);
+}
+
 void CommitTransaction(void)
 {   /* must save roots around this call */
   revision_t cur_time;
@@ -828,6 +843,8 @@
           "*************************************\n",
           (long)cur_time);
 
+  TurnPrivateWithBackupToProtected(d, cur_time);
+
   revision_t localrev = stm_private_rev_num;
   UpdateProtectedChainHeads(d, cur_time, localrev);
   smp_wmb();
@@ -1057,6 +1074,7 @@
 
     thread_descriptor = NULL;
 
+    g2l_delete(&d->private_to_backup);
     gcptrlist_delete(&d->list_of_read_objects);
     gcptrlist_delete(&d->abortinfo);
     free(d->longest_abort_info);
diff --git a/c3/et.h b/c3/et.h
--- a/c3/et.h
+++ b/c3/et.h
@@ -116,6 +116,7 @@
   unsigned int num_spinloops[SPINLOOP_REASONS];
   struct GcPtrList list_of_read_objects;
   struct GcPtrList abortinfo;
+  struct G2L private_to_backup;
   char *longest_abort_info;
   long long longest_abort_info_time;
   struct FXCache recent_reads_cache;
@@ -175,6 +176,7 @@
 gcptr stm_RepeatReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
 gcptr _stm_nonrecord_barrier(gcptr, int *);
+gcptr stm_get_backup_copy(gcptr);
 
 int DescriptorInit(void);
 void DescriptorDone(void);
diff --git a/c3/nursery.c b/c3/nursery.c
--- a/c3/nursery.c
+++ b/c3/nursery.c
@@ -180,12 +180,14 @@
     memcpy(localobj, globalobj, size);
 
     assert(!(localobj->h_tid & GCFLAG_NURSERY_MOVED));
+#if 0
     localobj->h_tid &= ~(GCFLAG_VISITED           |
                          GCFLAG_PUBLIC_TO_PRIVATE |
                          GCFLAG_PREBUILT_ORIGINAL |
                          GCFLAG_WRITE_BARRIER     |
                          GCFLAG_OLD);
     localobj->h_revision = stm_private_rev_num;
+#endif
     return localobj;
 }
 
diff --git a/c3/test/support.py b/c3/test/support.py
--- a/c3/test/support.py
+++ b/c3/test/support.py
@@ -68,6 +68,7 @@
     void stm_start_sharedlock(void);
     void stm_stop_sharedlock(void);
     void AbortTransaction(int);
+    gcptr stm_get_backup_copy(gcptr);
 
     gcptr getptr(gcptr, long);
     void setptr(gcptr, long, gcptr);
@@ -90,12 +91,15 @@
     /* some constants normally private that are useful in the tests */
     #define WORD                     ...
     #define GC_PAGE_SIZE             ...
+    #define GCFLAG_OLD               ...
     #define GCFLAG_VISITED           ...
+    #define GCFLAG_PUBLIC            ...
+    #define GCFLAG_PREBUILT_ORIGINAL ...
+    #define GCFLAG_BACKUP_COPY       ...
     #define GCFLAG_PUBLIC_TO_PRIVATE ...
-    #define GCFLAG_PREBUILT_ORIGINAL ...
     #define GCFLAG_WRITE_BARRIER     ...
     #define GCFLAG_NURSERY_MOVED     ...
-    #define GCFLAG_OLD               ...
+    #define GCFLAG_STOLEN            ...
     #define GCFLAG_STUB              ...
     #define ABRT_MANUAL              ...
     typedef struct { ...; } page_header_t;
@@ -509,3 +513,17 @@
 
 def abort_and_retry():
     lib.AbortTransaction(lib.ABRT_MANUAL)
+
+def classify(p):
+    private = p.h_revision == lib.get_private_rev_num()
+    public  = (p.h_tid & GCFLAG_PUBLIC) != 0
+    backup  = (p.h_tid & GCFLAG_BACKUP_COPY) != 0
+    assert private + public + backup <= 1
+    if private:
+        return "private"
+    if public:
+        return "public"
+    if backup:
+        return "backup"
+    else:
+        return "protected"
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -7,6 +7,8 @@
     lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
 
 def teardown_function(_):
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
     lib.stm_finalize()
 
 
@@ -16,12 +18,14 @@
     assert r < 0 and r % 2 == 1
     assert p.h_revision == r
     assert p.h_tid == lib.gettid(p) | 0    # no GC flags
+    assert classify(p) == "private"
 
 def test_write_barrier_private():
     p = nalloc(HDR)
     assert lib.stm_write_barrier(p) == p
     assert p.h_revision == lib.get_private_rev_num()
     assert p.h_tid == lib.gettid(p) | 0    # no GC flags
+    assert classify(p) == "private"
 
 def test_protected_no_backup():
     p = nalloc(HDR)
@@ -34,6 +38,7 @@
     assert r != r2
     assert p.h_revision == r
     assert p.h_tid == lib.gettid(p) | 0    # no GC flags
+    assert classify(p) == "protected"
 
 def test_private_with_backup():
     p = nalloc(HDR)
@@ -41,9 +46,11 @@
     lib.stm_begin_inevitable_transaction()
     r2 = lib.get_private_rev_num()
     assert p.h_revision != r2
+    assert classify(p) == "protected"
     p2 = lib.stm_write_barrier(p)
     assert p2 == p       # does not move
     assert p.h_revision == r2
+    assert classify(p) == "private"
 
 def test_get_backup_copy():
     p = nalloc(HDR + WORD)
@@ -59,6 +66,21 @@
     assert pback.h_tid == p.h_tid | GCFLAG_BACKUP_COPY
     assert lib.rawgetlong(pback, 0) == 78927812
     assert lib.rawgetlong(p, 0) == 927122
+    assert classify(p) == "private"
+    assert classify(pback) == "backup"
 
 def test_protected_with_backup():
-    xxx
+    p = nalloc(HDR + WORD)
+    lib.setlong(p, 0, 78927812)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    lib.setlong(p, 0, 927122)
+    pback = lib.stm_get_backup_copy(p)
+    assert pback != p
+    assert p.h_revision == lib.get_private_rev_num()
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert lib.stm_get_backup_copy(p) == ffi.NULL
+    assert classify(p) == "protected"
+    assert classify(pback) == "backup"
+    assert ffi.cast("revision_t *", p.h_revision) == pback

From noreply at buildbot.pypy.org  Wed Jun  5 16:02:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 16:02:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the test.
Message-ID: <20130605140217.3DCC81C3333@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64797:cb46a58acf67
Date: 2013-06-05 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/cb46a58acf67/

Log:	Fix the test.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -407,5 +407,4 @@
 
         log = self.run(main, [300])
         loop, = log.loops_by_id("long_op")
-        assert loop.match("""
-        """)
+        assert len(loop.ops_by_id("long_op")) == 0

From noreply at buildbot.pypy.org  Wed Jun  5 16:44:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 16:44:04 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add a passing test
Message-ID: <20130605144404.107E61C1401@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r66:48bc3f6660a8
Date: 2013-06-05 16:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/48bc3f6660a8/

Log:	Add a passing test

diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -84,3 +84,24 @@
     assert classify(p) == "protected"
     assert classify(pback) == "backup"
     assert ffi.cast("revision_t *", p.h_revision) == pback
+
+def test_protected_backup_reused():
+    p = nalloc(HDR + WORD)
+    lib.setlong(p, 0, 78927812)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    lib.setlong(p, 0, 927122)
+    pback = lib.stm_get_backup_copy(p)
+    assert pback != p
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert lib.stm_get_backup_copy(p) == ffi.NULL
+    assert classify(p) == "protected"
+    assert classify(pback) == "backup"
+    assert lib.rawgetlong(p, 0) == 927122
+    assert lib.rawgetlong(pback, 0) == 78927812    # but should not be used
+    lib.setlong(p, 0, 43891)
+    assert p.h_revision == lib.get_private_rev_num()
+    assert pback == lib.stm_get_backup_copy(p)
+    assert lib.rawgetlong(p, 0) == 43891
+    assert lib.rawgetlong(pback, 0) == 927122

From noreply at buildbot.pypy.org  Wed Jun  5 16:52:42 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 16:52:42 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add a passing test:
Message-ID: <20130605145242.291C41C1401@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64798:649d62298da1
Date: 2013-06-05 15:37 +0200
http://bitbucket.org/pypy/pypy/changeset/649d62298da1/

Log:	add a passing test:

diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -106,5 +106,6 @@
                                       'aaa': 'bbb'}
 
     def test_decode_object_nonstring_key(self):
-        pass # write me when we have numbers
-    
+        import _fastjson
+        raises(ValueError, "_fastjson.loads('{42: 43}')")
+        

From noreply at buildbot.pypy.org  Wed Jun  5 16:52:43 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 16:52:43 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: decoding arrays
Message-ID: <20130605145243.7B2BD1C1401@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64799:0fce473a480a
Date: 2013-06-05 15:53 +0200
http://bitbucket.org/pypy/pypy/changeset/0fce473a480a/

Log:	decoding arrays

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -72,6 +72,9 @@
             return self.decode_string()
         elif ch.isdigit() or ch == '-':
             return self.decode_numeric()
+        elif ch == '[':
+            self.next()
+            return self.decode_array()
         elif ch == '{':
             self.next()
             return self.decode_object()
@@ -134,6 +137,30 @@
             self._raise("Expected digit at char %d", self.i)
         return intval, count
         
+    def decode_array(self):
+        start = self.i
+        w_list = self.space.newlist([])
+        self.skip_whitespace()
+        while not self.eof():
+            ch = self.peek()
+            if ch == ']':
+                self.next()
+                return w_list
+            w_item = self.decode_any()
+            self.space.call_method(w_list, 'append', w_item)
+            self.skip_whitespace()
+            if self.eof():
+                break
+            ch = self.next()
+            if ch == ']':
+                return w_list
+            elif ch == ',':
+                pass
+            else:
+                self._raise("Unexpected '%s' when decoding array (char %d)",
+                            ch, self.i)
+        self._raise("Unterminated array starting at char %d", start)
+
 
     def decode_object(self):
         start = self.i
@@ -150,7 +177,7 @@
             if self.last_type != TYPE_STRING:
                 self._raise("Key name must be string for object starting at char %d", start)
             self.skip_whitespace()
-            ch = self.next()
+            ch = self.next() # XXX
             if ch != ':':
                 self._raise("No ':' found at char %d", self.i)
             self.skip_whitespace()
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -109,3 +109,11 @@
         import _fastjson
         raises(ValueError, "_fastjson.loads('{42: 43}')")
         
+    def test_decode_array(self):
+        import _fastjson
+        assert _fastjson.loads('[]') == []
+        assert _fastjson.loads('[  ]') == []
+        assert _fastjson.loads('[1]') == [1]
+        assert _fastjson.loads('[1, 2]') == [1, 2]
+        raises(ValueError, "_fastjson.loads('[1: 2]')")
+        raises(ValueError, "_fastjson.loads('[1, 2')")

From noreply at buildbot.pypy.org  Wed Jun  5 16:52:44 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 16:52:44 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add two tests and the corresponding
	fixes
Message-ID: <20130605145244.BCE711C1401@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64800:ca39af17b48f
Date: 2013-06-05 15:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ca39af17b48f/

Log:	add two tests and the corresponding fixes

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -177,7 +177,9 @@
             if self.last_type != TYPE_STRING:
                 self._raise("Key name must be string for object starting at char %d", start)
             self.skip_whitespace()
-            ch = self.next() # XXX
+            if self.eof():
+                break
+            ch = self.next()
             if ch != ':':
                 self._raise("No ':' found at char %d", self.i)
             self.skip_whitespace()
@@ -185,6 +187,8 @@
             w_value = self.decode_any()
             self.space.setitem(w_dict, w_name, w_value)
             self.skip_whitespace()
+            if self.eof():
+                break
             ch = self.next()
             if ch == '}':
                 return w_dict
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -104,6 +104,8 @@
         s = '{"hello": "world", "aaa": "bbb"}'
         assert _fastjson.loads(s) == {'hello': 'world',
                                       'aaa': 'bbb'}
+        raises(ValueError, _fastjson.loads, '{"key"')
+        raises(ValueError, _fastjson.loads, '{"key": 42')
 
     def test_decode_object_nonstring_key(self):
         import _fastjson

From noreply at buildbot.pypy.org  Wed Jun  5 16:52:46 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 16:52:46 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: raise the appropriate applevel
 exception if we get an invalid utf8
Message-ID: <20130605145246.22CD81C1401@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64801:86bc837fcae3
Date: 2013-06-05 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/86bc837fcae3/

Log:	raise the appropriate applevel exception if we get an invalid utf8

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -213,7 +213,7 @@
                 content_utf8 = self.getslice(start, self.i-1)
                 if bits & 0x80:
                     # the 8th bit is set, it's an utf8 strnig
-                    content_unicode = content_utf8.decode('utf-8')
+                    content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
                 else:
                     # ascii only, faster to decode
                     content_unicode = content_utf8.decode('ascii')
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -69,6 +69,12 @@
         s = r'"\u1234"'
         assert _fastjson.loads(s) == u'\u1234'
 
+    def test_invalid_utf_8(self):
+        import _fastjson
+        s = '"\xe0"' # this is an invalid UTF8 sequence inside a string
+        raises(UnicodeDecodeError, "_fastjson.loads(s)")
+
+
     def test_decode_numeric(self):
         import _fastjson
         def check(s, val):

From noreply at buildbot.pypy.org  Wed Jun  5 16:52:47 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  5 Jun 2013 16:52:47 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: parsing of null, true and false
Message-ID: <20130605145247.A045A1C1401@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64802:5e34f815e7f7
Date: 2013-06-05 16:51 +0200
http://bitbucket.org/pypy/pypy/changeset/5e34f815e7f7/

Log:	parsing of null, true and false

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -78,10 +78,47 @@
         elif ch == '{':
             self.next()
             return self.decode_object()
+        elif ch == 'n':
+            self.next()
+            return self.decode_null()
+        elif ch == 't':
+            self.next()
+            return self.decode_true()
+        elif ch == 'f':
+            self.next()
+            return self.decode_false()
         else:
             self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
                         ch, self.i)
 
+    def decode_null(self):
+        N = len('ull')
+        if (self.i+N <= len(self.s) and
+            self.next() == 'u' and
+            self.next() == 'l' and
+            self.next() == 'l'):
+            return self.space.w_None
+        self._raise("Error when decoding null at char %d", self.i)
+
+    def decode_true(self):
+        N = len('rue')
+        if (self.i+N <= len(self.s) and
+            self.next() == 'r' and
+            self.next() == 'u' and
+            self.next() == 'e'):
+            return self.space.w_True
+        self._raise("Error when decoding true at char %d", self.i)
+
+    def decode_false(self):
+        N = len('alse')
+        if (self.i+N <= len(self.s) and
+            self.next() == 'a' and
+            self.next() == 'l' and
+            self.next() == 's' and
+            self.next() == 'e'):
+            return self.space.w_False
+        self._raise("Error when decoding false at char %d", self.i)
+
     def decode_numeric(self):
         intval = self.parse_integer()
         #
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -19,6 +19,28 @@
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._fastjson": True}
 
+    def test_decode_constants(self):
+        import _fastjson
+        assert _fastjson.loads('null') is None
+        raises(ValueError, _fastjson.loads, 'nul')
+        raises(ValueError, _fastjson.loads, 'nu')
+        raises(ValueError, _fastjson.loads, 'n')
+        raises(ValueError, _fastjson.loads, 'nuXX')
+        #
+        assert _fastjson.loads('true') is True
+        raises(ValueError, _fastjson.loads, 'tru')
+        raises(ValueError, _fastjson.loads, 'tr')
+        raises(ValueError, _fastjson.loads, 't')
+        raises(ValueError, _fastjson.loads, 'trXX')
+        #
+        assert _fastjson.loads('false') is False
+        raises(ValueError, _fastjson.loads, 'fals')
+        raises(ValueError, _fastjson.loads, 'fal')
+        raises(ValueError, _fastjson.loads, 'fa')
+        raises(ValueError, _fastjson.loads, 'f')
+        raises(ValueError, _fastjson.loads, 'falXX')
+        
+
     def test_decode_string(self):
         import _fastjson
         res = _fastjson.loads('"hello"')

From noreply at buildbot.pypy.org  Wed Jun  5 16:59:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 16:59:17 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130605145917.D71111C1401@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67:baa39610ff9b
Date: 2013-06-05 16:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/baa39610ff9b/

Log:	Next test

diff --git a/c3/doc-objects.txt b/c3/doc-objects.txt
--- a/c3/doc-objects.txt
+++ b/c3/doc-objects.txt
@@ -50,7 +50,7 @@
 - original obj after GC killed the backup                 GT
 
 Public objects:
-- prebuilt object, never modified                         -1
+- prebuilt object, never modified                          1
 - other public object, never modified                     GT
 - outdated, has a protected copy              HANDLE to prot/priv copy
 - outdated, target stolen              ptr to a more recent public copy
diff --git a/c3/test/support.py b/c3/test/support.py
--- a/c3/test/support.py
+++ b/c3/test/support.py
@@ -436,6 +436,7 @@
 def palloc(size):
     "Get a ``prebuilt'' object."
     p = lib.pseudoprebuilt(size, 42 + size)
+    assert p.h_revision == 1
     return p
 
 def palloc_refs(nrefs):
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -105,3 +105,12 @@
     assert pback == lib.stm_get_backup_copy(p)
     assert lib.rawgetlong(p, 0) == 43891
     assert lib.rawgetlong(pback, 0) == 927122
+
+def test_prebuilt_is_public():
+    p = palloc(HDR)
+    assert p.h_revision == 1
+    assert p.h_tid == lib.gettid(p) | (GCFLAG_OLD |
+                                       GCFLAG_VISITED |
+                                       GCFLAG_PUBLIC |
+                                       GCFLAG_PREBUILT_ORIGINAL)
+    assert classify(p) == "public"

From noreply at buildbot.pypy.org  Wed Jun  5 16:59:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 16:59:19 +0200 (CEST)
Subject: [pypy-commit] stmgc default: public->private
Message-ID: <20130605145919.0E3F71C1401@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r68:2f624ecb97a7
Date: 2013-06-05 16:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/2f624ecb97a7/

Log:	public->private

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -317,29 +317,32 @@
 
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
-  abort();
-#if 0
   if (R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
     {
       wlog_t *entry;
       gcptr L;
       G2L_FIND(d->public_to_private, R, entry, goto not_found);
       L = entry->val;
-      assert(L->h_revision == stm_local_revision);
+      assert(L->h_revision == stm_private_rev_num);   /* private object */
       return L;
     }
-  else
-    R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
+  R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
  not_found:;
   gcptr L = stmgc_duplicate(R, 0);
-  assert(L->h_revision == stm_local_revision);
+  assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
+  assert(!(L->h_tid & GCFLAG_STOLEN));
+  assert(!(L->h_tid & GCFLAG_STUB));
+  L->h_tid &= ~(GCFLAG_OLD               |
+                GCFLAG_VISITED           |
+                GCFLAG_PUBLIC            |
+                GCFLAG_PREBUILT_ORIGINAL |
+                GCFLAG_PUBLIC_TO_PRIVATE |
+                GCFLAG_WRITE_BARRIER     |
+                0);
+  L->h_revision = stm_private_rev_num;
   g2l_insert(&d->public_to_private, R, L);
-  gcptrlist_insert(&d->public_to_young, R);
-  AddInReadSet(d, R);
-  /*mark*/
   return L;
-#endif
 }
 
 gcptr stm_WriteBarrier(gcptr P)
@@ -348,7 +351,12 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
-  W = LocalizeProtected(d, P);
+  /*P = stm_DirectReadBarrier(P);*/
+
+  if (P->h_tid & GCFLAG_PUBLIC)
+    W = LocalizePublic(d, P);
+  else
+    W = LocalizeProtected(d, P);
 
   fprintf(stderr, "write_barrier: %p -> %p\n", P, W);
 
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -114,3 +114,15 @@
                                        GCFLAG_PUBLIC |
                                        GCFLAG_PREBUILT_ORIGINAL)
     assert classify(p) == "public"
+
+def test_change_prebuilt_object():
+    p = palloc(HDR + WORD)
+    lib.rawsetlong(p, 0, 28971289)
+    flags = p.h_tid
+    assert (flags & GCFLAG_PUBLIC_TO_PRIVATE) == 0
+    assert classify(p) == "public"
+    p2 = lib.stm_write_barrier(p)
+    assert p2 != p
+    assert classify(p) == "public"
+    assert classify(p2) == "private"
+    assert p.h_tid == flags | GCFLAG_PUBLIC_TO_PRIVATE

From noreply at buildbot.pypy.org  Wed Jun  5 16:59:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 16:59:20 +0200 (CEST)
Subject: [pypy-commit] stmgc default: This test passes,
 but probably only because it's simple enough.
Message-ID: <20130605145920.1D1901C1401@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r69:a31993e9805a
Date: 2013-06-05 16:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/a31993e9805a/

Log:	This test passes, but probably only because it's simple enough.

diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -115,9 +115,8 @@
                                        GCFLAG_PREBUILT_ORIGINAL)
     assert classify(p) == "public"
 
-def test_change_prebuilt_object():
-    p = palloc(HDR + WORD)
-    lib.rawsetlong(p, 0, 28971289)
+def test_prebuilt_object_to_private():
+    p = palloc(HDR)
     flags = p.h_tid
     assert (flags & GCFLAG_PUBLIC_TO_PRIVATE) == 0
     assert classify(p) == "public"
@@ -126,3 +125,17 @@
     assert classify(p) == "public"
     assert classify(p2) == "private"
     assert p.h_tid == flags | GCFLAG_PUBLIC_TO_PRIVATE
+
+def test_commit_change_to_prebuilt_object():
+    p = palloc(HDR + WORD)
+    lib.rawsetlong(p, 0, 28971289)
+    p2 = lib.stm_write_barrier(p)
+    assert p2 != p
+    assert classify(p) == "public"
+    assert classify(p2) == "private"
+    lib.rawsetlong(p, 0, 1289222)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert classify(p) == "public"
+    assert classify(p2) == "protected"
+    assert p.h_revision == int(ffi.cast("revision_t", p2)) + 2

From noreply at buildbot.pypy.org  Wed Jun  5 17:44:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 17:44:35 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fixes
Message-ID: <20130605154435.42EDF1C1007@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r70:5c86454b6e59
Date: 2013-06-05 17:44 +0200
http://bitbucket.org/pypy/stmgc/changeset/5c86454b6e59/

Log:	Fixes

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -716,15 +716,6 @@
 #endif
       L->h_revision = new_revision;
 
-      if (is_young(L))
-        {
-          item->val = (gcptr)(((revision_t)L) | 2);
-#ifdef DUMP_EXTRA
-          fprintf(stderr, "PUBLIC-TO-PROTECTED:\n");
-          /*mark*/
-#endif
-        }
-
     } G2L_LOOP_END;
 
   smp_wmb(); /* a memory barrier: make sure the new L->h_revisions are visible
@@ -742,11 +733,12 @@
       assert(R->h_revision != localrev);
 
 #ifdef DUMP_EXTRA
-      fprintf(stderr, "%p->h_revision = %p (UpdateChainHeads2)\n",
+      fprintf(stderr, "%p->h_revision = %p | 2 (UpdateChainHeads2)\n",
               R, (gcptr)v);
       /*mark*/
 #endif
-      ACCESS_ONCE(R->h_revision) = v;
+      assert(!(v & 3));
+      ACCESS_ONCE(R->h_revision) = v | 2;
 
       if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
         {
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -133,9 +133,11 @@
     assert p2 != p
     assert classify(p) == "public"
     assert classify(p2) == "private"
-    lib.rawsetlong(p, 0, 1289222)
+    lib.rawsetlong(p2, 0, 1289222)
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
     assert classify(p) == "public"
     assert classify(p2) == "protected"
     assert p.h_revision == int(ffi.cast("revision_t", p2)) + 2
+    assert lib.rawgetlong(p, 0) == 28971289
+    assert lib.rawgetlong(p2, 0) == 1289222

From noreply at buildbot.pypy.org  Wed Jun  5 18:23:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 18:23:06 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Starting on the read barrier.
Message-ID: <20130605162306.F0E131C11B7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r71:27f0605ccd79
Date: 2013-06-05 18:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/27f0605ccd79/

Log:	Starting on the read barrier.

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -137,10 +137,9 @@
 #endif
 }
 
+#if 0
 static inline gcptr AddInReadSet(struct tx_descriptor *d, gcptr R)
 {
-  abort();
-#if 0
   fprintf(stderr, "AddInReadSet(%p)\n", R);
   d->count_reads++;
   if (!fxcache_add(&d->recent_reads_cache, R)) {
@@ -155,37 +154,22 @@
       //      return Localize(d, R);
       //  }
   return R;
+}
 #endif
-}
 
-gcptr stm_DirectReadBarrier(gcptr G)
+gcptr stm_DirectReadBarrier(gcptr P)
 {
-  abort();
-#if 0
-  gcptr R;
   struct tx_descriptor *d = thread_descriptor;
-  assert(d->active >= 1);
 
-  /* XXX optimize me based on common patterns */
-  R = HeadOfRevisionChainList(d, G);
+  if (P->h_tid & GCFLAG_PUBLIC)
+    {
+      abort();
+      /*...*/
+    }
 
-  if (R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
-    {
-      if (gcptrlist_size(&d->stolen_objects) > 0)
-        stmgc_normalize_stolen_objects();
-
-      wlog_t *entry;
-      gcptr L;
-      G2L_FIND(d->public_to_private, R, entry, goto not_found);
-      L = entry->val;
-      assert(L->h_revision == stm_local_revision);
-      return L;
-
-    not_found:;
-    }
-  R = AddInReadSet(d, R);
-  return R;
-#endif
+  gcptrlist_insert(&d->list_of_read_objects, P);
+  fxcache_add(&d->recent_reads_cache, P);
+  return P;
 }
 
 static gcptr _latest_gcptr(gcptr R)
@@ -371,6 +355,14 @@
   return entry->val;
 }
 
+gcptr stm_get_read_obj(long index)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  if (index < gcptrlist_size(&d->list_of_read_objects))
+    return d->list_of_read_objects.items[index];
+  return NULL;
+}
+
 /************************************************************/
 
 static revision_t GetGlobalCurTime(struct tx_descriptor *d)
diff --git a/c3/et.h b/c3/et.h
--- a/c3/et.h
+++ b/c3/et.h
@@ -177,6 +177,7 @@
 gcptr stm_WriteBarrier(gcptr);
 gcptr _stm_nonrecord_barrier(gcptr, int *);
 gcptr stm_get_backup_copy(gcptr);
+gcptr stm_get_read_obj(long);  /* debugging */
 
 int DescriptorInit(void);
 void DescriptorDone(void);
diff --git a/c3/lists.c b/c3/lists.c
--- a/c3/lists.c
+++ b/c3/lists.c
@@ -223,6 +223,8 @@
 
 /************************************************************/
 
+__thread char *stm_read_barrier_cache;
+
 void _fxcache_reset(struct FXCache *fxcache)
 {
     fxcache->shift = 0;
diff --git a/c3/lists.h b/c3/lists.h
--- a/c3/lists.h
+++ b/c3/lists.h
@@ -178,15 +178,17 @@
    more.
 */
 
-#define FX_ENTRIES   8192
-#define FX_TOTAL     (FX_ENTRIES * 2)
+#define FX_MASK      65535
+#define FX_ENTRIES   ((FX_MASK + 1) / sizeof(char *))
+#define FX_TOTAL     (FX_ENTRIES * 4 / 3)
 
 struct FXCache {
-    char *cache_start;
     revision_t shift;
     revision_t cache[FX_TOTAL];
 };
 
+extern __thread char *stm_read_barrier_cache;
+
 void _fxcache_reset(struct FXCache *fxcache);
 
 static inline void fxcache_clear(struct FXCache *fxcache)
@@ -194,7 +196,16 @@
     fxcache->shift++;
     if (fxcache->shift > FX_TOTAL - FX_ENTRIES)
         _fxcache_reset(fxcache);
-    fxcache->cache_start = (char *)(fxcache->cache + fxcache->shift);
+    stm_read_barrier_cache = (char *)(fxcache->cache + fxcache->shift);
+}
+
+#define FXCACHE_AT(obj)  \
+    (*(gcptr *)(stm_read_barrier_cache + ((revision_t)(obj) & FX_MASK)))
+
+static inline void fxcache_add(struct FXCache *fxcache, gcptr newobj)
+{
+    assert(stm_read_barrier_cache == (char*)(fxcache->cache + fxcache->shift));
+    FXCACHE_AT(newobj) = newobj;
 }
 
 /************************************************************/
diff --git a/c3/stmsync.c b/c3/stmsync.c
--- a/c3/stmsync.c
+++ b/c3/stmsync.c
@@ -77,12 +77,10 @@
 
 gcptr stm_read_barrier(gcptr obj)
 {
-    /* XXX inline in the caller */
-    abort();
-#if 0
-    if (UNLIKELY(obj->h_revision != stm_local_revision))
+    /* XXX inline in the caller, optimize to get the smallest code */
+    if (UNLIKELY((obj->h_revision != stm_private_rev_num) &&
+                 (FXCACHE_AT(obj) != obj)))
         obj = stm_DirectReadBarrier(obj);
-#endif
     return obj;
 }
 
diff --git a/c3/test/support.py b/c3/test/support.py
--- a/c3/test/support.py
+++ b/c3/test/support.py
@@ -69,6 +69,7 @@
     void stm_stop_sharedlock(void);
     void AbortTransaction(int);
     gcptr stm_get_backup_copy(gcptr);
+    gcptr stm_get_read_obj(long index);
 
     gcptr getptr(gcptr, long);
     void setptr(gcptr, long, gcptr);
@@ -528,3 +529,14 @@
         return "backup"
     else:
         return "protected"
+
+def list_of_read_objects():
+    result = []
+    index = 0
+    while 1:
+        p = lib.stm_get_read_obj(index)
+        if p == ffi.NULL:
+            break
+        result.append(p)
+        index += 1
+    return result
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -141,3 +141,18 @@
     assert p.h_revision == int(ffi.cast("revision_t", p2)) + 2
     assert lib.rawgetlong(p, 0) == 28971289
     assert lib.rawgetlong(p2, 0) == 1289222
+
+def test_read_barrier_private():
+    p = nalloc(HDR)
+    assert lib.stm_read_barrier(p) == p     # no effect
+    assert p.h_tid == gettid(p)
+    assert p.h_revision == lib.get_private_rev_num()
+    assert list_of_read_objects() == []
+
+def test_read_barrier_protected():
+    p = nalloc(HDR)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert list_of_read_objects() == []
+    assert lib.stm_read_barrier(p) == p     # record as a read object
+    assert list_of_read_objects() == [p]

From noreply at buildbot.pypy.org  Wed Jun  5 18:23:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 18:23:08 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Read barrier on public objects
Message-ID: <20130605162308.3345F1C11B7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r72:b9dfe1e2b66e
Date: 2013-06-05 18:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/b9dfe1e2b66e/

Log:	Read barrier on public objects

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -65,78 +65,6 @@
 
 /************************************************************/
 
-static gcptr HeadOfRevisionChainList(struct tx_descriptor *d, gcptr G)
-{
-  abort();
-#if 0
-  gcptr R = G;
-  revision_t v;
-
- retry:
-  v = ACCESS_ONCE(R->h_revision);
-  if (!(v & 1))  // "is a pointer", i.e.
-    {            //      "has a more recent revision"
-      if (v & 2)
-        {
-        old_to_young:
-          v &= ~2;
-          if (UNLIKELY(!stmgc_is_young_in(d, (gcptr)v)))
-            {
-              stmgc_public_to_foreign_protected(R);
-              goto retry;
-            }
-          R = (gcptr)v;
-          goto retry;
-        }
-
-      gcptr R_prev = R;
-      R = (gcptr)v;
-
-    retry_threelevels:
-      v = ACCESS_ONCE(R->h_revision);
-      if (!(v & 1))  // "is a pointer", i.e.
-        {            //      "has a more recent revision"
-          if (v & 2)
-            goto old_to_young;
-
-          /* we update R_prev->h_revision as a shortcut */
-          /* XXX check if this really gives a worse performance than only
-             doing this write occasionally based on a counter in d */
-          gcptr R_next = (gcptr)v;
-          if (R_next->h_revision == stm_local_revision)
-            {
-              /* must not update an older h_revision to go directly to
-                 the private copy at the end of a chain of protected
-                 objects! */
-              return R_next;
-            }
-          if (R_prev->h_tid & GCFLAG_STOLEN)
-            {
-              /* must not update the h_revision of a stolen object! */
-              R_prev = R;
-              R = R_next;
-              goto retry_threelevels;
-            }
-          R_prev->h_revision = v;
-          R = R_next;
-          goto retry;
-        }
-    }
-
-  if (UNLIKELY(v > d->start_time))   // object too recent?
-    {
-      if (v >= LOCKED)
-        {
-          SpinLoop(SPLP_LOCKED_INFLIGHT);
-          goto retry;                // spinloop until it is no longer LOCKED
-        }
-      ValidateNow(d);                  // try to move start_time forward
-      goto retry;                      // restart searching from R
-    }
-  return R;
-#endif
-}
-
 #if 0
 static inline gcptr AddInReadSet(struct tx_descriptor *d, gcptr R)
 {
@@ -163,12 +91,57 @@
 
   if (P->h_tid & GCFLAG_PUBLIC)
     {
-      abort();
-      /*...*/
+      /* follow the chained list of h_revision's as long as they are
+         regular pointers */
+      revision_t v;
+
+    retry:
+      v = ACCESS_ONCE(P->h_revision);
+      if (!(v & 1))  // "is a pointer", i.e.
+        {            //      "has a more recent revision"
+          if (v & 2)
+            {
+            old_to_young:
+              abort();
+            }
+          assert(P->h_tid & GCFLAG_PUBLIC);
+
+          gcptr P_prev = P;
+          P = (gcptr)v;
+
+          /* if we land on a P in read_barrier_cache: just return it */
+          if (FXCACHE_AT(P) == P)
+            return P;
+
+          v = ACCESS_ONCE(P->h_revision);
+          if (!(v & 1))  // "is a pointer", i.e.
+            {            //      "has a more recent revision"
+              if (v & 2)
+                goto old_to_young;
+              assert(P->h_tid & GCFLAG_PUBLIC);
+
+              /* we update P_prev->h_revision as a shortcut */
+              /* XXX check if this really gives a worse performance than only
+                 doing this write occasionally based on a counter in d */
+              P_prev->h_revision = v;
+              P = (gcptr)v;
+              goto retry;
+            }
+        }
+
+      if (UNLIKELY(v > d->start_time))   // object too recent?
+        {
+          if (v >= LOCKED)
+            {
+              SpinLoop(SPLP_LOCKED_INFLIGHT);
+              goto retry;           // spinloop until it is no longer LOCKED
+            }
+          ValidateNow(d);                  // try to move start_time forward
+          goto retry;                      // restart searching from P
+        }
     }
-
+  fxcache_add(&d->recent_reads_cache, P);
   gcptrlist_insert(&d->list_of_read_objects, P);
-  fxcache_add(&d->recent_reads_cache, P);
   return P;
 }
 
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -156,3 +156,25 @@
     assert list_of_read_objects() == []
     assert lib.stm_read_barrier(p) == p     # record as a read object
     assert list_of_read_objects() == [p]
+
+def test_read_barrier_public():
+    p = palloc(HDR)
+    assert lib.stm_read_barrier(p) == p
+    assert list_of_read_objects() == [p]
+
+def test_read_barrier_public_outdated():
+    p1 = palloc(HDR)
+    p2 = palloc(HDR)
+    p1.h_revision = ffi.cast("revision_t", p2)
+    assert lib.stm_read_barrier(p1) == p2
+    assert list_of_read_objects() == [p2]
+
+def test_read_barrier_public_shortcut():
+    p1 = palloc(HDR)
+    p2 = palloc(HDR)
+    p3 = palloc(HDR)
+    p1.h_revision = ffi.cast("revision_t", p2)
+    p2.h_revision = ffi.cast("revision_t", p3)
+    assert lib.stm_read_barrier(p1) == p3
+    assert list_of_read_objects() == [p3]
+    assert p1.h_revision == int(ffi.cast("revision_t", p3))   # shortcutted

From noreply at buildbot.pypy.org  Wed Jun  5 18:30:07 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed,  5 Jun 2013 18:30:07 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Make the test less specific
Message-ID: <20130605163007.564A61C11B7@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64803:1be2efa62157
Date: 2013-06-05 18:29 +0200
http://bitbucket.org/pypy/pypy/changeset/1be2efa62157/

Log:	Make the test less specific

diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -44,21 +44,21 @@
 
     def test_external_loop(self):
         from numpypy import arange, nditer, array
-        a = arange(6).reshape(2,3)
+        a = arange(12).reshape(2,3,2)
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop']):
             r.append(x)
             n += 1
         assert n == 1
-        assert (array(r) == [0, 1, 2, 3, 4, 5]).all()
+        assert (array(r) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]).all()
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop'], order='F'):
             r.append(x)
             n += 1
         assert n == 3
-        assert (array(r) == [[0, 3], [1, 4], [2, 5]]).all()
+        assert (array(r) == [[0, 6], [2, 8], [4, 10], [1, 7], [3, 9], [5, 11]]).all()
 
     def test_interface(self):
         from numpypy import arange, nditer, zeros

From noreply at buildbot.pypy.org  Wed Jun  5 18:49:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 18:49:08 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test.
Message-ID: <20130605164908.E70C41C01CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r73:e3da2bafef96
Date: 2013-06-05 18:44 +0200
http://bitbucket.org/pypy/stmgc/changeset/e3da2bafef96/

Log:	Next test.

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -129,6 +129,18 @@
             }
         }
 
+      if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
+        {
+          wlog_t *item;
+          G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
+
+          P = item->val;
+          assert(!(P->h_tid & GCFLAG_PUBLIC));
+          assert(P->h_revision == stm_private_rev_num);
+          return P;
+        }
+    no_private_obj:
+
       if (UNLIKELY(v > d->start_time))   // object too recent?
         {
           if (v >= LOCKED)
@@ -299,6 +311,13 @@
                 0);
   L->h_revision = stm_private_rev_num;
   g2l_insert(&d->public_to_private, R, L);
+  fprintf(stderr, "write_barrier: adding %p -> %p to public_to_private\n",
+          R, L);
+
+  /* must remove R from the read_barrier_cache, because returning R is no
+     longer a valid result */
+  fxcache_remove(&d->recent_reads_cache, R);
+
   return L;
 }
 
@@ -308,7 +327,7 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
-  /*P = stm_DirectReadBarrier(P);*/
+  P = stm_read_barrier(P);
 
   if (P->h_tid & GCFLAG_PUBLIC)
     W = LocalizePublic(d, P);
diff --git a/c3/lists.h b/c3/lists.h
--- a/c3/lists.h
+++ b/c3/lists.h
@@ -208,6 +208,12 @@
     FXCACHE_AT(newobj) = newobj;
 }
 
+static inline void fxcache_remove(struct FXCache *fxcache, gcptr oldobj)
+{
+    assert(stm_read_barrier_cache == (char*)(fxcache->cache + fxcache->shift));
+    FXCACHE_AT(oldobj) = NULL;
+}
+
 /************************************************************/
 
 #endif
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -178,3 +178,16 @@
     assert lib.stm_read_barrier(p1) == p3
     assert list_of_read_objects() == [p3]
     assert p1.h_revision == int(ffi.cast("revision_t", p3))   # shortcutted
+
+def test_read_barrier_public_to_private():
+    p = palloc(HDR)
+    p2 = lib.stm_write_barrier(p)
+    assert p2 != p
+    assert classify(p) == "public"
+    assert classify(p2) == "private"
+    assert list_of_read_objects() == [p]
+    assert p.h_tid & GCFLAG_PUBLIC
+    assert p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
+    p3 = lib.stm_read_barrier(p)
+    assert p3 == p2
+    assert list_of_read_objects() == [p]

From noreply at buildbot.pypy.org  Wed Jun  5 18:49:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 18:49:10 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add some more debugging prints
Message-ID: <20130605164910.104101C01CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74:3a3dfae01632
Date: 2013-06-05 18:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/3a3dfae01632/

Log:	Add some more debugging prints

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -85,9 +85,10 @@
 }
 #endif
 
-gcptr stm_DirectReadBarrier(gcptr P)
+gcptr stm_DirectReadBarrier(gcptr G)
 {
   struct tx_descriptor *d = thread_descriptor;
+  gcptr P = G;
 
   if (P->h_tid & GCFLAG_PUBLIC)
     {
@@ -111,7 +112,10 @@
 
           /* if we land on a P in read_barrier_cache: just return it */
           if (FXCACHE_AT(P) == P)
-            return P;
+            {
+              fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P);
+              return P;
+            }
 
           v = ACCESS_ONCE(P->h_revision);
           if (!(v & 1))  // "is a pointer", i.e.
@@ -137,6 +141,7 @@
           P = item->val;
           assert(!(P->h_tid & GCFLAG_PUBLIC));
           assert(P->h_revision == stm_private_rev_num);
+          fprintf(stderr, "read_barrier: %p -> %p public_to_private\n", G, P);
           return P;
         }
     no_private_obj:
@@ -151,6 +156,11 @@
           ValidateNow(d);                  // try to move start_time forward
           goto retry;                      // restart searching from P
         }
+      fprintf(stderr, "read_barrier: %p -> %p public\n", G, P);
+    }
+  else
+    {
+      fprintf(stderr, "read_barrier: %p -> %p protected\n", G, P);
     }
   fxcache_add(&d->recent_reads_cache, P);
   gcptrlist_insert(&d->list_of_read_objects, P);

From noreply at buildbot.pypy.org  Wed Jun  5 20:48:16 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 20:48:16 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Introduce "handles" as written down in
	doc-objects.txt
Message-ID: <20130605184816.8D54A1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75:d8c52869d637
Date: 2013-06-05 20:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/d8c52869d637/

Log:	Introduce "handles" as written down in doc-objects.txt

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -296,6 +296,7 @@
 
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
+  assert(R->h_tid & GCFLAG_PUBLIC);
   if (R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
     {
       wlog_t *entry;
@@ -726,13 +727,22 @@
       assert(!is_young(R));
       assert(R->h_revision != localrev);
 
+      /* XXX compactify and don't leak! */
+      revision_t *handle_block = stm_malloc(3 * WORD);
+      handle_block = (revision_t *)
+        ((((intptr_t)handle_block) + HANDLE_BLOCK_SIZE-1)
+         & ~(HANDLE_BLOCK_SIZE-1));
+      handle_block[0] = d->my_lock;
+      handle_block[1] = v;
+
+      revision_t w = ((revision_t)(handle_block + 1)) + 2;
+
 #ifdef DUMP_EXTRA
-      fprintf(stderr, "%p->h_revision = %p | 2 (UpdateChainHeads2)\n",
-              R, (gcptr)v);
+      fprintf(stderr, "%p->h_revision = %p (UpdateChainHeads2)\n",
+              R, (gcptr)w);
       /*mark*/
 #endif
-      assert(!(v & 3));
-      ACCESS_ONCE(R->h_revision) = v | 2;
+      ACCESS_ONCE(R->h_revision) = w;
 
       if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
         {
diff --git a/c3/et.h b/c3/et.h
--- a/c3/et.h
+++ b/c3/et.h
@@ -13,6 +13,8 @@
 
 #define LOCKED  ((INTPTR_MAX - 0xffff) | 1)
 
+#define HANDLE_BLOCK_SIZE   (2 * WORD)
+
 /* Description of the flags
  * ------------------------
  *
diff --git a/c3/test/support.py b/c3/test/support.py
--- a/c3/test/support.py
+++ b/c3/test/support.py
@@ -84,6 +84,7 @@
     gcptr pseudoprebuilt(size_t size, int tid);
     revision_t get_private_rev_num(void);
     revision_t get_start_time(void);
+    revision_t get_my_lock(void);
 
     gcptr *addr_of_thread_local(void);
     int in_nursery(gcptr);
@@ -92,6 +93,8 @@
     /* some constants normally private that are useful in the tests */
     #define WORD                     ...
     #define GC_PAGE_SIZE             ...
+    #define LOCKED                   ...
+    #define HANDLE_BLOCK_SIZE        ...
     #define GCFLAG_OLD               ...
     #define GCFLAG_VISITED           ...
     #define GCFLAG_PUBLIC            ...
@@ -118,6 +121,7 @@
     extern void stmgcpage_add_prebuilt_root(gcptr);
     extern void stm_clear_between_tests(void);
     extern revision_t get_private_rev_num(void);
+    extern local_gcpages_t *stm_local_gcpages(void);
 
     int gettid(gcptr obj)
     {
@@ -205,6 +209,11 @@
         return thread_descriptor->start_time;
     }
 
+    revision_t get_my_lock(void)
+    {
+        return thread_descriptor->my_lock;
+    }
+
     gcptr *addr_of_thread_local(void)
     {
         return &stm_thread_local_obj;
@@ -540,3 +549,11 @@
         result.append(p)
         index += 1
     return result
+
+def decode_handle(r):
+    assert (r & 3) == 2
+    p = r & ~(lib.HANDLE_BLOCK_SIZE-1)
+    my_lock = ffi.cast("revision_t *", p)[0]
+    assert my_lock >= lib.LOCKED
+    ptr = ffi.cast("gcptr *", r - 2)[0]
+    return ptr, my_lock
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -138,7 +138,7 @@
     lib.stm_begin_inevitable_transaction()
     assert classify(p) == "public"
     assert classify(p2) == "protected"
-    assert p.h_revision == int(ffi.cast("revision_t", p2)) + 2
+    assert decode_handle(p.h_revision) == (p2, lib.get_my_lock())
     assert lib.rawgetlong(p, 0) == 28971289
     assert lib.rawgetlong(p2, 0) == 1289222
 

From noreply at buildbot.pypy.org  Wed Jun  5 21:03:22 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Jun 2013 21:03:22 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Read barriers through handle objects.
Message-ID: <20130605190322.35B451C01CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r76:69fa4d0304c6
Date: 2013-06-05 21:03 +0200
http://bitbucket.org/pypy/stmgc/changeset/69fa4d0304c6/

Log:	Read barriers through handle objects.

diff --git a/c3/et.c b/c3/et.c
--- a/c3/et.c
+++ b/c3/et.c
@@ -89,22 +89,18 @@
 {
   struct tx_descriptor *d = thread_descriptor;
   gcptr P = G;
+  revision_t v;
 
   if (P->h_tid & GCFLAG_PUBLIC)
     {
       /* follow the chained list of h_revision's as long as they are
          regular pointers */
-      revision_t v;
-
     retry:
       v = ACCESS_ONCE(P->h_revision);
       if (!(v & 1))  // "is a pointer", i.e.
         {            //      "has a more recent revision"
           if (v & 2)
-            {
-            old_to_young:
-              abort();
-            }
+            goto old_to_young;
           assert(P->h_tid & GCFLAG_PUBLIC);
 
           gcptr P_prev = P;
@@ -162,9 +158,42 @@
     {
       fprintf(stderr, "read_barrier: %p -> %p protected\n", G, P);
     }
+
+ register_in_list_of_read_objects:
   fxcache_add(&d->recent_reads_cache, P);
   gcptrlist_insert(&d->list_of_read_objects, P);
   return P;
+
+ old_to_young:;
+  revision_t target_lock;
+  target_lock = *(revision_t *)(v & ~(HANDLE_BLOCK_SIZE-1));
+  if (target_lock == d->my_lock)
+    {
+      P = (gcptr)(*(revision_t *)(v - 2));
+      assert(!(P->h_tid & GCFLAG_PUBLIC));
+      if (P->h_revision == stm_private_rev_num)
+        {
+          fprintf(stderr, "read_barrier: %p -> %p handle "
+                  "private\n", G, P);
+          return P;
+        }
+      else if (FXCACHE_AT(P) == P)
+        {
+          fprintf(stderr, "read_barrier: %p -> %p handle "
+                  "protected fxcache\n", G, P);
+          return P;
+        }
+      else
+        {
+          fprintf(stderr, "read_barrier: %p -> %p handle "
+                  "protected\n", G, P);
+          goto register_in_list_of_read_objects;
+        }
+    }
+  else
+    {
+      abort();   // stealing
+    }
 }
 
 static gcptr _latest_gcptr(gcptr R)
diff --git a/c3/test/test_et.py b/c3/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c3/test/test_et.py
@@ -191,3 +191,35 @@
     p3 = lib.stm_read_barrier(p)
     assert p3 == p2
     assert list_of_read_objects() == [p]
+
+def test_read_barrier_handle_protected():
+    p = palloc(HDR)
+    p2 = lib.stm_write_barrier(p)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert classify(p) == "public"
+    assert classify(p2) == "protected"
+    assert list_of_read_objects() == []
+    p3 = lib.stm_read_barrier(p)
+    assert p3 == p2
+    assert list_of_read_objects() == [p2]
+    p4 = lib.stm_read_barrier(p)
+    assert p4 == p2
+    assert list_of_read_objects() == [p2]
+
+def test_read_barrier_handle_private():
+    p = palloc(HDR)
+    p2 = lib.stm_write_barrier(p)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    p2b = lib.stm_write_barrier(p)
+    assert p2b == p2
+    assert classify(p) == "public"
+    assert classify(p2) == "private"
+    assert list_of_read_objects() == [p2]
+    p3 = lib.stm_read_barrier(p)
+    assert p3 == p2
+    assert list_of_read_objects() == [p2]
+    p4 = lib.stm_read_barrier(p)
+    assert p4 == p2
+    assert list_of_read_objects() == [p2]

From noreply at buildbot.pypy.org  Wed Jun  5 23:47:02 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed,  5 Jun 2013 23:47:02 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix
Message-ID: <20130605214702.9907A1C1007@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64805:88ea391ffde6
Date: 2013-06-05 12:00 -0700
http://bitbucket.org/pypy/pypy/changeset/88ea391ffde6/

Log:	fix

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -462,13 +462,13 @@
         sys.flags = type(sys.flags)(flags)
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
-    sys._xoptions = dict(x.split('=', 1) if '=' in x else (x, True)
-                         for x in options['_xoptions'])
-
         if sys.flags.optimize >= 1:
             import __pypy__
             __pypy__.set_debug(False)
 
+    sys._xoptions = dict(x.split('=', 1) if '=' in x else (x, True)
+                         for x in options['_xoptions'])
+
 ##    if not we_are_translated():
 ##        for key in sorted(options):
 ##            print '%40s: %s' % (key, options[key])

From noreply at buildbot.pypy.org  Wed Jun  5 23:47:01 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed,  5 Jun 2013 23:47:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130605214701.4BDF11C01CD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64804:54af5b1ed1de
Date: 2013-06-05 11:49 -0700
http://bitbucket.org/pypy/pypy/changeset/54af5b1ed1de/

Log:	merge default

diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -119,6 +119,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,7 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -50,3 +50,5 @@
 .. branch: win32-fixes3
 Skip and fix some non-translated (own) tests for win32 builds
 
+.. branch: ctypes-byref
+Add the '_obj' attribute on ctypes pointer() and byref() objects
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,8 +2,9 @@
 # App-level version of py.py.
 # See test/test_app_main.
 
-# Missing vs CPython: -b, -d, -OO, -v, -x, -3
-"""\
+# Missing vs CPython: -b, -d, -v, -x, -3
+from __future__ import print_function, unicode_literals
+USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 -c cmd : program passed in as string (terminates option list)
@@ -12,7 +13,8 @@
 -i     : inspect interactively after running script; forces a prompt even
          if stdin does not appear to be a terminal; also PYTHONINSPECT=x
 -m mod : run library module as a script (terminates option list)
--O     : dummy optimization flag for compatibility with CPython
+-O     : skip assert statements
+-OO    : remove docstrings when importing modules in addition to -O
 -q     : don't print version and copyright messages on interactive startup
 -R     : ignored (see http://bugs.python.org/issue14621)
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
@@ -28,8 +30,6 @@
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
 """
-from __future__ import print_function, unicode_literals
-USAGE1 = __doc__
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
 Other environment variables:
@@ -465,6 +465,10 @@
     sys._xoptions = dict(x.split('=', 1) if '=' in x else (x, True)
                          for x in options['_xoptions'])
 
+        if sys.flags.optimize >= 1:
+            import __pypy__
+            __pypy__.set_debug(False)
+
 ##    if not we_are_translated():
 ##        for key in sorted(options):
 ##            print '%40s: %s' % (key, options[key])
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -246,6 +246,8 @@
         if w_len is None:
             w_len = space.len(self.w_consts)
             space.setitem(self.w_consts, w_key, w_len)
+        if space.int_w(w_len) == 0:
+            self.scope.doc_removable = False
         return space.int_w(w_len)
 
     def _make_key(self, obj):
@@ -618,6 +620,7 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+    ops.JUMP_IF_NOT_DEBUG : 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -263,6 +263,7 @@
                 start = 1
                 doc_expr.walkabout(self)
                 self.name_op("__doc__", ast.Store)
+                self.scope.doc_removable = True
             for i in range(start, len(body)):
                 body[i].walkabout(self)
             return True
@@ -435,6 +436,7 @@
     def visit_Assert(self, asrt):
         self.update_position(asrt.lineno)
         end = self.new_block()
+        self.emit_jump(ops.JUMP_IF_NOT_DEBUG, end)
         asrt.test.accept_jump_if(self, True, end)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "AssertionError")
         if asrt.msg:
@@ -1225,6 +1227,8 @@
         flags = 0
         if not self.cell_vars and not self.free_vars:
             flags |= consts.CO_NOFREE
+        if self.scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
         return flags
 
 
@@ -1244,6 +1248,8 @@
             flags |= consts.CO_VARARGS
         if scope.has_keywords_arg:
             flags |= consts.CO_VARKEYWORDS
+        if scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
         if not self.cell_vars and not self.free_vars:
             flags |= consts.CO_NOFREE
         return PythonCodeGenerator._get_code_flags(self) | flags
@@ -1260,6 +1266,7 @@
             doc_expr = None
         if doc_expr is not None:
             self.add_const(doc_expr.s)
+            self.scope.doc_removable = True
             start = 1
         else:
             self.add_const(self.space.w_None)
@@ -1339,3 +1346,9 @@
             # This happens when nobody references the cell
             self.load_const(self.space.w_None)
         self.emit_op(ops.RETURN_VALUE)
+
+    def _get_code_flags(self):
+        flags = 0
+        if self.scope.doc_removable:
+            flags |= consts.CO_KILL_DOCSTRING
+        return PythonCodeGenerator._get_code_flags(self) | flags
diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -16,6 +16,8 @@
 CO_FUTURE_PRINT_FUNCTION = 0x10000
 CO_FUTURE_UNICODE_LITERALS = 0x20000
 CO_FUTURE_BARRY_AS_BDFL = 0x40000
+#pypy specific:
+CO_KILL_DOCSTRING = 0x100000
 
 PyCF_MASK = (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT |
              CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION |
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -42,6 +42,7 @@
         self.has_free = False
         self.child_has_free = False
         self.nested = False
+        self.doc_removable = False
 
     def lookup(self, name):
         """Find the scope of identifier 'name'."""
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -842,6 +842,58 @@
         """
         self.simple_test(source, 'ok', 1)
 
+    def test_remove_docstring(self):
+        source = '"module_docstring"\n' + """if 1:
+        def f1():
+            'docstring'
+        def f2():
+            'docstring'
+            return 'docstring'
+        def f3():
+            'foo'
+            return 'bar'
+        class C1():
+            'docstring'
+        class C2():
+            __doc__ = 'docstring'
+        class C3():
+            field = 'not docstring'
+        class C4():
+            'docstring'
+            field = 'docstring'
+        """
+        code_w = compile_with_astcompiler(source, 'exec', self.space)
+        code_w.remove_docstrings(self.space)
+        dict_w = self.space.newdict();
+        code_w.exec_code(self.space, dict_w, dict_w)
+
+        yield self.check, dict_w, "f1.__doc__", None
+        yield self.check, dict_w, "f2.__doc__", 'docstring'
+        yield self.check, dict_w, "f2()", 'docstring'
+        yield self.check, dict_w, "f3.__doc__", None
+        yield self.check, dict_w, "f3()", 'bar'
+        yield self.check, dict_w, "C1.__doc__", None
+        yield self.check, dict_w, "C2.__doc__", 'docstring'
+        yield self.check, dict_w, "C3.field", 'not docstring'
+        yield self.check, dict_w, "C4.field", 'docstring'
+        yield self.check, dict_w, "C4.__doc__", 'docstring'
+        yield self.check, dict_w, "C4.__doc__", 'docstring'
+        yield self.check, dict_w, "__doc__", None
+
+    def test_assert_skipping(self):
+        space = self.space
+        mod = space.getbuiltinmodule('__pypy__')
+        w_set_debug = space.getattr(mod, space.wrap('set_debug'))
+        space.call_function(w_set_debug, space.w_False)
+
+        source = """if 1:
+        assert False
+        """
+        try:
+            self.run(source)
+        finally:
+            space.call_function(w_set_debug, space.w_True)
+
     def test_raise_from(self):
         test = """if 1:
         def f():
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -12,7 +12,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
-    CO_GENERATOR)
+    CO_GENERATOR, CO_KILL_DOCSTRING)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import compute_hash, we_are_translated
@@ -240,6 +240,13 @@
                 return w_first
         return space.w_None
 
+    def remove_docstrings(self, space):
+        if self.co_flags & CO_KILL_DOCSTRING:
+            self.co_consts_w[0] = space.w_None
+        for w_co in self.co_consts_w:
+            if isinstance(w_co, PyCode):
+                w_co.remove_docstrings(space)
+
     def _to_code(self):
         """For debugging only."""
         consts = [None] * len(self.co_consts_w)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -855,6 +855,11 @@
         self.popvalue()
         return next_instr
 
+    def JUMP_IF_NOT_DEBUG(self, jumpby, next_instr):
+        if not self.space.sys.debug:
+            next_instr += jumpby
+        return next_instr
+
     def GET_ITER(self, oparg, next_instr):
         w_iterable = self.popvalue()
         w_iterator = self.space.iter(w_iterable)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -59,6 +59,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'dictstrategy'              : 'interp_dict.dictstrategy',
+        'set_debug'                 : 'interp_magic.set_debug',
     }
 
     submodules = {
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -98,6 +98,13 @@
 def newlist_hint(space, sizehint):
     return space.newlist_hint(sizehint)
 
+ at unwrap_spec(debug=bool)
+def set_debug(space, debug):
+    space.sys.debug = debug
+    space.setitem(space.builtin.w_dict,
+                  space.wrap('__debug__'),
+                  space.wrap(debug))
+
 @unwrap_spec(estimate=int)
 def add_memory_pressure(estimate):
     rgc.add_memory_pressure(estimate)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -992,6 +992,13 @@
             if not space.is_true(space.sys.get('dont_write_bytecode')):
                 write_compiled_module(space, code_w, cpathname, mode, mtime)
 
+    try:
+        optimize = space.sys.get_flag('optimize')
+    except Exception:
+        optimize = 0
+    if optimize >= 2:
+        code_w.remove_docstrings(space)
+
     update_code_filenames(space, code_w, pathname)
     exec_code_module(space, w_mod, code_w, pathname, cpathname)
 
@@ -1086,6 +1093,13 @@
                               "Bad magic number in %s", cpathname)
     #print "loading pyc file:", cpathname
     code_w = read_compiled_module(space, cpathname, source)
+    try:
+        optimize = space.sys.get_flag('optimize')
+    except Exception:
+        optimize = 0
+    if optimize >= 2:
+        code_w.remove_docstrings(space)
+
     exec_code_module(space, w_mod, code_w, cpathname, cpathname, write_paths)
 
     return w_mod
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -13,6 +13,9 @@
 from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage
 from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.rstring import StringBuilder
+
 
 MIXIN_32 = (long_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (long_typedef,) if LONG_BIT == 64 else ()
@@ -42,7 +45,23 @@
     return func_with_new_name(new, name + "_box_new"), staticmethod(_get_dtype), func_with_new_name(descr_reduce, "descr_reduce")
 
 
-class PrimitiveBox(object):
+class Box(object):
+    _mixin_ = True
+
+    def reduce(self, space):
+        from rpython.rlib.rstring import StringBuilder
+        from rpython.rtyper.lltypesystem import rffi, lltype
+
+        numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(numpypy, MixedModule)
+        multiarray = numpypy.get("multiarray")
+        assert isinstance(multiarray, MixedModule)
+        scalar = multiarray.get("scalar")
+
+        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
+        return ret
+
+class PrimitiveBox(Box):
     _mixin_ = True
 
     def __init__(self, value):
@@ -54,27 +73,19 @@
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self.value)
 
-    def reduce(self, space):
-        from rpython.rlib.rstring import StringBuilder
-        from rpython.rtyper.lltypesystem import rffi, lltype
-
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
-        assert isinstance(multiarray, MixedModule)
-        scalar = multiarray.get("scalar")
-
+    def raw_str(self):
         value = lltype.malloc(rffi.CArray(lltype.typeOf(self.value)), 1, flavor="raw")
         value[0] = self.value
 
         builder = StringBuilder()
         builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.value)))
+        ret = builder.build()
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(builder.build())])])
         lltype.free(value, flavor="raw")
+
         return ret
 
-class ComplexBox(object):
+class ComplexBox(Box):
     _mixin_ = True
 
     def __init__(self, real, imag=0.):
@@ -90,25 +101,17 @@
     def convert_imag_to(self, dtype):
         return dtype.box(self.imag)
 
-    def reduce(self, space):
-        from rpython.rlib.rstring import StringBuilder
-        from rpython.rtyper.lltypesystem import rffi, lltype
-
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
-        assert isinstance(multiarray, MixedModule)
-        scalar = multiarray.get("scalar")
-
+    def raw_str(self):
         value = lltype.malloc(rffi.CArray(lltype.typeOf(self.real)), 2, flavor="raw")
         value[0] = self.real
         value[1] = self.imag
 
         builder = StringBuilder()
         builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.real)) * 2)
+        ret = builder.build()
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(builder.build())])])
         lltype.free(value, flavor="raw")
+
         return ret
 
 class W_GenericBox(W_Root):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -786,6 +786,7 @@
         from rpython.rtyper.lltypesystem import rffi
         from rpython.rlib.rstring import StringBuilder
         from pypy.interpreter.mixedmodule import MixedModule
+        from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
 
         numpypy = space.getbuiltinmodule("_numpypy")
         assert isinstance(numpypy, MixedModule)
@@ -796,7 +797,14 @@
         parameters = space.newtuple([space.gettypefor(W_NDimArray), space.newtuple([space.wrap(0)]), space.wrap("b")])
 
         builder = StringBuilder()
-        builder.append_charpsize(self.implementation.get_storage(), self.implementation.get_storage_size())
+        if isinstance(self.implementation, SliceArray):
+            iter = self.implementation.create_iter()
+            while not iter.done():
+                box = iter.getitem()
+                builder.append(box.raw_str())
+                iter.next()
+        else:
+            builder.append_charpsize(self.implementation.get_storage(), self.implementation.get_storage_size())
 
         state = space.newtuple([
                 space.wrap(1),      # version
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1801,6 +1801,13 @@
         pickled_data = dumps(a)
         assert (loads(pickled_data) == a).all()
 
+    def test_pickle_slice(self):
+        from cPickle import loads, dumps
+        import numpypy as numpy
+
+        a = numpy.arange(10.).reshape((5, 2))[::2]
+        assert (loads(dumps(a)) == a).all()
+
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import numpypy
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -8,7 +8,12 @@
 
 def test_bigint():
     from rpython.rlib.rbigint import rbigint
+    assert not pypypolicy.look_inside_function(rbigint.eq.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.ne.im_func)
     assert not pypypolicy.look_inside_function(rbigint.lt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.le.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.gt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.ge.im_func)
 
 def test_rlocale():
     from rpython.rlib.rlocale import setlocale
diff --git a/pypy/module/pypyjit/test_pypy_c/test_intbound.py b/pypy/module/pypyjit/test_pypy_c/test_intbound.py
--- a/pypy/module/pypyjit/test_pypy_c/test_intbound.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_intbound.py
@@ -231,6 +231,7 @@
         assert loop.match("""
             i8 = int_lt(i6, 300)
             guard_true(i8, descr=...)
+            guard_not_invalidated?
             i10 = int_lshift(i6, 1)
             i12 = int_add_ovf(i5, 1)
             guard_no_overflow(descr=...)
@@ -253,6 +254,7 @@
         assert loop.match("""
             i8 = int_lt(i6, 300)
             guard_true(i8, descr=...)
+            guard_not_invalidated?
             i10 = int_add_ovf(i5, 8)
             guard_no_overflow(descr=...)
             i12 = int_add(i6, 1)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -74,7 +74,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_mixed_type_loop(self):
         def main(n):
             i = 0.0
@@ -94,7 +93,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_cached_pure_func_of_equal_fields(self):
         def main(n):
             class A(object):
@@ -196,7 +194,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_chain_of_guards(self):
         src = """
         class A(object):
@@ -220,7 +217,6 @@
         loops = log.loops_by_filename(self.filepath)
         assert len(loops) == 1
 
-
     def test_unpack_iterable_non_list_tuple(self):
         def main(n):
             import array
@@ -258,7 +254,6 @@
             jump(..., descr=...)
         """)
 
-
     def test_dont_trace_every_iteration(self):
         def main(a, b):
             i = sa = 0
@@ -289,7 +284,6 @@
         assert log.result == 300 * (-10 % -20)
         assert log.jit_summary.tracing_no == 1
 
-
     def test_overflow_checking(self):
         """
         This test only checks that we get the expected result, not that any
@@ -298,7 +292,8 @@
         def main():
             import sys
             def f(a,b):
-                if a < 0: return -1
+                if a < 0:
+                    return -1
                 return a-b
             #
             total = sys.maxint - 2147483647
@@ -309,7 +304,6 @@
         #
         self.run_and_check(main, [])
 
-
     def test_global(self):
         log = self.run("""
         i = 0
@@ -404,3 +398,13 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(log.loops_by_id("exc_info")) == 1
+
+    def test_long_comparison(self):
+        def main(n):
+            while n:
+                12345L > 123L  # ID: long_op
+                n -= 1
+
+        log = self.run(main, [300])
+        loop, = log.loops_by_id("long_op")
+        assert len(loop.ops_by_id("long_op")) == 0
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -7,6 +7,7 @@
 
 class Module(MixedModule):
     """Sys Builtin Module. """
+    _immutable_fields_ = ["defaultencoding?", "debug?"]
 
     def __init__(self, space, w_name):
         """NOT_RPYTHON""" # because parent __init__ isn't
@@ -17,6 +18,7 @@
         self.w_default_encoder = None
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
+        self.debug = True
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))', 
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
@@ -108,6 +108,13 @@
 
             py.test.raises(TypeError, delitem, p, 0)
 
+    def test_byref(self):
+        for ct, pt in zip(ctype_types, python_types):
+            i = ct(42)
+            p = byref(i)
+            assert type(p._obj) is ct
+            assert p._obj.value == 42
+
     def test_pointer_to_pointer(self):
         x = c_int(32)
         y = c_int(42)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -800,6 +800,9 @@
         self.popvalue()
         return next_instr
 
+    def JUMP_IF_NOT_DEBUG(self, target, next_instr):
+        return next_instr
+
     def GET_ITER(self, oparg, next_instr):
         w_iterable = self.popvalue()
         w_iterator = self.space.iter(w_iterable)
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1911,6 +1911,29 @@
                                      [BoxPtr(x)], 'int').value
         assert res == -19
 
+    def test_cast_int_to_float(self):
+        if not self.cpu.supports_floats:
+            py.test.skip("requires floats")
+        for x in [-10, -1, 0, 3, 42, sys.maxint-1]:
+            res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
+                                         [BoxInt(x)],  'float').value
+            assert longlong.getrealfloat(res) == float(x)
+            res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
+                                         [ConstInt(x)],  'float').value
+            assert longlong.getrealfloat(res) == float(x)
+
+    def test_cast_float_to_int(self):
+        if not self.cpu.supports_floats:
+            py.test.skip("requires floats")
+        for x in [-24.23, -5.3, 0.0, 3.1234, 11.1, 0.1]:
+            v = longlong.getfloatstorage(x)
+            res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
+                                         [BoxFloat(v)],  'int').value
+            assert res == int(x)
+            res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
+                                         [ConstFloat(v)],  'int').value
+            assert res == int(x)
+
     def test_convert_float_bytes(self):
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")
diff --git a/rpython/jit/codewriter/policy.py b/rpython/jit/codewriter/policy.py
--- a/rpython/jit/codewriter/policy.py
+++ b/rpython/jit/codewriter/policy.py
@@ -39,8 +39,6 @@
         return True # look into everything by default
 
     def _reject_function(self, func):
-        if hasattr(func, '_jit_look_inside_'):
-            return not func._jit_look_inside_
         # explicitly elidable functions are always opaque
         if getattr(func, '_elidable_function_', False):
             return True
@@ -58,8 +56,11 @@
         except AttributeError:
             see_function = True
         else:
-            see_function = (self.look_inside_function(func) and not
-                            self._reject_function(func))
+            if hasattr(func, '_jit_look_inside_'):
+                see_function = func._jit_look_inside_   # override guessing
+            else:
+                see_function = (self.look_inside_function(func) and not
+                                self._reject_function(func))
             contains_loop = contains_loop and not getattr(
                     func, '_jit_unroll_safe_', False)
 
diff --git a/rpython/jit/codewriter/test/test_policy.py b/rpython/jit/codewriter/test/test_policy.py
--- a/rpython/jit/codewriter/test/test_policy.py
+++ b/rpython/jit/codewriter/test/test_policy.py
@@ -65,6 +65,20 @@
     graph = support.getgraph(h, [5])
     assert not JitPolicy().look_inside_graph(graph)
 
+def test_look_inside():
+    def h1(x):
+        return x + 1
+    @jit.look_inside    # force True, even if look_inside_function() thinks not
+    def h2(x):
+        return x + 2
+    class MyPolicy(JitPolicy):
+        def look_inside_function(self, func):
+            return False
+    graph1 = support.getgraph(h1, [5])
+    graph2 = support.getgraph(h2, [5])
+    assert not MyPolicy().look_inside_graph(graph1)
+    assert MyPolicy().look_inside_graph(graph2)
+
 def test_loops():
     def g(x):
         i = 0
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -2,12 +2,14 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.history import BoxInt, ConstInt
+
 MAXINT = maxint
 MININT = -maxint - 1
 
+
 class IntBound(object):
     _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
-    
+
     def __init__(self, lower, upper):
         self.has_upper = True
         self.has_lower = True
@@ -29,7 +31,7 @@
 
     def make_lt(self, other):
         return self.make_le(other.add(-1))
- 
+
     def make_ge(self, other):
         if other.has_lower:
             if not self.has_lower or other.lower > self.lower:
@@ -86,7 +88,7 @@
                 r = True
 
         return r
-    
+
     def add(self, offset):
         res = self.clone()
         try:
@@ -101,7 +103,7 @@
 
     def mul(self, value):
         return self.mul_bound(IntBound(value, value))
-    
+
     def add_bound(self, other):
         res = self.clone()
         if other.has_upper:
@@ -115,7 +117,7 @@
             try:
                 res.lower = ovfcheck(res.lower + other.lower)
             except OverflowError:
-                res.has_lower = False            
+                res.has_lower = False
         else:
             res.has_lower = False
         return res
@@ -133,7 +135,7 @@
             try:
                 res.lower = ovfcheck(res.lower - other.upper)
             except OverflowError:
-                res.has_lower = False            
+                res.has_lower = False
         else:
             res.has_lower = False
         return res
@@ -196,7 +198,6 @@
         else:
             return IntUnbounded()
 
-
     def contains(self, val):
         if self.has_lower and val < self.lower:
             return False
@@ -216,7 +217,7 @@
         elif self.has_upper:
             return False
         return True
-        
+
     def __repr__(self):
         if self.has_lower:
             l = '%d' % self.lower
@@ -249,7 +250,7 @@
             guards.append(op)
             op = ResOperation(rop.GUARD_TRUE, [res], None)
             guards.append(op)
-    
+
 
 class IntUpperBound(IntBound):
     def __init__(self, upper):
@@ -285,7 +286,7 @@
         self._raise()
     def make_constant(self, value):
         self._raise()
-    def intersect(self, other):        
+    def intersect(self, other):
         self._raise()
 
 def min4(t):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -83,6 +83,13 @@
     func._jit_look_inside_ = False
     return func
 
+def look_inside(func):
+    """ Make sure the JIT traces inside decorated function, even
+    if the rest of the module is not visible to the JIT
+    """
+    func._jit_look_inside_ = True
+    return func
+
 def unroll_safe(func):
     """ JIT can safely unroll loops in this function and this will
     not lead to code explosion
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -190,9 +190,9 @@
             
         
     @staticmethod
+    @jit.elidable
     def frombool(b):
-        # This function is marked as pure, so you must not call it and
-        # then modify the result.
+        # You must not call this function and then modify the result.
         if b:
             return ONERBIGINT
         return NULLRBIGINT
@@ -251,6 +251,7 @@
         return _decimalstr_to_bigint(s)
 
     @staticmethod
+    @jit.elidable
     def frombytes(s, byteorder, signed):
         if byteorder not in ('big', 'little'):
             raise InvalidEndiannessError()
@@ -383,9 +384,11 @@
     def tolonglong(self):
         return _AsLongLong(self)
 
+    @jit.look_inside
     def tobool(self):
         return self.sign != 0
 
+    @jit.elidable
     def touint(self):
         if self.sign == -1:
             raise ValueError("cannot convert negative integer to unsigned int")
@@ -410,13 +413,16 @@
             raise ValueError("cannot convert negative integer to unsigned int")
         return _AsULonglong_ignore_sign(self)
 
+    @jit.elidable
     def uintmask(self):
         return _AsUInt_mask(self)
 
+    @jit.elidable
     def ulonglongmask(self):
         """Return r_ulonglong(self), truncating."""
         return _AsULonglong_mask(self)
 
+    @jit.elidable
     def tofloat(self):
         return _AsDouble(self)
 
@@ -448,6 +454,7 @@
             i += 1
         return True
 
+    @jit.look_inside
     def ne(self, other):
         return not self.eq(other)
 
@@ -486,12 +493,15 @@
             i -= 1
         return False
 
+    @jit.look_inside
     def le(self, other):
         return not other.lt(self)
 
+    @jit.look_inside
     def gt(self, other):
         return other.lt(self)
 
+    @jit.look_inside
     def ge(self, other):
         return not self.lt(other)
 
@@ -592,6 +602,7 @@
             
         return div
 
+    @jit.look_inside
     def div(self, other):
         return self.floordiv(other)
 
@@ -792,14 +803,17 @@
             z = z.sub(c)
         return z
 
+    @jit.elidable
     def neg(self):
         return rbigint(self._digits, -self.sign, self.size)
 
+    @jit.elidable
     def abs(self):
         if self.sign != -1:
             return self
         return rbigint(self._digits, 1, self.size)
 
+    @jit.elidable
     def invert(self): #Implement ~x as -(x + 1)
         if self.sign == 0:
             return ONENEGATIVERBIGINT
@@ -909,12 +923,14 @@
     def or_(self, other):
         return _bitwise(self, '|', other)
 
+    @jit.elidable
     def oct(self):
         if self.sign == 0:
             return '0L'
         else:
             return _format(self, BASE8, '0', 'L')
 
+    @jit.elidable
     def hex(self):
         return _format(self, BASE16, '0x', 'L')
 
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -332,7 +332,7 @@
             newstr = s2.malloc(len1 + len2)
             newstr.copy_contents_from_str(s1, newstr, 0, 0, len1)
         else:
-            newstr = s1.malloc(len1 + len2)            
+            newstr = s1.malloc(len1 + len2)
             newstr.copy_contents(s1, newstr, 0, 0, len1)
         if typeOf(s2) == Ptr(STR):
             newstr.copy_contents_from_str(s2, newstr, 0, len1, len2)

From noreply at buildbot.pypy.org  Wed Jun  5 23:47:03 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed,  5 Jun 2013 23:47:03 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: apply stdlib changes from e953dfbc7f0a
Message-ID: <20130605214703.D1AAA1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64806:739db6f6edc0
Date: 2013-06-05 14:44 -0700
http://bitbucket.org/pypy/pypy/changeset/739db6f6edc0/

Log:	apply stdlib changes from e953dfbc7f0a

diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -182,5 +182,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/3/test/test_code.py b/lib-python/3/test/test_code.py
--- a/lib-python/3/test/test_code.py
+++ b/lib-python/3/test/test_code.py
@@ -81,7 +81,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 >>> def keywordonly_args(a,b,*,k1):
diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -218,7 +218,7 @@
 Kw-only arguments: 0
 Number of locals:  1
 Stack size:        4
-Flags:             OPTIMIZED, NEWLOCALS, NOFREE
+Flags:             OPTIMIZED, NEWLOCALS, NOFREE, 0x100000
 Constants:
    0: %r
    1: '__func__'

From noreply at buildbot.pypy.org  Wed Jun  5 23:47:05 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed,  5 Jun 2013 23:47:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: have recursive Cache build failures
 trigger a more specific RuntimeError
Message-ID: <20130605214705.2596C1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r64807:e258a99e3784
Date: 2013-06-05 14:46 -0700
http://bitbucket.org/pypy/pypy/changeset/e258a99e3784/

Log:	have recursive Cache build failures trigger a more specific
	RuntimeError

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -917,7 +917,8 @@
 
     try:
         optimize = space.sys.get_flag('optimize')
-    except Exception:
+    except RuntimeError:
+        # during bootstrapping
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)
@@ -1018,7 +1019,8 @@
     code_w = read_compiled_module(space, cpathname, source)
     try:
         optimize = space.sys.get_flag('optimize')
-    except Exception:
+    except RuntimeError:
+        # during bootstrapping
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)
diff --git a/rpython/rlib/cache.py b/rpython/rlib/cache.py
--- a/rpython/rlib/cache.py
+++ b/rpython/rlib/cache.py
@@ -44,8 +44,8 @@
                 return self.content[key]
             except KeyError:
                 if key in self._building:
-                    raise Exception, "%s recursive building of %r" % (
-                        self, key)
+                    raise RuntimeError("%s recursive building of %r" %
+                                       (self, key))
                 self._building[key] = True
                 try:
                     result = self._build(key)

From noreply at buildbot.pypy.org  Wed Jun  5 23:48:58 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed,  5 Jun 2013 23:48:58 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: (amaury) oops, don't double wrap
Message-ID: <20130605214858.34FD41C01CD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64808:162c471d831c
Date: 2013-06-05 14:48 -0700
http://bitbucket.org/pypy/pypy/changeset/162c471d831c/

Log:	(amaury) oops, don't double wrap

diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -24,7 +24,7 @@
                                space.newtuple([w_filename,
                                                space.wrap(self.lineno),
                                                space.wrap(self.offset),
-                                               space.wrap(w_text),
+                                               w_text,
                                                space.wrap(self.lastlineno)])])
 
     def __str__(self):

From noreply at buildbot.pypy.org  Thu Jun  6 10:54:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  6 Jun 2013 10:54:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: The front-end never generates pure
 operations with all-constant
Message-ID: <20130606085440.5B9EE1C101F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64809:fc0d2316ad35
Date: 2013-06-06 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/fc0d2316ad35/

Log:	The front-end never generates pure operations with all-constant
	arguments. The x86 backend doesn't support them.

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1918,9 +1918,10 @@
             res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
                                          [BoxInt(x)],  'float').value
             assert longlong.getrealfloat(res) == float(x)
-            res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
-                                         [ConstInt(x)],  'float').value
-            assert longlong.getrealfloat(res) == float(x)
+            # --- the front-end never generates CAST_INT_TO_FLOAT(Const)
+            #res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
+            #                             [ConstInt(x)],  'float').value
+            #assert longlong.getrealfloat(res) == float(x)
 
     def test_cast_float_to_int(self):
         if not self.cpu.supports_floats:
@@ -1930,9 +1931,10 @@
             res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
                                          [BoxFloat(v)],  'int').value
             assert res == int(x)
-            res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
-                                         [ConstFloat(v)],  'int').value
-            assert res == int(x)
+            # --- the front-end never generates CAST_FLOAT_TO_INT(Const)
+            #res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
+            #                             [ConstFloat(v)],  'int').value
+            #assert res == int(x)
 
     def test_convert_float_bytes(self):
         if not self.cpu.supports_floats:

From noreply at buildbot.pypy.org  Thu Jun  6 10:57:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  6 Jun 2013 10:57:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: Revert 7e3fde300fa3: the rebuild is
 needed in some cases. The test
Message-ID: <20130606085703.601241C101F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64810:c48d25dd8c61
Date: 2013-06-06 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/c48d25dd8c61/

Log:	Revert 7e3fde300fa3: the rebuild is needed in some cases. The test
	fails right now on linux64.

diff --git a/pypy/module/test_lib_pypy/test_os_wait.py b/pypy/module/test_lib_pypy/test_os_wait.py
--- a/pypy/module/test_lib_pypy/test_os_wait.py
+++ b/pypy/module/test_lib_pypy/test_os_wait.py
@@ -1,7 +1,7 @@
 # Generates the resource cache
-#from __future__ import absolute_import
-#from lib_pypy.ctypes_config_cache import rebuild
-#rebuild.rebuild_one('resource.ctc.py')
+from __future__ import absolute_import
+from lib_pypy.ctypes_config_cache import rebuild
+rebuild.rebuild_one('resource.ctc.py')
 
 import os
 

From noreply at buildbot.pypy.org  Thu Jun  6 11:14:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  6 Jun 2013 11:14:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Attempt to get rid of the occasional
 failure of test_methodcache.py.
Message-ID: <20130606091435.8D8C41C0135@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64811:9c8f04b5d3a9
Date: 2013-06-06 11:10 +0200
http://bitbucket.org/pypy/pypy/changeset/9c8f04b5d3a9/

Log:	Attempt to get rid of the occasional failure of test_methodcache.py.

diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -4,7 +4,28 @@
 class AppTestMethodCaching(test_typeobject.AppTestTypeObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
 
+    def setup_class(cls):
+        # This is for the following tests, which are a bit fragile and
+        # historically have been failing once in a while.  With this hack,
+        # they are run up to 5 times in a row, saving the frame of the
+        # failed attempt.  This means occasional collisions should work
+        # differently during the retry.
+        cls.w_retry = cls.space.appexec([], """():
+            def retry(run):
+                keepalive = []
+                for i in range(4):
+                    try:
+                        return run()
+                    except AssertionError:
+                        import sys
+                        keepalive.append(sys.exc_info())
+                return run()
+            return retry
+        """)
+
     def test_mix_classes(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -25,6 +46,8 @@
         assert sum(cache_counter) == 30
 
     def test_class_that_cannot_be_cached(self):
+      @self.retry
+      def run():
         import __pypy__
         class X:
             pass
@@ -50,6 +73,8 @@
         assert sum(cache_counter) == 20
  
     def test_change_methods(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -88,6 +113,8 @@
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -107,6 +134,8 @@
         assert sum(cache_counter) == 30
   
     def test_many_names(self):
+      @self.retry
+      def run():
         import __pypy__
         for j in range(20):
             class A(object):
@@ -160,6 +189,8 @@
         assert e.foo == 3
 
     def test_custom_metaclass(self):
+      @self.retry
+      def run():
         import __pypy__
         for j in range(20):
             class MetaA(type):
@@ -181,6 +212,8 @@
             raise AssertionError("cache_counter = %r" % (cache_counter,))
 
     def test_mutate_class(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             x = 1

From noreply at buildbot.pypy.org  Thu Jun  6 11:18:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  6 Jun 2013 11:18:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: Comment
Message-ID: <20130606091858.C81D11C06EC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64812:109953b1b160
Date: 2013-06-06 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/109953b1b160/

Log:	Comment

diff --git a/pypy/module/test_lib_pypy/test_os_wait.py b/pypy/module/test_lib_pypy/test_os_wait.py
--- a/pypy/module/test_lib_pypy/test_os_wait.py
+++ b/pypy/module/test_lib_pypy/test_os_wait.py
@@ -1,4 +1,4 @@
-# Generates the resource cache
+# Generates the resource cache (it might be there already, but maybe not)
 from __future__ import absolute_import
 from lib_pypy.ctypes_config_cache import rebuild
 rebuild.rebuild_one('resource.ctc.py')

From noreply at buildbot.pypy.org  Thu Jun  6 13:56:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  6 Jun 2013 13:56:08 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Move the code to a new directory "c4"
	as I need it.
Message-ID: <20130606115608.2569A1C13A7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r77:9baaba7aa86d
Date: 2013-06-06 13:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/9baaba7aa86d/

Log:	Move the code to a new directory "c4" as I need it.

diff --git a/c3/atomic_ops.h b/c4/atomic_ops.h
rename from c3/atomic_ops.h
rename to c4/atomic_ops.h
diff --git a/c3/dbgmem.c b/c4/dbgmem.c
rename from c3/dbgmem.c
rename to c4/dbgmem.c
diff --git a/c3/dbgmem.h b/c4/dbgmem.h
rename from c3/dbgmem.h
rename to c4/dbgmem.h
diff --git a/c3/doc-objects.txt b/c4/doc-objects.txt
rename from c3/doc-objects.txt
rename to c4/doc-objects.txt
diff --git a/c3/doc-stmgc.txt b/c4/doc-stmgc.txt
rename from c3/doc-stmgc.txt
rename to c4/doc-stmgc.txt
diff --git a/c3/et.c b/c4/et.c
rename from c3/et.c
rename to c4/et.c
--- a/c3/et.c
+++ b/c4/et.c
@@ -192,10 +192,13 @@
     }
   else
     {
-      abort();   // stealing
+      /* stealing */
+      fprintf(stderr, "read_barrier: %p -> stealing %p...", G, (gcptr)v);
+      abort();
     }
 }
 
+#if 0
 static gcptr _latest_gcptr(gcptr R)
 {
   /* don't use, for tests only */
@@ -270,6 +273,7 @@
       return obj;
     }
 }
+#endif
 
 #if 0
 void *stm_DirectReadBarrierFromR(void *G1, void *R_Container1, size_t offset)
@@ -294,6 +298,14 @@
 #endif
 }
 
+gcptr stmgc_duplicate(gcptr P)
+{
+  size_t size = stmcb_size(P);
+  gcptr L = stm_malloc(size);
+  memcpy(L, P, size);
+  return L;
+}
+
 static gcptr LocalizeProtected(struct tx_descriptor *d, gcptr P)
 {
   gcptr B;
@@ -307,7 +319,7 @@
   if (P->h_revision & 1)
     {
       /* does not have a backup yet */
-      B = stmgc_duplicate(P, 0);
+      B = stmgc_duplicate(P);
       B->h_tid |= GCFLAG_BACKUP_COPY;
     }
   else
@@ -338,7 +350,7 @@
   R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
  not_found:;
-  gcptr L = stmgc_duplicate(R, 0);
+  gcptr L = stmgc_duplicate(R);
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(L->h_tid & GCFLAG_STOLEN));
   assert(!(L->h_tid & GCFLAG_STUB));
@@ -556,7 +568,7 @@
 
   gcptrlist_clear(&d->list_of_read_objects);
   g2l_clear(&d->private_to_backup);
-  stmgc_abort_transaction(d);
+  abort();//stmgc_abort_transaction(d);
 
   fprintf(stderr,
           "\n"
@@ -620,7 +632,7 @@
   }
   assert(d->list_of_read_objects.size == 0);
   assert(!g2l_any_entry(&d->private_to_backup));
-  stmgc_start_transaction(d);
+  assert(!g2l_any_entry(&d->public_to_private));
 
   d->count_reads = 1;
   fxcache_clear(&d->recent_reads_cache);
@@ -750,10 +762,10 @@
       gcptr R = item->addr;
       revision_t v = (revision_t)item->val;
 
+      assert(R->h_tid & GCFLAG_PUBLIC);
       assert(R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
       assert(!(R->h_tid & GCFLAG_NURSERY_MOVED));
       assert(!(R->h_tid & GCFLAG_STOLEN));
-      assert(!is_young(R));
       assert(R->h_revision != localrev);
 
       /* XXX compactify and don't leak! */
@@ -773,6 +785,7 @@
 #endif
       ACCESS_ONCE(R->h_revision) = w;
 
+#if 0
       if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
         {
           /* cannot possibly get here more than once for a given value of R */
@@ -781,12 +794,14 @@
           pthread_mutex_unlock(&mutex_prebuilt_gcroots);
           /*mark*/
         }
+#endif
 
     } G2L_LOOP_END;
 
   g2l_clear(&d->public_to_private);
 }
 
+#if 0
 void UpdateProtectedChainHeads(struct tx_descriptor *d, revision_t cur_time,
                                revision_t localrev)
 {
@@ -805,6 +820,7 @@
       L->h_revision = new_revision;
     }
 }
+#endif
 
 void TurnPrivateWithBackupToProtected(struct tx_descriptor *d,
                                       revision_t cur_time)
@@ -829,7 +845,7 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
-  stmgc_stop_transaction(d);
+  spinlock_acquire(d->collection_lock, 'C');  /* committing */
   AcquireLocks(d);
 
   if (is_inevitable(d))
@@ -851,10 +867,10 @@
           if (cur_time & 1)
             {                    // there is another inevitable transaction
               CancelLocks(d);
-              stmgc_suspend_commit_transaction(d);
+              spinlock_release(d->collection_lock);
               inev_mutex_acquire();   // wait until released
               inev_mutex_release();
-              stmgc_stop_transaction(d);
+              spinlock_acquire(d->collection_lock, 'C');
               AcquireLocks(d);
               continue;
             }
@@ -879,8 +895,8 @@
   TurnPrivateWithBackupToProtected(d, cur_time);
 
   revision_t localrev = stm_private_rev_num;
-  UpdateProtectedChainHeads(d, cur_time, localrev);
-  smp_wmb();
+  //UpdateProtectedChainHeads(d, cur_time, localrev);
+  //smp_wmb();
 
   revision_t newrev = -(cur_time + 1);
   assert(newrev & 1);
@@ -890,7 +906,7 @@
 
   UpdateChainHeads(d, cur_time, localrev);
 
-  stmgc_committed_transaction(d);
+  spinlock_release(d->collection_lock);
   d->num_commits++;
   d->active = 0;
   stm_stop_sharedlock();
@@ -1107,6 +1123,7 @@
 
     thread_descriptor = NULL;
 
+    g2l_delete(&d->public_to_private);
     g2l_delete(&d->private_to_backup);
     gcptrlist_delete(&d->list_of_read_objects);
     gcptrlist_delete(&d->abortinfo);
diff --git a/c3/et.h b/c4/et.h
rename from c3/et.h
rename to c4/et.h
--- a/c3/et.h
+++ b/c4/et.h
@@ -13,6 +13,7 @@
 
 #define LOCKED  ((INTPTR_MAX - 0xffff) | 1)
 
+#define WORD                sizeof(gcptr)
 #define HANDLE_BLOCK_SIZE   (2 * WORD)
 
 /* Description of the flags
@@ -100,11 +101,13 @@
 #define SPINLOOP_REASONS      4
 
 struct tx_descriptor {
-  NURSERY_FIELDS_DECL
-  local_gcpages_t *local_gcpages;
   jmp_buf *setjmp_buf;
   revision_t start_time;
   revision_t my_lock;
+  revision_t collection_lock;
+  gcptr *shadowstack;
+  gcptr **shadowstack_end_ref;
+
   long atomic;   /* 0 = not atomic, > 0 atomic */
   unsigned long count_reads;
   unsigned long reads_size_limit;        /* see should_break_tr. */
@@ -119,6 +122,7 @@
   struct GcPtrList list_of_read_objects;
   struct GcPtrList abortinfo;
   struct G2L private_to_backup;
+  struct G2L public_to_private;
   char *longest_abort_info;
   long long longest_abort_info_time;
   struct FXCache recent_reads_cache;
diff --git a/c3/fprintcolor.c b/c4/fprintcolor.c
rename from c3/fprintcolor.c
rename to c4/fprintcolor.c
diff --git a/c3/fprintcolor.h b/c4/fprintcolor.h
rename from c3/fprintcolor.h
rename to c4/fprintcolor.h
diff --git a/c3/lists.c b/c4/lists.c
rename from c3/lists.c
rename to c4/lists.c
diff --git a/c3/lists.h b/c4/lists.h
rename from c3/lists.h
rename to c4/lists.h
diff --git a/c3/stmgc.h b/c4/stmgc.h
rename from c3/stmgc.h
rename to c4/stmgc.h
diff --git a/c3/stmimpl.h b/c4/stmimpl.h
rename from c3/stmimpl.h
rename to c4/stmimpl.h
--- a/c3/stmimpl.h
+++ b/c4/stmimpl.h
@@ -30,8 +30,6 @@
 #include "fprintcolor.h"
 #include "lists.h"
 #include "dbgmem.h"
-#include "nursery.h"
-#include "gcpage.h"
 #include "et.h"
 #include "stmsync.h"
 
diff --git a/c3/stmsync.c b/c4/stmsync.c
rename from c3/stmsync.c
rename to c4/stmsync.c
--- a/c3/stmsync.c
+++ b/c4/stmsync.c
@@ -59,19 +59,19 @@
 {
     int r = DescriptorInit();
     assert(r == 1);
-    stmgc_init_tls();
+    //stmgc_init_tls();
     init_shadowstack();
-    stmgcpage_init_tls();
+    //stmgcpage_init_tls();
     BeginInevitableTransaction();
 }
 
 void stm_finalize(void)
 {
-    stmgc_minor_collect();   /* force everything out of the nursery */
+    //stmgc_minor_collect();   /* force everything out of the nursery */
     CommitTransaction();
-    stmgcpage_done_tls();
+    //stmgcpage_done_tls();
     done_shadowstack();
-    stmgc_done_tls();
+    //stmgc_done_tls();
     DescriptorDone();
 }
 
@@ -95,18 +95,16 @@
 
 gcptr stm_allocate(size_t size, unsigned long tid)
 {
-    gcptr result = stm_allocate_object_of_size(size);
+    gcptr result = stm_malloc(size);
     assert(tid == (tid & STM_USER_TID_MASK));
     result->h_tid = tid;
+    result->h_revision = stm_private_rev_num;
     return result;
 }
 
 gcptr _stm_allocate_old(size_t size, unsigned long tid)
 {
-    gcptr result = _stm_allocate_object_of_size_old(size);
-    assert(tid == (tid & STM_USER_TID_MASK));
-    result->h_tid = tid | GCFLAG_OLD;
-    return result;
+    abort();
 }
 
 /************************************************************/
@@ -237,12 +235,12 @@
 {
     int err = pthread_rwlock_rdlock(&rwlock_shared);
     assert(err == 0);
-    assert(stmgc_nursery_hiding(thread_descriptor, 0));
+    //assert(stmgc_nursery_hiding(thread_descriptor, 0));
 }
 
 void stm_stop_sharedlock(void)
 {
-    assert(stmgc_nursery_hiding(thread_descriptor, 1));
+    //assert(stmgc_nursery_hiding(thread_descriptor, 1));
     int err = pthread_rwlock_unlock(&rwlock_shared);
     assert(err == 0);
 }
@@ -303,3 +301,15 @@
        which takes priority here */
     stm_start_sharedlock();
 }
+
+/************************************************************/
+
+void stm_clear_between_tests(void)
+{
+    fprintf(stderr, "\n"
+            "===============================================================\n"
+            "========================[  START  ]============================\n"
+            "===============================================================\n"
+            "\n");
+    //gcptrlist_clear(&stm_prebuilt_gcroots);
+}
diff --git a/c3/stmsync.h b/c4/stmsync.h
rename from c3/stmsync.h
rename to c4/stmsync.h
--- a/c3/stmsync.h
+++ b/c4/stmsync.h
@@ -11,4 +11,6 @@
 
 void stm_possible_safe_point(void);
 
+void stm_clear_between_tests(void);
+
 #endif
diff --git a/c3/test/support.py b/c4/test/support.py
rename from c3/test/support.py
rename to c4/test/support.py
--- a/c3/test/support.py
+++ b/c4/test/support.py
@@ -5,11 +5,11 @@
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 header_files = [os.path.join(parent_dir, _n) for _n in
-                "et.h lists.h nursery.h gcpage.h "
+                "et.h lists.h "
                 "stmsync.h dbgmem.h fprintcolor.h "
                 "stmgc.h stmimpl.h atomic_ops.h".split()]
 source_files = [os.path.join(parent_dir, _n) for _n in
-                "et.c lists.c nursery.c gcpage.c "
+                "et.c lists.c "
                 "stmsync.c dbgmem.c fprintcolor.c".split()]
 
 _pycache_ = os.path.join(parent_dir, 'test', '__pycache__')
@@ -40,7 +40,7 @@
     #define PREBUILT_FLAGS         ...
     #define PREBUILT_REVISION      ...
 
-    gcptr stm_allocate_object_of_size(size_t size);
+    //gcptr stm_allocate_object_of_size(size_t size);
     gcptr stm_allocate(size_t size, unsigned long tid);
     void stm_push_root(gcptr);
     gcptr stm_pop_root(void);
@@ -55,15 +55,15 @@
     void stm_set_transaction_length(long length_max);
 
     /* extra non-public code */
-    gcptr stmgcpage_malloc(size_t size);
-    void stmgcpage_free(gcptr obj);
-    long stmgcpage_count(int quantity);
-    void stmgcpage_possibly_major_collect(int);
+    //gcptr stmgcpage_malloc(size_t size);
+    //void stmgcpage_free(gcptr obj);
+    //long stmgcpage_count(int quantity);
+    //void stmgcpage_possibly_major_collect(int);
     revision_t stm_global_cur_time(void);
-    void stmgcpage_add_prebuilt_root(gcptr);
+    //void stmgcpage_add_prebuilt_root(gcptr);
     void stm_clear_between_tests(void);
-    void stmgc_minor_collect(void);
-    gcptr _stm_nonrecord_barrier(gcptr, int *);
+    //void stmgc_minor_collect(void);
+    //gcptr _stm_nonrecord_barrier(gcptr, int *);
     int stm_dbgmem_is_active(void *p, int allow_outside);
     void stm_start_sharedlock(void);
     void stm_stop_sharedlock(void);
@@ -86,8 +86,8 @@
     revision_t get_start_time(void);
     revision_t get_my_lock(void);
 
-    gcptr *addr_of_thread_local(void);
-    int in_nursery(gcptr);
+    //gcptr *addr_of_thread_local(void);
+    //int in_nursery(gcptr);
     void stm_initialize_tests(int max_aborts);
 
     /* some constants normally private that are useful in the tests */
@@ -106,22 +106,20 @@
     #define GCFLAG_STOLEN            ...
     #define GCFLAG_STUB              ...
     #define ABRT_MANUAL              ...
-    typedef struct { ...; } page_header_t;
+    //typedef struct { ...; } page_header_t;
 ''')
 
 lib = ffi.verify(r'''
     #include "stmgc.h"
     #include "stmimpl.h"
 
-    extern gcptr stmgcpage_malloc(size_t size);
-    extern void stmgcpage_free(gcptr obj);
-    extern long stmgcpage_count(int quantity);
-    extern void stmgcpage_possibly_major_collect(int);
+    //extern gcptr stmgcpage_malloc(size_t size);
+    //extern void stmgcpage_free(gcptr obj);
+    //extern long stmgcpage_count(int quantity);
+    //extern void stmgcpage_possibly_major_collect(int);
     extern revision_t stm_global_cur_time(void);
-    extern void stmgcpage_add_prebuilt_root(gcptr);
-    extern void stm_clear_between_tests(void);
+    //extern void stmgcpage_add_prebuilt_root(gcptr);
     extern revision_t get_private_rev_num(void);
-    extern local_gcpages_t *stm_local_gcpages(void);
 
     int gettid(gcptr obj)
     {
@@ -214,17 +212,17 @@
         return thread_descriptor->my_lock;
     }
 
-    gcptr *addr_of_thread_local(void)
+    /*gcptr *addr_of_thread_local(void)
     {
         return &stm_thread_local_obj;
-    }
+    }*/
 
-    int in_nursery(gcptr obj)
+    /*int in_nursery(gcptr obj)
     {
         assert(stm_dbgmem_is_active(obj, 1));
         struct tx_descriptor *d = thread_descriptor;
         return (d->nursery <= (char*)obj && ((char*)obj) < d->nursery_end);
-    }
+    }*/
 
     void stm_initialize_tests(int max_aborts)
     {
@@ -275,7 +273,7 @@
 
 HDR = ffi.sizeof("struct stm_object_s")
 WORD = lib.WORD
-PAGE_ROOM = lib.GC_PAGE_SIZE - ffi.sizeof("page_header_t")
+#PAGE_ROOM = lib.GC_PAGE_SIZE - ffi.sizeof("page_header_t")
 for name in lib.__dict__:
     if name.startswith('GCFLAG_') or name.startswith('PREBUILT_'):
         globals()[name] = getattr(lib, name)
@@ -416,7 +414,7 @@
     lib.settid(p, 42 + size)
     return p
 
-ofree = lib.stmgcpage_free
+#ofree = lib.stmgcpage_free
 
 def oalloc_refs(nrefs):
     "Allocate an 'old' object, i.e. outside any nursery, with nrefs pointers"
diff --git a/c3/test/test_et.py b/c4/test/test_et.py
rename from c3/test/test_et.py
rename to c4/test/test_et.py
--- a/c3/test/test_et.py
+++ b/c4/test/test_et.py
@@ -223,3 +223,15 @@
     p4 = lib.stm_read_barrier(p)
     assert p4 == p2
     assert list_of_read_objects() == [p2]
+
+def test_stealing_protected_without_backup():
+    p = palloc(HDR + WORD)
+    def f1(r):
+        lib.setlong(p, 0, 2782172)
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        r.set(2)
+    def f2(r):
+        r.wait(2)
+        assert lib.getlong(p, 0) == 2782172
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Thu Jun  6 17:22:33 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu,  6 Jun 2013 17:22:33 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Refactor the way nditer iterates
Message-ID: <20130606152233.E4E2D1C13A7@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64813:8c3a4fc396d3
Date: 2013-06-06 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/8c3a4fc396d3/

Log:	Refactor the way nditer iterates

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -5,10 +5,41 @@
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                              shape_agreement_multiple)
-from pypy.module.micronumpy.iter import MultiDimViewIterator
+from pypy.module.micronumpy.iter import MultiDimViewIterator, SliceIterator
 from pypy.module.micronumpy import support
 from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
 
+class AbstractIterator(object):
+    def done(self):
+        raise NotImplementedError("Abstract Class")
+
+    def next(self):
+        raise NotImplementedError("Abstract Class")
+
+    def getitem(self, array):
+        raise NotImplementedError("Abstract Class")
+
+class IteratorMixin(object):
+    _mixin_ = True
+    def __init__(self, it, op_flags):
+        self.it = it
+        self.op_flags = op_flags
+
+    def done(self):
+        return self.it.done()
+
+    def next(self):
+        self.it.next()
+
+    def getitem(self, space, array):
+        return self.op_flags.get_it_item(space, array, self.it)
+
+class BoxIterator(IteratorMixin):
+    pass
+
+class SliceIterator(IteratorMixin):
+    pass
+
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
     ret = []
     if space.is_w(w_op_flags, space.w_None):
@@ -53,6 +84,13 @@
     #it.dtype.setitem(res, 0, it.getitem())
     return W_NDimArray(res)
 
+def get_readonly_slice(space, array, it):
+    #XXX Not readonly
+    return W_NDimArray(it.getslice())
+
+def get_readwrite_slice(space, array, it):
+    return W_NDimArray(it.getslice())
+
 def parse_op_flag(space, lst):
     op_flag = OpFlag()
     for w_item in lst:
@@ -191,11 +229,11 @@
         self.iters=[]
         self.shape = iter_shape = shape_agreement_multiple(space, self.seq)
         if self.external_loop:
-            xxx find longest contiguous shape
+            #XXX find longest contiguous shape
             iter_shape = iter_shape[1:]
         for i in range(len(self.seq)):
-            self.iters.append(get_iter(space, self.order,
-                            self.seq[i].implementation, iter_shape))
+            self.iters.append(BoxIterator(get_iter(space, self.order,
+                            self.seq[i].implementation, iter_shape), self.op_flags[i]))
 
     def descr_iter(self, space):
         return space.wrap(self)
@@ -220,8 +258,7 @@
             raise OperationError(space.w_StopIteration, space.w_None)
         res = []
         for i in range(len(self.iters)):
-            res.append(self.op_flags[i].get_it_item(space, self.seq[i],
-                                                    self.iters[i]))
+            res.append(self.iters[i].getitem(space, self.seq[i]))
             self.iters[i].next()
         if len(res) <2:
             return res[0]
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -32,13 +32,13 @@
 shape dimension
   which is back 25 and forward 1,
   which is x.strides[1] * (x.shape[1] - 1) + x.strides[0]
-so if we precalculate the overflow backstride as 
+so if we precalculate the overflow backstride as
 [x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))]
 we can go faster.
 All the calculations happen in next()
 
 next_skip_x() tries to do the iteration for a number of steps at once,
-but then we cannot gaurentee that we only overflow one single shape 
+but then we cannot gaurentee that we only overflow one single shape
 dimension, perhaps we could overflow times in one big step.
 """
 
@@ -266,6 +266,30 @@
     def reset(self):
         self.offset %= self.size
 
+class SliceIterator(object):
+    def __init__(self, arr, stride, backstride, shape, dtype=None):
+        self.step = 0
+        self.arr = arr
+        self.stride = stride
+        self.backstride = backstride
+        self.shape = shape
+        if dtype is None:
+            dtype = arr.implementation.dtype
+        self.dtype = dtype
+        self._done = False
+
+    def done():
+        return self._done
+
+    def next():
+        self.step += self.arr.implementation.dtype.get_size()
+        if self.step == self.backstride - self.implementation.dtype.get_size():
+            self._done = True
+
+    def getslice(self):
+        from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
+        return SliceArray(self.step, [self.stride], [self.backstride], self.shape, self.arr.implementation, self.arr, self.dtype)
+
 class AxisIterator(base.BaseArrayIterator):
     def __init__(self, array, shape, dim, cumultative):
         self.shape = shape
@@ -288,7 +312,7 @@
         self.dim = dim
         self.array = array
         self.dtype = array.dtype
-        
+
     def setitem(self, elem):
         self.dtype.setitem(self.array, self.offset, elem)
 

From noreply at buildbot.pypy.org  Thu Jun  6 20:26:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  6 Jun 2013 20:26:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue1501: minor English fixes
Message-ID: <20130606182615.2BB351C3386@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64814:da8f1e729f4d
Date: 2013-06-06 20:25 +0200
http://bitbucket.org/pypy/pypy/changeset/da8f1e729f4d/

Log:	issue1501: minor English fixes

diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:
 
 * Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
 

From noreply at buildbot.pypy.org  Fri Jun  7 12:14:55 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Fri,  7 Jun 2013 12:14:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20130607101455.832931C0135@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: py3k
Changeset: r64815:eb7f8348ff30
Date: 2013-06-07 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/eb7f8348ff30/

Log:	hg merge default

diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:
 
 * Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -994,7 +994,8 @@
 
     try:
         optimize = space.sys.get_flag('optimize')
-    except Exception:
+    except RuntimeError:
+        # during bootstrapping
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)
@@ -1095,7 +1096,8 @@
     code_w = read_compiled_module(space, cpathname, source)
     try:
         optimize = space.sys.get_flag('optimize')
-    except Exception:
+    except RuntimeError:
+        # during bootstrapping
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)
diff --git a/pypy/module/test_lib_pypy/test_os_wait.py b/pypy/module/test_lib_pypy/test_os_wait.py
--- a/pypy/module/test_lib_pypy/test_os_wait.py
+++ b/pypy/module/test_lib_pypy/test_os_wait.py
@@ -1,6 +1,7 @@
-# Generates the resource cache
-#from lib_pypy.ctypes_config_cache import rebuild
-#rebuild.rebuild_one('resource.ctc.py')
+# Generates the resource cache (it might be there already, but maybe not)
+from __future__ import absolute_import
+from lib_pypy.ctypes_config_cache import rebuild
+rebuild.rebuild_one('resource.ctc.py')
 
 import os
 
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -4,7 +4,28 @@
 class AppTestMethodCaching(test_typeobject.AppTestTypeObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
 
+    def setup_class(cls):
+        # This is for the following tests, which are a bit fragile and
+        # historically have been failing once in a while.  With this hack,
+        # they are run up to 5 times in a row, saving the frame of the
+        # failed attempt.  This means occasional collisions should work
+        # differently during the retry.
+        cls.w_retry = cls.space.appexec([], """():
+            def retry(run):
+                keepalive = []
+                for i in range(4):
+                    try:
+                        return run()
+                    except AssertionError:
+                        import sys
+                        keepalive.append(sys.exc_info())
+                return run()
+            return retry
+        """)
+
     def test_mix_classes(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -32,6 +53,8 @@
         # calling space.str_w, which .encode('ascii') the string, thus
         # creating new strings all the time. The problem should be solved when
         # we implement proper unicode identifiers in py3k
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -70,6 +93,8 @@
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -89,6 +114,8 @@
         assert sum(cache_counter) == 30
   
     def test_many_names(self):
+      @self.retry
+      def run():
         import __pypy__
         laste = None
         for j in range(20):
@@ -143,6 +170,8 @@
         assert e.foo == 3
 
     def test_custom_metaclass(self):
+      @self.retry
+      def run():
         import __pypy__
         for j in range(20):
             class MetaA(type):
@@ -164,6 +193,8 @@
             raise AssertionError("cache_counter = %r" % (cache_counter,))
 
     def test_mutate_class(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             x = 1
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1918,9 +1918,10 @@
             res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
                                          [BoxInt(x)],  'float').value
             assert longlong.getrealfloat(res) == float(x)
-            res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
-                                         [ConstInt(x)],  'float').value
-            assert longlong.getrealfloat(res) == float(x)
+            # --- the front-end never generates CAST_INT_TO_FLOAT(Const)
+            #res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
+            #                             [ConstInt(x)],  'float').value
+            #assert longlong.getrealfloat(res) == float(x)
 
     def test_cast_float_to_int(self):
         if not self.cpu.supports_floats:
@@ -1930,9 +1931,10 @@
             res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
                                          [BoxFloat(v)],  'int').value
             assert res == int(x)
-            res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
-                                         [ConstFloat(v)],  'int').value
-            assert res == int(x)
+            # --- the front-end never generates CAST_FLOAT_TO_INT(Const)
+            #res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
+            #                             [ConstFloat(v)],  'int').value
+            #assert res == int(x)
 
     def test_convert_float_bytes(self):
         if not self.cpu.supports_floats:
diff --git a/rpython/rlib/cache.py b/rpython/rlib/cache.py
--- a/rpython/rlib/cache.py
+++ b/rpython/rlib/cache.py
@@ -44,8 +44,8 @@
                 return self.content[key]
             except KeyError:
                 if key in self._building:
-                    raise Exception, "%s recursive building of %r" % (
-                        self, key)
+                    raise RuntimeError("%s recursive building of %r" %
+                                       (self, key))
                 self._building[key] = True
                 try:
                     result = self._build(key)

From noreply at buildbot.pypy.org  Fri Jun  7 12:15:04 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Fri,  7 Jun 2013 12:15:04 +0200 (CEST)
Subject: [pypy-commit] pypy py3k-memoryview: Fix test_array.py.
Message-ID: <20130607101504.E9A731C1015@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: py3k-memoryview
Changeset: r64817:da366d307364
Date: 2013-06-07 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/da366d307364/

Log:	Fix test_array.py.

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -231,7 +231,7 @@
         bytes representation.
         """
         cbuf = self._charbuf_start()
-        s = rffi.charpsize2str(cbuf, self.len * self.itemsize)
+        s = rffi.charpsize2str(cbuf, self.len * self.itemsize_)
         self._charbuf_stop()
         return self.space.wrapbytes(s)
 
@@ -257,15 +257,15 @@
         machine values, as if it had been read from a file using the
         fromfile() method).
         """
-        if len(s) % self.itemsize != 0:
+        if len(s) % self.itemsize_ != 0:
             msg = 'string length not a multiple of item size'
             raise OperationError(self.space.w_ValueError, self.space.wrap(msg))
         oldlen = self.len
-        new = len(s) / self.itemsize
+        new = len(s) / self.itemsize_
         self.setlen(oldlen + new)
         cbuf = self._charbuf_start()
         for i in range(len(s)):
-            cbuf[oldlen * self.itemsize + i] = s[i]
+            cbuf[oldlen * self.itemsize_ + i] = s[i]
         self._charbuf_stop()
 
     @unwrap_spec(n=int)
@@ -276,14 +276,14 @@
         array.  Also called as read.
         """
         try:
-            size = ovfcheck(self.itemsize * n)
+            size = ovfcheck(self.itemsize_ * n)
         except OverflowError:
             raise MemoryError
         w_item = space.call_method(w_f, 'read', space.wrap(size))
         item = space.bytes_w(w_item)
         if len(item) < size:
-            n = len(item) % self.itemsize
-            elems = max(0, len(item) - (len(item) % self.itemsize))
+            n = len(item) % self.itemsize_
+            elems = max(0, len(item) - (len(item) % self.itemsize_))
             if n != 0:
                 item = item[0:elems]
             self.descr_frombytes(space, item)
@@ -393,7 +393,7 @@
         rffi.c_memcpy(
             rffi.cast(rffi.VOIDP, w_a._buffer_as_unsigned()),
             rffi.cast(rffi.VOIDP, self._buffer_as_unsigned()),
-            self.len * self.itemsize
+            self.len * self.itemsize_
         )
         return w_a
 
@@ -403,18 +403,18 @@
         Byteswap all items of the array.  If the items in the array are not 1, 2,
         4, or 8 bytes in size, RuntimeError is raised.
         """
-        if self.itemsize not in [1, 2, 4, 8]:
+        if self.itemsize_ not in [1, 2, 4, 8]:
             msg = "byteswap not supported for this array"
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
         bytes = self._charbuf_start()
-        tmp = [bytes[0]] * self.itemsize
-        for start in range(0, self.len * self.itemsize, self.itemsize):
-            stop = start + self.itemsize - 1
-            for i in range(self.itemsize):
+        tmp = [bytes[0]] * self.itemsize_
+        for start in range(0, self.len * self.itemsize_, self.itemsize_):
+            stop = start + self.itemsize_ - 1
+            for i in range(self.itemsize_):
                 tmp[i] = bytes[start + i]
-            for i in range(self.itemsize):
+            for i in range(self.itemsize_):
                 bytes[stop - i] = tmp[i]
         self._charbuf_stop()
 

From noreply at buildbot.pypy.org  Fri Jun  7 12:15:03 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Fri,  7 Jun 2013 12:15:03 +0200 (CEST)
Subject: [pypy-commit] pypy py3k-memoryview: hg merge py3k
Message-ID: <20130607101503.93DA31C055C@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: py3k-memoryview
Changeset: r64816:5fe7421e014c
Date: 2013-06-07 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/5fe7421e014c/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 26889 lines

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -1,30 +1,16 @@
-"""Provide access to Python's configuration information.  The specific
-configuration variables available depend heavily on the platform and
-configuration.  The values may be retrieved using
-get_config_var(name), and the list of variables is available via
-get_config_vars().keys().  Additional convenience functions are also
-available.
-
-Written by:   Fred L. Drake, Jr.
-Email:        <fdrake at acm.org>
-"""
-
-__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
-
-import sys
-
 
 # The content of this file is redirected from
 # sysconfig_cpython or sysconfig_pypy.
+# All underscore names are imported too, because
+# people like to use undocumented sysconfig._xxx
+# directly.
 
+import sys
 if '__pypy__' in sys.builtin_module_names:
-    from distutils.sysconfig_pypy import *
-    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
-    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+    from distutils import sysconfig_pypy as _sysconfig_module
 else:
-    from distutils.sysconfig_cpython import *
-    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
-    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
+    from distutils import sysconfig_cpython as _sysconfig_module
+globals().update(_sysconfig_module.__dict__)
 
 _USE_CLANG = None
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -9,7 +9,7 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
 import os
 import re
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -1,9 +1,17 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -49,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -71,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,21 +134,25 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
 }
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
+}
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
@@ -164,7 +168,7 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+    return _levelToName.get(level, ("Level %s" % level))
 
 def addLevelName(level, levelName):
     """
@@ -174,8 +178,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -183,9 +187,9 @@
     if isinstance(level, int):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
@@ -277,7 +281,7 @@
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - long(ct)) * 1000
+        self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads and thread:
             self.thread = thread.get_ident()
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -156,7 +156,7 @@
         h = klass(*args)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -187,7 +187,7 @@
     opts = cp.options(sectname)
     if "level" in opts:
         level = cp.get(sectname, "level")
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = cp.get(sectname, "handlers")
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,11 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    self._sock._decref_socketios()
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_codecs.py b/lib-python/2.7/test/test_codecs.py
--- a/lib-python/2.7/test/test_codecs.py
+++ b/lib-python/2.7/test/test_codecs.py
@@ -2,7 +2,11 @@
 import unittest
 import codecs
 import locale
-import sys, StringIO, _testcapi
+import sys, StringIO
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 class Queue(object):
     """
@@ -1387,7 +1391,7 @@
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
-            if encoding not in broken_incremental_coders:
+            if encoding not in broken_incremental_coders and _testcapi:
                 # check incremental decoder/encoder (fetched via the Python
                 # and C API) and iterencode()/iterdecode()
                 try:
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -65,7 +65,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
         finally:
             logging._releaseLock()
 
@@ -97,8 +98,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -7,7 +7,8 @@
 import subprocess
 from copy import copy, deepcopy
 
-from test.test_support import run_unittest, TESTFN, unlink, get_attribute
+from test.test_support import (run_unittest, TESTFN, unlink, get_attribute,
+                               import_module)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -236,7 +237,10 @@
 
     def test_get_config_h_filename(self):
         config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
+        # import_module skips the test when the CPython C Extension API
+        # appears to not be supported
+        self.assertTrue(os.path.isfile(config_h) or
+                        not import_module('_testcapi'), config_h)
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -1,6 +1,9 @@
 """Test cases for traceback module"""
 
-from _testcapi import traceback_print
+try:
+    from _testcapi import traceback_print
+except ImportError:
+    traceback_print = None
 from StringIO import StringIO
 import sys
 import unittest
@@ -176,6 +179,8 @@
 class TracebackFormatTests(unittest.TestCase):
 
     def test_traceback_format(self):
+        if traceback_print is None:
+            raise unittest.SkipTest('Requires _testcapi')
         try:
             raise KeyError('blah')
         except KeyError:
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -1609,7 +1609,10 @@
         self.assertEqual("{}".format(u), '__unicode__ overridden')
 
     def test_encode_decimal(self):
-        from _testcapi import unicode_encodedecimal
+        try:
+            from _testcapi import unicode_encodedecimal
+        except ImportError:
+            raise unittest.SkipTest('Requires _testcapi')
         self.assertEqual(unicode_encodedecimal(u'123'),
                          b'123')
         self.assertEqual(unicode_encodedecimal(u'\u0663.\u0661\u0664'),
diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -4,7 +4,7 @@
 modules (currently limited to C extensions, should accommodate C++
 extensions ASAP)."""
 
-import sys, os, re
+import sys, os, re, imp
 from distutils.core import Command
 from distutils.errors import *
 from distutils.sysconfig import customize_compiler, get_python_version
@@ -35,6 +35,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext(Command):
 
@@ -671,10 +676,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols(self, ext):
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -1,9 +1,17 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -49,18 +57,14 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
+    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
     global _config_vars
     _config_vars = g
@@ -70,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py
--- a/lib-python/3/distutils/unixccompiler.py
+++ b/lib-python/3/distutils/unixccompiler.py
@@ -134,7 +134,7 @@
         executables['ranlib'] = ["ranlib"]
         executables['linker_so'] += ['-undefined', 'dynamic_lookup']
 
-        for k, v in executables.iteritems():
+        for k, v in executables.items():
             if v and v[0] == 'cc':
                 v += ['-arch', arch]
 
diff --git a/lib-python/3/importlib/test/extension/test_case_sensitivity.py b/lib-python/3/importlib/test/extension/test_case_sensitivity.py
--- a/lib-python/3/importlib/test/extension/test_case_sensitivity.py
+++ b/lib-python/3/importlib/test/extension/test_case_sensitivity.py
@@ -9,6 +9,7 @@
 @util.case_insensitive_tests
 class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
 
+    @ext_util.skip_unless__testcapi
     def find_module(self):
         good_name = ext_util.NAME
         bad_name = good_name.upper()
diff --git a/lib-python/3/importlib/test/extension/test_finder.py b/lib-python/3/importlib/test/extension/test_finder.py
--- a/lib-python/3/importlib/test/extension/test_finder.py
+++ b/lib-python/3/importlib/test/extension/test_finder.py
@@ -8,6 +8,7 @@
 
     """Test the finder for extension modules."""
 
+    @util.skip_unless__testcapi
     def find_module(self, fullname):
         importer = _bootstrap._FileFinder(util.PATH,
                                           _bootstrap._ExtensionFinderDetails())
diff --git a/lib-python/3/importlib/test/extension/test_loader.py b/lib-python/3/importlib/test/extension/test_loader.py
--- a/lib-python/3/importlib/test/extension/test_loader.py
+++ b/lib-python/3/importlib/test/extension/test_loader.py
@@ -11,6 +11,7 @@
 
     """Test load_module() for extension modules."""
 
+    @ext_util.skip_unless__testcapi
     def load_module(self, fullname):
         loader = _bootstrap._ExtensionFileLoader(ext_util.NAME,
                                                 ext_util.FILEPATH)
diff --git a/lib-python/3/importlib/test/extension/util.py b/lib-python/3/importlib/test/extension/util.py
--- a/lib-python/3/importlib/test/extension/util.py
+++ b/lib-python/3/importlib/test/extension/util.py
@@ -1,6 +1,7 @@
 import imp
 import os
 import sys
+import unittest
 
 PATH = None
 EXT = None
@@ -27,3 +28,8 @@
 except StopIteration:
     pass
 del _file_exts
+
+
+def skip_unless__testcapi(func):
+    msg = "Requires the CPython C Extension API ({!r} module)".format(NAME)
+    return unittest.skipUnless(PATH, msg)(func)
diff --git a/lib-python/3/logging/__init__.py b/lib-python/3/logging/__init__.py
--- a/lib-python/3/logging/__init__.py
+++ b/lib-python/3/logging/__init__.py
@@ -129,20 +129,22 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
+}
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
 }
 
 def getLevelName(level):
@@ -159,7 +161,7 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+    return _levelToName.get(level, ("Level %s" % level))
 
 def addLevelName(level, levelName):
     """
@@ -169,8 +171,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -178,9 +180,9 @@
     if isinstance(level, int):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
diff --git a/lib-python/3/logging/config.py b/lib-python/3/logging/config.py
--- a/lib-python/3/logging/config.py
+++ b/lib-python/3/logging/config.py
@@ -144,7 +144,7 @@
         h = klass(*args)
         if "level" in section:
             level = section["level"]
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -191,7 +191,7 @@
     log = root
     if "level" in section:
         level = section["level"]
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = section["handlers"]
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in section:
             level = section["level"]
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -182,5 +182,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/3/test/test_code.py b/lib-python/3/test/test_code.py
--- a/lib-python/3/test/test_code.py
+++ b/lib-python/3/test/test_code.py
@@ -81,7 +81,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 >>> def keywordonly_args(a,b,*,k1):
@@ -104,7 +104,10 @@
 
 import unittest
 import weakref
-import _testcapi
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 from test import support
 
 
@@ -127,6 +130,7 @@
 
 class CodeTest(unittest.TestCase):
 
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_newempty(self):
         co = _testcapi.code_newempty("filename", "funcname", 15)
         self.assertEqual(co.co_filename, "filename")
diff --git a/lib-python/3/test/test_codecs.py b/lib-python/3/test/test_codecs.py
--- a/lib-python/3/test/test_codecs.py
+++ b/lib-python/3/test/test_codecs.py
@@ -2,7 +2,11 @@
 import unittest
 import codecs
 import locale
-import sys, _testcapi, io
+import sys, io
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 class Queue(object):
     """
@@ -1417,7 +1421,7 @@
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
-            if encoding not in broken_incremental_coders:
+            if encoding not in broken_incremental_coders and _testcapi:
                 # check incremental decoder/encoder (fetched via the Python
                 # and C API) and iterencode()/iterdecode()
                 try:
diff --git a/lib-python/3/test/test_descrtut.py b/lib-python/3/test/test_descrtut.py
--- a/lib-python/3/test/test_descrtut.py
+++ b/lib-python/3/test/test_descrtut.py
@@ -188,7 +188,6 @@
      '__mul__',
      '__ne__',
      '__new__',
-     '__radd__',
      '__reduce__',
      '__reduce_ex__',
      '__repr__',
diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -218,7 +218,7 @@
 Kw-only arguments: 0
 Number of locals:  1
 Stack size:        4
-Flags:             OPTIMIZED, NEWLOCALS, NOFREE
+Flags:             OPTIMIZED, NEWLOCALS, NOFREE, 0x100000
 Constants:
    0: %r
    1: '__func__'
diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -6,6 +6,10 @@
 import pickle
 import weakref
 import errno
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 from test.support import (TESTFN, unlink, run_unittest, captured_output,
                           gc_collect, cpython_only)
@@ -762,6 +766,7 @@
         self.assertIn("maximum recursion depth exceeded", str(v))
 
 
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_MemoryError(self):
         # PyErr_NoMemory always raises the same exception instance.
         # Check that the traceback is not doubled.
@@ -820,6 +825,7 @@
         self.assertEqual(error5.a, 1)
         self.assertEqual(error5.__doc__, "")
 
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_memory_error_cleanup(self):
         # Issue #5437: preallocated MemoryError instances should not keep
         # traceback objects alive.
diff --git a/lib-python/3/test/test_logging.py b/lib-python/3/test/test_logging.py
--- a/lib-python/3/test/test_logging.py
+++ b/lib-python/3/test/test_logging.py
@@ -69,7 +69,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
             self.logger_states = logger_states = {}
             for name in saved_loggers:
                 logger_states[name] = getattr(saved_loggers[name],
@@ -113,8 +114,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -13,7 +13,7 @@
 
 from test.support import (run_unittest, TESTFN, unlink, get_attribute,
                           captured_stdout, skip_unless_symlink,
-                          impl_detail)
+                          impl_detail, import_module)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -232,7 +232,10 @@
 
     def test_get_config_h_filename(self):
         config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
+        # import_module skips the test when the CPython C Extension API
+        # appears to not be supported
+        self.assertTrue(os.path.isfile(config_h) or
+                        not import_module('_testcapi'), config_h)
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
diff --git a/lib-python/3/test/test_traceback.py b/lib-python/3/test/test_traceback.py
--- a/lib-python/3/test/test_traceback.py
+++ b/lib-python/3/test/test_traceback.py
@@ -1,6 +1,9 @@
 """Test cases for traceback module"""
 
-from _testcapi import traceback_print, exception_print
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 from io import StringIO
 import sys
 import unittest
@@ -154,6 +157,7 @@
 
 class TracebackFormatTests(unittest.TestCase):
 
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_traceback_format(self):
         try:
             raise KeyError('blah')
@@ -162,7 +166,7 @@
             traceback_fmt = 'Traceback (most recent call last):\n' + \
                             ''.join(traceback.format_tb(tb))
             file_ = StringIO()
-            traceback_print(tb, file_)
+            _testcapi.traceback_print(tb, file_)
             python_fmt  = file_.getvalue()
         else:
             raise Error("unable to create test traceback string")
@@ -326,10 +330,11 @@
     # This checks built-in reporting by the interpreter.
     #
 
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def get_report(self, e):
         e = self.get_exception(e)
         with captured_output("stderr") as s:
-            exception_print(e)
+            _testcapi.exception_print(e)
         return s.getvalue()
 
 
diff --git a/lib-python/3/test/test_unicode.py b/lib-python/3/test/test_unicode.py
--- a/lib-python/3/test/test_unicode.py
+++ b/lib-python/3/test/test_unicode.py
@@ -12,6 +12,10 @@
 import warnings
 from test import support, string_tests
 import _string
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 # decorator to skip tests on narrow builds
 requires_wide_build = unittest.skipIf(sys.maxunicode == 65535,
@@ -1659,6 +1663,7 @@
         self.assertEqual(text, 'repr=abc\ufffd')
 
     # Test PyUnicode_AsWideChar()
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_aswidechar(self):
         from _testcapi import unicode_aswidechar
         support.import_module('ctypes')
@@ -1696,6 +1701,7 @@
         self.assertEqual(wchar, nonbmp + '\0')
 
     # Test PyUnicode_AsWideCharString()
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_aswidecharstring(self):
         from _testcapi import unicode_aswidecharstring
         support.import_module('ctypes')
@@ -1769,6 +1775,7 @@
             ]])
         self.assertRaises(TypeError, _string.formatter_field_name_split, 1)
 
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_encode_decimal(self):
         from _testcapi import unicode_encodedecimal
         self.assertEqual(unicode_encodedecimal('123'),
@@ -1794,6 +1801,7 @@
         self.assertEqual(unicode_encodedecimal("123\u20ac\u0660", "replace"),
                          b'123?0')
 
+    @unittest.skipUnless(_testcapi, 'Requires _testcapi')
     def test_transform_decimal(self):
         from _testcapi import unicode_transformdecimaltoascii as transform_decimal
         self.assertEqual(transform_decimal('123'),
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -123,7 +123,7 @@
     RegrTest('test_bz2.py', usemodules='bz2'),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
-    RegrTest('test_capi.py'),
+    RegrTest('test_capi.py', usemodules='cpyext'),
     RegrTest('test_cfgparser.py'),
     RegrTest('test_cgi.py'),
     RegrTest('test_charmapcodec.py', core=True),
@@ -163,7 +163,7 @@
     RegrTest('test_cprofile.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
     RegrTest('test_csv.py', usemodules='_csv'),
-    RegrTest('test_ctypes.py', usemodules="_rawffi thread"),
+    RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
     RegrTest('test_curses.py'),
     RegrTest('test_datetime.py', usemodules='binascii struct'),
     RegrTest('test_dbm.py'),
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -28,7 +28,7 @@
         # XXX check if typedict['_type_'] is any sane
         # XXX remember about paramfunc
         obj = type.__new__(self, name, cls, typedict)
-        for k, v in d.iteritems():
+        for k, v in d.items():
             setattr(obj, k, v)
         if '_type_' in typedict:
             self.set_type(obj, typedict['_type_'])
@@ -119,6 +119,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -166,8 +166,7 @@
         if self is StructOrUnion:
             return
         if '_fields_' not in self.__dict__:
-            self._fields_ = []
-            _set_shape(self, [], self._is_union)
+            self._fields_ = []  # As a side-effet, this also sets the ffishape.
 
     __setattr__ = struct_setattr
 
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python3')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:PyInit__ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -60,7 +60,7 @@
         return "{}({})".format(name, ', '.join(tmp))
 
     def __reduce__(self):
-        d = dict((k, v) for k, v in self.__dict__.iteritems() if k not in
+        d = dict((k, v) for k, v in self.__dict__.items() if k not in
                 ('func', 'args', 'keywords'))
         if len(d) == 0:
             d = None
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_pypy_testcapi.py
copy from lib_pypy/_testcapi.py
copy to lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,14 +1,20 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
     """
     thisdir = os.path.dirname(__file__)
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -19,13 +25,13 @@
         ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
+    res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
                            extra_preargs=ccflags)
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
+    output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
@@ -37,7 +43,7 @@
             library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:PyInit__testcapi']
+                        '/EXPORT:PyInit_' + modulename]
     else:
         libraries = []
         extra_ldargs = []
@@ -49,9 +55,7 @@
         libraries=libraries,
         extra_preargs=extra_ldargs)
 
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,57 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python3')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:PyInit__testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testcapi'")
+else:
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.6"
-__version_info__ = (0, 6)
+__version__ = "0.7"
+__version_info__ = (0, 7)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -73,15 +73,15 @@
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        BVoidP = self._get_cached_btype(model.voidp_type)
+        self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(BVoidP, 0)
+                FFI.NULL = self.cast(self.BVoidP, 0)
                 FFI.CData, FFI.CType = backend._get_types()
         else:
             # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(BVoidP, 0)
+            self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False):
@@ -346,6 +346,12 @@
         self._cdefsources.extend(ffi_to_include._cdefsources)
         self._cdefsources.append(']')
 
+    def new_handle(self, x):
+        return self._backend.newp_handle(self.BVoidP, x)
+
+    def from_handle(self, x):
+        return self._backend.from_handle(x)
+
 
 def _make_ffi_library(ffi, libname, flags):
     import os
@@ -372,8 +378,8 @@
             BType = ffi._get_cached_btype(tp)
             try:
                 value = backendlib.load_function(BType, name)
-            except KeyError:
-                raise AttributeError(name)
+            except KeyError as e:
+                raise AttributeError('%s: %s' % (name, e))
             library.__dict__[name] = value
             return
         #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -16,6 +16,7 @@
 class CTypesData(object):
     __metaclass__ = CTypesType
     __slots__ = ['__weakref__']
+    __name__ = '<cdata>'
 
     def __init__(self, *args):
         raise TypeError("cannot instantiate %r" % (self.__class__,))
@@ -491,6 +492,8 @@
         elif BItem in (getbtype(model.PrimitiveType('signed char')),
                        getbtype(model.PrimitiveType('unsigned char'))):
             kind = 'bytep'
+        elif BItem is getbtype(model.void_type):
+            kind = 'voidp'
         else:
             kind = 'generic'
         #
@@ -546,13 +549,13 @@
             def __setitem__(self, index, value):
                 self._as_ctype_ptr[index] = BItem._to_ctypes(value)
 
-            if kind == 'charp':
+            if kind == 'charp' or kind == 'voidp':
                 @classmethod
-                def _arg_to_ctypes(cls, value):
-                    if isinstance(value, bytes):
-                        return ctypes.c_char_p(value)
+                def _arg_to_ctypes(cls, *value):
+                    if value and isinstance(value[0], bytes):
+                        return ctypes.c_char_p(value[0])
                     else:
-                        return super(CTypesPtr, cls)._arg_to_ctypes(value)
+                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
 
             if kind == 'charp' or kind == 'bytep':
                 def _to_string(self, maxlen):
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -15,6 +15,20 @@
     def patch_extension_kwds(self, kwds):
         pass
 
+    def find_module(self, module_name, path, so_suffix):
+        try:
+            f, filename, descr = imp.find_module(module_name, path)
+        except ImportError:
+            return None
+        if f is not None:
+            f.close()
+        # Note that after a setuptools installation, there are both .py
+        # and .so files with the same basename.  The code here relies on
+        # imp.find_module() locating the .so in priority.
+        if descr[0] != so_suffix:
+            return None
+        return filename
+
     def collect_types(self):
         self._typesdict = {}
         self._generate("collecttype")
@@ -142,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -427,9 +444,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -687,7 +704,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
     # ----------
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -1,4 +1,4 @@
-import sys
+import sys, os
 import types
 
 from . import model, ffiplatform
@@ -20,6 +20,16 @@
         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
+    def find_module(self, module_name, path, so_suffix):
+        basename = module_name + so_suffix
+        if path is None:
+            path = sys.path
+        for dirname in path:
+            filename = os.path.join(dirname, basename)
+            if os.path.isfile(filename):
+                return filename
+        return None
+
     def collect_types(self):
         pass      # not needed in the generic engine
 
@@ -64,6 +74,9 @@
         class FFILibrary(types.ModuleType):
             _cffi_generic_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir
         library = FFILibrary("")
         #
         # finally, call the loaded_gen_xxx() functions.  This will set
@@ -158,21 +171,22 @@
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
-            if any(isinstance(type, model.StructOrUnion) for type in tp.args):
+            if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
-                for i, type in enumerate(tp.args):
-                    if isinstance(type, model.StructOrUnion):
-                        type = model.PointerType(type)
-                        indirections.append((i, type))
-                    indirect_args.append(type)
+                for i, typ in enumerate(tp.args):
+                    if isinstance(typ, model.StructOrUnion):
+                        typ = model.PointerType(typ)
+                        indirections.append((i, typ))
+                    indirect_args.append(typ)
                 tp = model.FunctionPtrType(tuple(indirect_args),
                                            tp.result, tp.ellipsis)
             BFunc = self.ffi._get_cached_btype(tp)
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
-            for i, type in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, type)
+            for i, typ in indirections:
+                newfunction = self._make_struct_wrapper(newfunction, i, typ)
         setattr(library, name, newfunction)
+        type(library)._cffi_dir.append(name)
 
     def _make_struct_wrapper(self, oldfunc, i, tp):
         backend = self.ffi._backend
@@ -216,9 +230,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -380,6 +394,7 @@
         is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
         value = self._load_constant(is_int, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # enums
@@ -427,6 +442,7 @@
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
             setattr(library, enumerator, enumvalue)
+            type(library)._cffi_dir.append(enumerator)
 
     # ----------
     # macros: for now only for integers
@@ -440,6 +456,7 @@
     def _loaded_gen_macro(self, tp, name, module, library):
         value = self._load_constant(True, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # global variables
@@ -465,6 +482,7 @@
                 BArray = self.ffi._get_cached_btype(tp)
                 value = self.ffi.cast(BArray, value)
             setattr(library, name, value)
+            type(library)._cffi_dir.append(name)
             return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
@@ -476,7 +494,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
 cffimod_header = r'''
 #include <stdio.h>
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -102,21 +102,10 @@
                 path = pkg.__path__
             else:
                 path = None
-            try:
-                f, filename, descr = imp.find_module(self.get_module_name(),
-                                                     path)
-            except ImportError:
+            filename = self._vengine.find_module(self.get_module_name(), path,
+                                                 _get_so_suffix())
+            if filename is None:
                 return
-            if f is not None:
-                f.close()
-            if filename.lower().endswith('.py'):
-                # on PyPy, if there are both .py and .pypy-19.so files in
-                # the same directory, the .py file is returned.  That's the
-                # case after a setuptools installation.  We never want to
-                # load the .py file here...
-                filename = filename[:-3] + _get_so_suffix()
-                if not os.path.isfile(filename):
-                    return
             self.modulefilename = filename
         self._vengine.collect_types()
         self._has_module = True
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
new file mode 100644
--- /dev/null
+++ b/lib_pypy/greenlet.egg-info
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: greenlet
+Version: 0.4.0
+Summary: Lightweight in-process concurrent programming
+Home-page: https://github.com/python-greenlet/greenlet
+Author: Ralf Schmitt (for CPython), PyPy team
+Author-email: pypy-dev at python.org
+License: MIT License
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/pypy/TODO b/pypy/TODO
--- a/pypy/TODO
+++ b/pypy/TODO
@@ -1,18 +1,5 @@
 TODO for the python3 test suite:
 
-* test_decimal:
-   In py3k, hash(-1) is now -2 (probably as an optimisation, because
-   PyObject_Hash() return -1 on exception).
-   It's important to be compatible, since other classes like Decimal
-   and Fractions have to return the same hashes for equivalent values.
-   IOW: int.__hash__ is part of the Python language specification.
-   The py3k-newhash branch has an updated float hash, int's hash is
-   still pending
-
-* test_fractions
-* test_numeric_tower
-   float.__hash__ has changed as well (fixed on py3k-newhash)
-
 * test_float
    nan = float('nan');  assert nan in [nan]
    This has always been true in CPython, it is now guaranteed that the
@@ -25,11 +12,6 @@
    Needs bytes/str changes. Probably easy. Work for this has begun on
    py3k-memoryview (by mjacob)
 
-* test_peepholer
-   'a in [1,2,3]' is rewritten as 'a in (1, 2, 3)'
-   and the tuple is a prebuilt constant.
-   Likewise, a set becomes a frozenset.
-
 * test_pep263
    Tracebacks should be able to print unicode source code. This is
    really due to the tokenizer not being fully unicode aware. The
@@ -45,16 +27,8 @@
 own-tests:
 
 * module/test_lib_pypy
-  These crash the buildbots (via SyntaxErrors): some were really made
-  to run under Python 2.x
-
-* interpreter.test.test_zzpickle_and_slow test_pickle_frame_with_exc
-  Due to W_OperationError not being pickleable. Probably be best for
-  the ExceptionHandlerBlock to push *sys.exc_info() instead of it,
-  like CPython does
-
-* module.bz2.test.test_bz2_file test_open_non_existent
-  Some really obscure GC stuff
+  These crash the buildbots (via SyntaxErrors): others were really
+  made to run under Python 2.x and so simply fail
 
 * module.cpyext.test.test_structseq test_StructSeq
   structseq now subclasses tuple on py3, which breaks how
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -67,7 +67,8 @@
     del working_modules["_minimal_curses"]
     del working_modules["_posixsubprocess"]
 
-#    del working_modules["cppyy"]  # not tested on win32
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # not tested on win32
 
     # The _locale module is needed by site.py on Windows
     default_modules["_locale"] = None
@@ -80,7 +81,8 @@
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
     del working_modules["_multiprocessing"]   # depends on rctime
-#    del working_modules["cppyy"]  # depends on ctypes
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # depends on ctypes
 
 
 module_dependencies = {
@@ -123,12 +125,10 @@
                     __import__(name)
             except (ImportError, CompilationError, py.test.skip.Exception), e:
                 errcls = e.__class__.__name__
-                config.add_warning(
+                raise Exception(
                     "The module %r is disabled\n" % (modname,) +
                     "because importing %s raised %s\n" % (name, errcls) +
                     str(e))
-                raise ConflictConfigError("--withmod-%s: %s" % (modname,
-                                                                errcls))
         return validator
     else:
         return None
@@ -215,10 +215,6 @@
                    "(the empty string and potentially single-char strings)",
                    default=False),
 
-        BoolOption("withsmalltuple",
-                   "use small tuples",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
@@ -356,6 +352,7 @@
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
+
     config.objspace.usemodules.suggest(**dict.fromkeys(modules, True))
 
 def enable_translationmodules(config):
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,7 @@
+
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -339,8 +339,9 @@
 
 + methods and other class attributes do not change after startup
 + single inheritance is fully supported
-+ simple mixins work too, but the mixed in class needs a ``_mixin_ = True``
-  class attribute
++ simple mixins somewhat work too, but the mixed in class needs a
+  ``_mixin_ = True`` class attribute. isinstance checks against the
+  mixin type will fail when translated.
 
 + classes are first-class objects too
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.0'
+release = '2.0.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -163,6 +163,9 @@
     $ genreflex MyClass.h
     $ g++ -fPIC -rdynamic -O2 -shared -I$REFLEXHOME/include MyClass_rflx.cpp -o libMyClassDict.so -L$REFLEXHOME/lib -lReflex
 
+Next, make sure that the library can be found through the dynamic lookup path
+(the ``LD_LIBRARY_PATH`` environment variable on Linux, ``PATH`` on Windows),
+for example by adding ".".
 Now you're ready to use the bindings.
 Since the bindings are designed to look pythonistic, it should be
 straightforward::
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:
 
 * Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
 
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -28,7 +28,8 @@
 Layers
 ------
 
-PyPy has layers. Those layers help us keep the respective parts separated enough
+PyPy has layers. Just like Ogres or onions.
+Those layers help us keep the respective parts separated enough
 to be worked on independently and make the complexity manageable. This is,
 again, just a sanity requirement for such a complex project. For example writing
 a new optimization for the JIT usually does **not** involve touching a Python
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -22,7 +22,8 @@
   will capture the revision number of this change for the release;
   some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
-  necessary
+  necessary; also update the version number in pypy/doc/conf.py,
+  and in pypy/doc/index.rst
 * update pypy/doc/contributor.rst (and possibly LICENSE)
 * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
   and create a fresh whatsnew_head.rst after the release
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -40,7 +40,7 @@
 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.0`_: the latest official release
+* `Release 2.0.2`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
@@ -110,7 +110,7 @@
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.0`: http://pypy.org/download.html
+.. _`Release 2.0.2`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,7 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/doc/release-2.0.1.rst b/pypy/doc/release-2.0.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.1.rst
@@ -0,0 +1,46 @@
+==============================
+PyPy 2.0.1 - Bohr Sm&#248;rrebr&#248;d
+==============================
+
+We're pleased to announce PyPy 2.0.1.  This is a stable bugfix release
+over `2.0`_.  You can download it here:
+  
+    http://pypy.org/download.html
+
+The fixes are mainly about fatal errors or crashes in our stdlib.  See
+below for more details.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+

From noreply at buildbot.pypy.org  Fri Jun  7 16:30:07 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  7 Jun 2013 16:30:07 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: fix test for -A
Message-ID: <20130607143007.AEC001C328A@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: numpypy-nditer
Changeset: r64818:5e7a972915e0
Date: 2013-06-07 09:55 +0300
http://bitbucket.org/pypy/pypy/changeset/5e7a972915e0/

Log:	fix test for -A

diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -57,7 +57,7 @@
         for x in nditer(a, flags=['external_loop'], order='F'):
             r.append(x)
             n += 1
-        assert n == 3
+        assert n == 6
         assert (array(r) == [[0, 6], [2, 8], [4, 10], [1, 7], [3, 9], [5, 11]]).all()
 
     def test_interface(self):

From noreply at buildbot.pypy.org  Fri Jun  7 16:30:09 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  7 Jun 2013 16:30:09 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: progress toward extern_loop
Message-ID: <20130607143009.1B79F1C328A@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: numpypy-nditer
Changeset: r64819:730edc51d980
Date: 2013-06-07 17:19 +0300
http://bitbucket.org/pypy/pypy/changeset/730edc51d980/

Log:	progress toward extern_loop

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
-                                             shape_agreement_multiple)
+                                          shape_agreement_multiple, calc_steps)
 from pypy.module.micronumpy.iter import MultiDimViewIterator, SliceIterator
 from pypy.module.micronumpy import support
 from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
@@ -229,11 +229,17 @@
         self.iters=[]
         self.shape = iter_shape = shape_agreement_multiple(space, self.seq)
         if self.external_loop:
-            #XXX find longest contiguous shape
-            iter_shape = iter_shape[1:]
+            steps = []
+            for seq in self.seq:
+                impl = seq.implementation
+                steps.append(calc_steps(impl.shape, impl.strides, self.order))
+            #XXX #find longest contiguous shape
+            print 'steps',steps,'tier_shape',iter_shape
+            iter_shape = [1]
         for i in range(len(self.seq)):
             self.iters.append(BoxIterator(get_iter(space, self.order,
-                            self.seq[i].implementation, iter_shape), self.op_flags[i]))
+                            self.seq[i].implementation, iter_shape),
+                            self.op_flags[i]))
 
     def descr_iter(self, space):
         return space.wrap(self)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -51,8 +51,8 @@
             rstrides.append(strides[i])
             rbackstrides.append(backstrides[i])
     if backwards:
-        rstrides = rstrides + [0] * (len(res_shape) - len(orig_shape))  
-        rbackstrides = rbackstrides + [0] * (len(res_shape) - len(orig_shape)) 
+        rstrides = rstrides + [0] * (len(res_shape) - len(orig_shape))
+        rbackstrides = rbackstrides + [0] * (len(res_shape) - len(orig_shape))
     else:
         rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides
         rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
@@ -62,7 +62,7 @@
     if (is_rec_type and space.isinstance_w(w_elem, space.w_tuple)):
         return True
     if (space.isinstance_w(w_elem, space.w_tuple) or
-        isinstance(w_elem, W_NDimArray) or    
+        isinstance(w_elem, W_NDimArray) or
         space.isinstance_w(w_elem, space.w_list)):
         return False
     return True
@@ -293,3 +293,27 @@
             rbackstrides[i] = backstrides[j]
             j += 1
     return rstrides, rbackstrides
+
+ at jit.unroll_safe
+def calc_steps(shape, strides, order='C'):
+    steps = []
+    if order == 'K':
+        if strides[0] < strides[-1]:
+            order = 'F'
+        else:
+            order = 'C'
+    if order == 'F' or order == 'A':
+        last_step = strides[0]
+        for i in range(len(shape)):
+            steps.append(strides[i] / last_step)
+            last_step *= shape[i]
+        if order == 'A':
+            pass
+            #XXX test for all(steps==steps[0])
+    elif order == 'C':
+        last_step = strides[-1]
+        for i in range(len(shape) - 1, -1, -1):
+            steps.insert(0, strides[i] / last_step)
+            last_step *= shape[i]
+    return steps
+

From noreply at buildbot.pypy.org  Fri Jun  7 16:30:10 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  7 Jun 2013 16:30:10 +0200 (CEST)
Subject: [pypy-commit] pypy argsort-segfault: add a test that corrupts memory
Message-ID: <20130607143010.5FFE91C328A@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: argsort-segfault
Changeset: r64820:f5b13c9f2485
Date: 2013-06-07 17:24 +0300
http://bitbucket.org/pypy/pypy/changeset/f5b13c9f2485/

Log:	add a test that corrupts memory

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -20,7 +20,7 @@
 def make_sort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
-    
+
     class Repr(object):
         def __init__(self, index_stride_size, stride_size, size, values,
                      indexes, index_start, start):
@@ -32,7 +32,12 @@
             self.values = values
             self.indexes = indexes
 
+        def __del__(self):
+            print 'Repr.del',self.values
+
         def getitem(self, item):
+            #print 'getting',item,'of',self.size,self.values
+            #print 'from',item*self.stride_size + self.start,'to',item*(self.stride_size+1) + self.start
             if count < 2:
                 v = raw_storage_getitem(TP, self.values, item * self.stride_size
                                     + self.start)
@@ -71,11 +76,11 @@
         def __init__(self, index_stride_size, stride_size, size):
             start = 0
             dtype = interp_dtype.get_dtype_cache(space).w_longdtype
-            self.indexes = dtype.itemtype.malloc(size*dtype.get_size())
-            self.values = alloc_raw_storage(size * stride_size, 
+            indexes = dtype.itemtype.malloc(size*dtype.get_size())
+            values = alloc_raw_storage(size * stride_size,
                                             track_allocation=False)
-            Repr.__init__(self, index_stride_size, stride_size, 
-                          size, self.values, self.indexes, start, start)
+            Repr.__init__(self, index_stride_size, stride_size,
+                          size, values, indexes, start, start)
 
         def __del__(self):
             free_raw_storage(self.indexes, track_allocation=False)
@@ -91,12 +96,14 @@
         return lst.size
 
     def arg_getitem_slice(lst, start, stop):
+        print 'arg_getitem_slice',lst.values
         retval = ArgArrayRepWithStorage(lst.index_stride_size, lst.stride_size,
                 stop-start)
         for i in range(stop-start):
             retval.setitem(i, lst.getitem(i+start))
+        print 'arg_getitem_slice done',lst
         return retval
-    
+
     if count < 2:
         def arg_lt(a, b):
             # Does numpy do <= ?
@@ -108,7 +115,7 @@
                     return True
                 elif a[0][i] > b[0][i]:
                     return False
-            # Does numpy do True?    
+            # Does numpy do True?
             return False
 
     ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
@@ -148,13 +155,16 @@
             stride_size = arr.strides[axis]
             index_stride_size = index_impl.strides[axis]
             axis_size = arr.shape[axis]
+            print '5'
             while not iter.done():
                 for i in range(axis_size):
                     raw_storage_setitem(storage, i * index_stride_size +
                                         index_iter.offset, i)
                 r = Repr(index_stride_size, stride_size, axis_size,
                          arr.get_storage(), storage, index_iter.offset, iter.offset)
+                print '6'
                 ArgSort(r).sort()
+                print '7'
                 iter.next()
                 index_iter.next()
         return index_arr
@@ -180,7 +190,7 @@
 
 class SortCache(object):
     built = False
-    
+
     def __init__(self, space):
         if self.built:
             return
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2496,6 +2496,13 @@
         b = a.argsort()
         assert (b[:3] == [0, 100, 200]).all()
 
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
     def test_argsort_axis(self):
         from numpypy import array
         a = array([[4, 2], [1, 3]])

From noreply at buildbot.pypy.org  Fri Jun  7 16:44:34 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri,  7 Jun 2013 16:44:34 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Implement the external_loop flag
 on the nditer class
Message-ID: <20130607144434.DDEC81C331A@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64821:4bea52d36621
Date: 2013-06-07 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/4bea52d36621/

Log:	Implement the external_loop flag on the nditer class

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -32,13 +32,13 @@
         self.it.next()
 
     def getitem(self, space, array):
-        return self.op_flags.get_it_item(space, array, self.it)
+        return self.op_flags.get_it_item[self.index](space, array, self.it)
 
-class BoxIterator(IteratorMixin):
-    pass
+class BoxIterator(IteratorMixin, AbstractIterator):
+    index = 0
 
-class SliceIterator(IteratorMixin):
-    pass
+class ExternalLoopIterator(IteratorMixin, AbstractIterator):
+    index = 1
 
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
     ret = []
@@ -73,7 +73,7 @@
         self.native_byte_order = False
         self.tmp_copy = ''
         self.allocate = False
-        self.get_it_item = get_readonly_item
+        self.get_it_item = (get_readonly_item, get_readonly_slice)
 
 def get_readonly_item(space, array, it):
     return space.wrap(it.getitem())
@@ -128,9 +128,9 @@
             raise OperationError(space.w_ValueError, space.wrap(
                     'op_flags must be a tuple or array of per-op flag-tuples'))
         if op_flag.rw == 'r':
-            op_flag.get_it_item = get_readonly_item
+            op_flag.get_it_item = (get_readonly_item, get_readonly_slice)
         elif op_flag.rw == 'rw':
-            op_flag.get_it_item = get_readwrite_item
+            op_flag.get_it_item = (get_readwrite_item, get_readwrite_slice)
     return op_flag
 
 def parse_func_flags(space, nditer, w_flags):
@@ -180,7 +180,8 @@
                 'Iterator flag EXTERNAL_LOOP cannot be used if an index or '
                 'multi-index is being tracked'))
 
-def get_iter(space, order, imp, shape):
+def get_iter(space, order, arr, shape):
+    imp = arr.implementation
     if order == 'K' or (order == 'C' and imp.order == 'C'):
         backward = False
     elif order =='F' and imp.order == 'C':
@@ -201,6 +202,18 @@
                                     shape, backward)
     return MultiDimViewIterator(imp, imp.dtype, imp.start, r[0], r[1], shape)
 
+def get_external_loop_iter(space, order, arr, shape):
+    imp = arr.implementation
+    if order == 'K' or (order == 'C' and imp.order == 'C'):
+        backward = False
+    elif order =='F' and imp.order == 'C':
+        backward = True
+    else:
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+                'not implemented yet'))
+
+    return SliceIterator(arr, imp.strides, imp.backstrides, shape, order=order, backward=backward)
+
 
 class W_NDIter(W_Root):
 
@@ -229,11 +242,13 @@
         self.iters=[]
         self.shape = iter_shape = shape_agreement_multiple(space, self.seq)
         if self.external_loop:
-            #XXX find longest contiguous shape
-            iter_shape = iter_shape[1:]
-        for i in range(len(self.seq)):
-            self.iters.append(BoxIterator(get_iter(space, self.order,
-                            self.seq[i].implementation, iter_shape), self.op_flags[i]))
+            for i in range(len(self.seq)):
+                self.iters.append(ExternalLoopIterator(get_external_loop_iter(space, self.order,
+                                self.seq[i], iter_shape), self.op_flags[i]))
+        else:
+            for i in range(len(self.seq)):
+                self.iters.append(BoxIterator(get_iter(space, self.order,
+                                self.seq[i], iter_shape), self.op_flags[i]))
 
     def descr_iter(self, space):
         return space.wrap(self)
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -46,6 +46,7 @@
      calculate_slice_strides
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.arrayimpl import base
+from pypy.module.micronumpy import support
 from rpython.rlib import jit
 
 # structures to describe slicing
@@ -267,28 +268,49 @@
         self.offset %= self.size
 
 class SliceIterator(object):
-    def __init__(self, arr, stride, backstride, shape, dtype=None):
-        self.step = 0
+    def __init__(self, arr, strides, backstrides, shape, order="C", backward=False, dtype=None):
+        self.indexes = [0] * (len(shape) - 1)
+        self.offset = 0
         self.arr = arr
-        self.stride = stride
-        self.backstride = backstride
-        self.shape = shape
         if dtype is None:
             dtype = arr.implementation.dtype
+        if backward:
+            self.slicesize = shape[0]
+            self.gap = [support.product(shape[1:]) * dtype.get_size()]
+            self.strides = strides[1:][::-1]
+            self.backstrides = backstrides[1:][::-1]
+            self.shape = shape[1:][::-1]
+            self.shapelen = len(self.shape)
+        else:
+            shape = [support.product(shape)]
+            self.strides, self.backstrides = support.calc_strides(shape, dtype, order)
+            self.slicesize = support.product(shape)
+            self.shapelen = 0
+            self.gap = self.strides
         self.dtype = dtype
         self._done = False
 
-    def done():
+    def done(self):
         return self._done
 
-    def next():
-        self.step += self.arr.implementation.dtype.get_size()
-        if self.step == self.backstride - self.implementation.dtype.get_size():
+    @jit.unroll_safe
+    def next(self):
+        offset = self.offset
+        for i in range(self.shapelen - 1, -1, -1):
+            if self.indexes[i] < self.shape[i] - 1:
+                self.indexes[i] += 1
+                offset += self.strides[i]
+                break
+            else:
+                self.indexes[i] = 0
+                offset -= self.backstrides[i]
+        else:
             self._done = True
+        self.offset = offset
 
     def getslice(self):
         from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
-        return SliceArray(self.step, [self.stride], [self.backstride], self.shape, self.arr.implementation, self.arr, self.dtype)
+        return SliceArray(self.offset, self.gap, self.backstrides, [self.slicesize], self.arr.implementation, self.arr, self.dtype)
 
 class AxisIterator(base.BaseArrayIterator):
     def __init__(self, array, shape, dim, cumultative):
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -44,21 +44,22 @@
 
     def test_external_loop(self):
         from numpypy import arange, nditer, array
-        a = arange(12).reshape(2,3,2)
+        a = arange(24).reshape(2, 3, 4)
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop']):
             r.append(x)
             n += 1
+        print r
         assert n == 1
-        assert (array(r) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]).all()
+        assert (array(r) == range(24)).all()
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop'], order='F'):
             r.append(x)
             n += 1
-        assert n == 3
-        assert (array(r) == [[0, 6], [2, 8], [4, 10], [1, 7], [3, 9], [5, 11]]).all()
+        assert n == 12
+        assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
 
     def test_interface(self):
         from numpypy import arange, nditer, zeros

From noreply at buildbot.pypy.org  Fri Jun  7 16:44:36 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri,  7 Jun 2013 16:44:36 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Backed out changeset 730edc51d980
Message-ID: <20130607144436.1BED41C331A@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64822:7107aa4daf89
Date: 2013-06-07 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/7107aa4daf89/

Log:	Backed out changeset 730edc51d980

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
-                                          shape_agreement_multiple, calc_steps)
+                                             shape_agreement_multiple)
 from pypy.module.micronumpy.iter import MultiDimViewIterator, SliceIterator
 from pypy.module.micronumpy import support
 from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
@@ -229,17 +229,11 @@
         self.iters=[]
         self.shape = iter_shape = shape_agreement_multiple(space, self.seq)
         if self.external_loop:
-            steps = []
-            for seq in self.seq:
-                impl = seq.implementation
-                steps.append(calc_steps(impl.shape, impl.strides, self.order))
-            #XXX #find longest contiguous shape
-            print 'steps',steps,'tier_shape',iter_shape
-            iter_shape = [1]
+            #XXX find longest contiguous shape
+            iter_shape = iter_shape[1:]
         for i in range(len(self.seq)):
             self.iters.append(BoxIterator(get_iter(space, self.order,
-                            self.seq[i].implementation, iter_shape),
-                            self.op_flags[i]))
+                            self.seq[i].implementation, iter_shape), self.op_flags[i]))
 
     def descr_iter(self, space):
         return space.wrap(self)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -51,8 +51,8 @@
             rstrides.append(strides[i])
             rbackstrides.append(backstrides[i])
     if backwards:
-        rstrides = rstrides + [0] * (len(res_shape) - len(orig_shape))
-        rbackstrides = rbackstrides + [0] * (len(res_shape) - len(orig_shape))
+        rstrides = rstrides + [0] * (len(res_shape) - len(orig_shape))  
+        rbackstrides = rbackstrides + [0] * (len(res_shape) - len(orig_shape)) 
     else:
         rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides
         rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
@@ -62,7 +62,7 @@
     if (is_rec_type and space.isinstance_w(w_elem, space.w_tuple)):
         return True
     if (space.isinstance_w(w_elem, space.w_tuple) or
-        isinstance(w_elem, W_NDimArray) or
+        isinstance(w_elem, W_NDimArray) or    
         space.isinstance_w(w_elem, space.w_list)):
         return False
     return True
@@ -293,27 +293,3 @@
             rbackstrides[i] = backstrides[j]
             j += 1
     return rstrides, rbackstrides
-
- at jit.unroll_safe
-def calc_steps(shape, strides, order='C'):
-    steps = []
-    if order == 'K':
-        if strides[0] < strides[-1]:
-            order = 'F'
-        else:
-            order = 'C'
-    if order == 'F' or order == 'A':
-        last_step = strides[0]
-        for i in range(len(shape)):
-            steps.append(strides[i] / last_step)
-            last_step *= shape[i]
-        if order == 'A':
-            pass
-            #XXX test for all(steps==steps[0])
-    elif order == 'C':
-        last_step = strides[-1]
-        for i in range(len(shape) - 1, -1, -1):
-            steps.insert(0, strides[i] / last_step)
-            last_step *= shape[i]
-    return steps
-

From noreply at buildbot.pypy.org  Fri Jun  7 16:44:37 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri,  7 Jun 2013 16:44:37 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Merge heads
Message-ID: <20130607144437.5A9961C331A@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64823:000f7ae8ea0c
Date: 2013-06-07 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/000f7ae8ea0c/

Log:	Merge heads

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -32,13 +32,13 @@
         self.it.next()
 
     def getitem(self, space, array):
-        return self.op_flags.get_it_item(space, array, self.it)
+        return self.op_flags.get_it_item[self.index](space, array, self.it)
 
-class BoxIterator(IteratorMixin):
-    pass
+class BoxIterator(IteratorMixin, AbstractIterator):
+    index = 0
 
-class SliceIterator(IteratorMixin):
-    pass
+class ExternalLoopIterator(IteratorMixin, AbstractIterator):
+    index = 1
 
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
     ret = []
@@ -73,7 +73,7 @@
         self.native_byte_order = False
         self.tmp_copy = ''
         self.allocate = False
-        self.get_it_item = get_readonly_item
+        self.get_it_item = (get_readonly_item, get_readonly_slice)
 
 def get_readonly_item(space, array, it):
     return space.wrap(it.getitem())
@@ -128,9 +128,9 @@
             raise OperationError(space.w_ValueError, space.wrap(
                     'op_flags must be a tuple or array of per-op flag-tuples'))
         if op_flag.rw == 'r':
-            op_flag.get_it_item = get_readonly_item
+            op_flag.get_it_item = (get_readonly_item, get_readonly_slice)
         elif op_flag.rw == 'rw':
-            op_flag.get_it_item = get_readwrite_item
+            op_flag.get_it_item = (get_readwrite_item, get_readwrite_slice)
     return op_flag
 
 def parse_func_flags(space, nditer, w_flags):
@@ -180,7 +180,8 @@
                 'Iterator flag EXTERNAL_LOOP cannot be used if an index or '
                 'multi-index is being tracked'))
 
-def get_iter(space, order, imp, shape):
+def get_iter(space, order, arr, shape):
+    imp = arr.implementation
     if order == 'K' or (order == 'C' and imp.order == 'C'):
         backward = False
     elif order =='F' and imp.order == 'C':
@@ -201,6 +202,18 @@
                                     shape, backward)
     return MultiDimViewIterator(imp, imp.dtype, imp.start, r[0], r[1], shape)
 
+def get_external_loop_iter(space, order, arr, shape):
+    imp = arr.implementation
+    if order == 'K' or (order == 'C' and imp.order == 'C'):
+        backward = False
+    elif order =='F' and imp.order == 'C':
+        backward = True
+    else:
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+                'not implemented yet'))
+
+    return SliceIterator(arr, imp.strides, imp.backstrides, shape, order=order, backward=backward)
+
 
 class W_NDIter(W_Root):
 
@@ -229,11 +242,13 @@
         self.iters=[]
         self.shape = iter_shape = shape_agreement_multiple(space, self.seq)
         if self.external_loop:
-            #XXX find longest contiguous shape
-            iter_shape = iter_shape[1:]
-        for i in range(len(self.seq)):
-            self.iters.append(BoxIterator(get_iter(space, self.order,
-                            self.seq[i].implementation, iter_shape), self.op_flags[i]))
+            for i in range(len(self.seq)):
+                self.iters.append(ExternalLoopIterator(get_external_loop_iter(space, self.order,
+                                self.seq[i], iter_shape), self.op_flags[i]))
+        else:
+            for i in range(len(self.seq)):
+                self.iters.append(BoxIterator(get_iter(space, self.order,
+                                self.seq[i], iter_shape), self.op_flags[i]))
 
     def descr_iter(self, space):
         return space.wrap(self)
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -46,6 +46,7 @@
      calculate_slice_strides
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.arrayimpl import base
+from pypy.module.micronumpy import support
 from rpython.rlib import jit
 
 # structures to describe slicing
@@ -267,28 +268,49 @@
         self.offset %= self.size
 
 class SliceIterator(object):
-    def __init__(self, arr, stride, backstride, shape, dtype=None):
-        self.step = 0
+    def __init__(self, arr, strides, backstrides, shape, order="C", backward=False, dtype=None):
+        self.indexes = [0] * (len(shape) - 1)
+        self.offset = 0
         self.arr = arr
-        self.stride = stride
-        self.backstride = backstride
-        self.shape = shape
         if dtype is None:
             dtype = arr.implementation.dtype
+        if backward:
+            self.slicesize = shape[0]
+            self.gap = [support.product(shape[1:]) * dtype.get_size()]
+            self.strides = strides[1:][::-1]
+            self.backstrides = backstrides[1:][::-1]
+            self.shape = shape[1:][::-1]
+            self.shapelen = len(self.shape)
+        else:
+            shape = [support.product(shape)]
+            self.strides, self.backstrides = support.calc_strides(shape, dtype, order)
+            self.slicesize = support.product(shape)
+            self.shapelen = 0
+            self.gap = self.strides
         self.dtype = dtype
         self._done = False
 
-    def done():
+    def done(self):
         return self._done
 
-    def next():
-        self.step += self.arr.implementation.dtype.get_size()
-        if self.step == self.backstride - self.implementation.dtype.get_size():
+    @jit.unroll_safe
+    def next(self):
+        offset = self.offset
+        for i in range(self.shapelen - 1, -1, -1):
+            if self.indexes[i] < self.shape[i] - 1:
+                self.indexes[i] += 1
+                offset += self.strides[i]
+                break
+            else:
+                self.indexes[i] = 0
+                offset -= self.backstrides[i]
+        else:
             self._done = True
+        self.offset = offset
 
     def getslice(self):
         from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
-        return SliceArray(self.step, [self.stride], [self.backstride], self.shape, self.arr.implementation, self.arr, self.dtype)
+        return SliceArray(self.offset, self.gap, self.backstrides, [self.slicesize], self.arr.implementation, self.arr, self.dtype)
 
 class AxisIterator(base.BaseArrayIterator):
     def __init__(self, array, shape, dim, cumultative):
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -44,21 +44,21 @@
 
     def test_external_loop(self):
         from numpypy import arange, nditer, array
-        a = arange(12).reshape(2,3,2)
+        a = arange(24).reshape(2, 3, 4)
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop']):
             r.append(x)
             n += 1
         assert n == 1
-        assert (array(r) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]).all()
+        assert (array(r) == range(24)).all()
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop'], order='F'):
             r.append(x)
             n += 1
-        assert n == 6
-        assert (array(r) == [[0, 6], [2, 8], [4, 10], [1, 7], [3, 9], [5, 11]]).all()
+        assert n == 12
+        assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
 
     def test_interface(self):
         from numpypy import arange, nditer, zeros

From noreply at buildbot.pypy.org  Fri Jun  7 23:21:21 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri,  7 Jun 2013 23:21:21 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Fix translation
Message-ID: <20130607212121.13D651C06B1@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64824:f406998d825f
Date: 2013-06-07 23:20 +0200
http://bitbucket.org/pypy/pypy/changeset/f406998d825f/

Log:	Fix translation

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -16,7 +16,7 @@
     def next(self):
         raise NotImplementedError("Abstract Class")
 
-    def getitem(self, array):
+    def getitem(self, space, array):
         raise NotImplementedError("Abstract Class")
 
 class IteratorMixin(object):
@@ -56,8 +56,8 @@
             if len(w_lst) != n:
                 raise OperationError(space.w_ValueError, space.wrap(
                    '%s must be a tuple or array of per-op flag-tuples' % name))
-            for item in space.listview(w_lst):
-                ret.append(parse_one_arg(space, item))
+            for item in w_lst:
+                ret.append(parse_one_arg(space, space.listview(item)))
         else:
             op_flag = parse_one_arg(space, w_lst)
             for i in range(n):
@@ -174,7 +174,7 @@
             nditer.zerosize_ok = True
         else:
             raise OperationError(space.w_ValueError, space.wrap(
-                    'Unexpected iterator global flag "%s"', item))
+                    'Unexpected iterator global flag "%s"' % item))
     if nditer.tracked_index and nditer.external_loop:
             raise OperationError(space.w_ValueError, space.wrap(
                 'Iterator flag EXTERNAL_LOOP cannot be used if an index or '
@@ -192,9 +192,9 @@
     if (imp.strides[0] < imp.strides[-1] and not backward) or \
        (imp.strides[0] > imp.strides[-1] and backward):
         # flip the strides. Is this always true for multidimension?
-        strides = [s for s in imp.strides[::-1]]
-        backstrides = [s for s in imp.backstrides[::-1]]
-        shape = [s for s in shape[::-1]]
+        strides = [imp.strides[i] for i in range(len(imp.strides) - 1, -1, -1)]
+        backstrides = [imp.backstrides[i] for i in range(len(imp.backstrides) - 1, -1, -1)]
+        shape = [imp.shape[i] for i in range(len(imp.shape) - 1, -1, -1)]
     else:
         strides = imp.strides
         backstrides = imp.backstrides
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -277,9 +277,12 @@
         if backward:
             self.slicesize = shape[0]
             self.gap = [support.product(shape[1:]) * dtype.get_size()]
-            self.strides = strides[1:][::-1]
-            self.backstrides = backstrides[1:][::-1]
-            self.shape = shape[1:][::-1]
+            self.strides = strides[1:]
+            self.backstrides = backstrides[1:]
+            self.shape = shape[1:]
+            self.strides.reverse()
+            self.backstrides.reverse()
+            self.shape.reverse()
             self.shapelen = len(self.shape)
         else:
             shape = [support.product(shape)]

From noreply at buildbot.pypy.org  Sat Jun  8 11:25:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Jun 2013 11:25:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move this logic after the checks,
 otherwise errors might be hidden
Message-ID: <20130608092503.354F61C009D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64825:0b26d350de79
Date: 2013-06-08 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/0b26d350de79/

Log:	Move this logic after the checks, otherwise errors might be hidden
	(on Windows). (This follows cffi/0b90939873ae)

diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -289,16 +289,6 @@
                "with verify() (see pypy/module/_cffi_backend/ctypefunc.py "
                "for details)"))
 
-        if USE_C_LIBFFI_MSVC and is_result_type:
-            # MSVC returns small structures in registers.  Pretend int32 or
-            # int64 return type.  This is needed as a workaround for what
-            # is really a bug of libffi_msvc seen as an independent library
-            # (ctypes has a similar workaround).
-            if ctype.size <= 4:
-                return clibffi.ffi_type_sint32
-            if ctype.size <= 8:
-                return clibffi.ffi_type_sint64
-
         # walk the fields, expanding arrays into repetitions; first,
         # only count how many flattened fields there are
         nflat = 0
@@ -318,6 +308,16 @@
                                "a struct with a zero-length array"))
             nflat += flat
 
+        if USE_C_LIBFFI_MSVC and is_result_type:
+            # MSVC returns small structures in registers.  Pretend int32 or
+            # int64 return type.  This is needed as a workaround for what
+            # is really a bug of libffi_msvc seen as an independent library
+            # (ctypes has a similar workaround).
+            if ctype.size <= 4:
+                return clibffi.ffi_type_sint32
+            if ctype.size <= 8:
+                return clibffi.ffi_type_sint64
+
         # allocate an array of (nflat + 1) ffi_types
         elements = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * (nflat + 1))
         elements = rffi.cast(FFI_TYPE_PP, elements)

From noreply at buildbot.pypy.org  Sat Jun  8 11:26:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Jun 2013 11:26:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Import cffi/28f10889b5aa.
Message-ID: <20130608092649.DE95A1C009D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64826:c7cda7189297
Date: 2013-06-08 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/c7cda7189297/

Log:	Import cffi/28f10889b5aa.

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -41,40 +41,43 @@
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
+        # NOTE: 'expected_*' is the numbers expected from GCC.
+        # The numbers expected from MSVC are not explicitly written
+        # in this file, and will just be taken from the compiler.
         ffi = FFI()
         ffi.cdef("struct s1 { %s };" % source)
         ctype = ffi.typeof("struct s1")
         # verify the information with gcc
-        if sys.platform != "win32":
-            ffi1 = FFI()
-            ffi1.cdef("""
-                static const int Gofs_y, Galign, Gsize;
-                struct s1 *try_with_value(int fieldnum, long long value);
-            """)
-            fnames = [name for name, cfield in ctype.fields
-                           if name and cfield.bitsize > 0]
-            setters = ['case %d: s.%s = value; break;' % iname
-                       for iname in enumerate(fnames)]
-            lib = ffi1.verify("""
-                struct s1 { %s };
-                struct sa { char a; struct s1 b; };
-                #define Gofs_y  offsetof(struct s1, y)
-                #define Galign  offsetof(struct sa, b)
-                #define Gsize   sizeof(struct s1)
-                struct s1 *try_with_value(int fieldnum, long long value)
-                {
-                    static struct s1 s;
-                    memset(&s, 0, sizeof(s));
-                    switch (fieldnum) { %s }
-                    return &s;
-                }
-            """ % (source, ' '.join(setters)))
-            assert lib.Gofs_y == expected_ofs_y
-            assert lib.Galign == expected_align
-            assert lib.Gsize  == expected_size
+        ffi1 = FFI()
+        ffi1.cdef("""
+            static const int Gofs_y, Galign, Gsize;
+            struct s1 *try_with_value(int fieldnum, long long value);
+        """)
+        fnames = [name for name, cfield in ctype.fields
+                       if name and cfield.bitsize > 0]
+        setters = ['case %d: s.%s = value; break;' % iname
+                   for iname in enumerate(fnames)]
+        lib = ffi1.verify("""
+            struct s1 { %s };
+            struct sa { char a; struct s1 b; };
+            #define Gofs_y  offsetof(struct s1, y)
+            #define Galign  offsetof(struct sa, b)
+            #define Gsize   sizeof(struct s1)
+            struct s1 *try_with_value(int fieldnum, long long value)
+            {
+                static struct s1 s;
+                memset(&s, 0, sizeof(s));
+                switch (fieldnum) { %s }
+                return &s;
+            }
+        """ % (source, ' '.join(setters)))
+        if sys.platform == 'win32':
+            expected_ofs_y = lib.Gofs_y
+            expected_align = lib.Galign
+            expected_size  = lib.Gsize
         else:
-            lib = None
-            fnames = None
+            assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (
+                expected_ofs_y, expected_align, expected_size)
         # the real test follows
         assert ffi.offsetof("struct s1", "y") == expected_ofs_y
         assert ffi.alignof("struct s1") == expected_align
@@ -99,10 +102,9 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
-        if lib is not None:
-            t = lib.try_with_value(fnames.index(name), value)
-            raw2 = ffi.buffer(t, len(raw1))[:]
-            assert raw1 == raw2
+        t = lib.try_with_value(fnames.index(name), value)
+        raw2 = ffi.buffer(t, len(raw1))[:]
+        assert raw1 == raw2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
@@ -136,9 +138,11 @@
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
         self.check("char x; int :0; char y;", 4, 1, 5)
+        self.check("char x; int :0; int :0; char y;", 4, 1, 5)
         self.check("char x; long long :0; char y;", L, 1, L + 1)
         self.check("short x, y; int :0; int :0;", 2, 2, 4)
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
     def test_error_cases(self):
         ffi = FFI()

From noreply at buildbot.pypy.org  Sat Jun  8 15:17:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Jun 2013 15:17:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130608131701.3F6691C125E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r78:98c0b1dadf35
Date: 2013-06-08 15:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/98c0b1dadf35/

Log:	in-progress

diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt
--- a/c4/doc-objects.txt
+++ b/c4/doc-objects.txt
@@ -1,3 +1,22 @@
+
+Design goal
+-----------
+
+stm_read_barrier(P) -> P: the read barrier (containing a call in the
+slow path) can be applied on a pointer to an object, and returns a
+possibly different pointer.  Afterwards, any reads from the object can
+be done normally (using the returned pointer).
+
+stm_write_barrier(P) -> P: the same for writes (actually read/write mode).
+
+The returned pointers are valid until a potential transaction break ---
+with the exception that the result of stm_read_barrier() will be
+invalidated by a stm_write_barrier() done on the same object.
+
+This means we must not modify an object in-place from thread A when
+thread B might be reading from it!  It is the basis for the design
+outlined in the sequel, in which "protected" objects are seen by only
+one thread, whereas "public" objects are seen by all threads.
 
 
 
@@ -8,29 +27,19 @@
 
   Private freshly created
              \                Private, with backup
-              \                ^  .          |  ^
-               \              /  .    commit |  |
-         commit \     modify /  .            |  |
-                 \          /  . commit      |  | modify
-                  V        /  V              |  |
-           Protected, no backup              V  |
+              \                ^           |  ^
+               \              /     commit |  |
+         commit \     modify /             |  |
+                 \          /              |  | modify
+                  V        /               |  |
+           Protected, no backup            V  |
                 ^    ^              Protected, with backup
                /     |       gc       |
        commit /      `----------------'
              /
             /
-    Private copy of               (the dotted arrow is followed if the
-    a public obj                   protected backup copy was stolen)
-
-
-
-     Protected backup copy
-                  \
-                   \
-           stealing \         commit of newer version
-                     \         ,-----------------.
-                      V        |                 V
-             Up-to-date public copy       Outdated public copy
+    Private copy of
+    a public obj
 
 
 
@@ -45,6 +54,7 @@
 Protected objects:
 - converted from fresh private obj                     (old PRN)
 - converted from a private obj with backup           ptr to backup
+- converted from a private obj from public                GT
 - backup copy of a private obj                    original h_revision
 - backup copy still attached to a protected               GT
 - original obj after GC killed the backup                 GT
@@ -73,13 +83,15 @@
 - the PRN (private revision number): odd, negative, changes for every
   transaction that commits
 
-- dict active_backup_copies = {private converted from protected: backup copy}
+- list active_backup_copies =
+  [(private converted from protected, backup copy)]
 
 - dict public_to_private = {public obj: private copy}
 
 - list read_set containing the objects in the read set, with possibly
   some duplicates (but hopefully not too many)
 
+- list stolen_objects = [(priv/prot object, public copy)]
 
 
 Kind of object copy            distinguishing feature
@@ -154,6 +166,8 @@
     update the original P->h_revision to point directly to the new
     public copy
 
+    add (P, new public copy) to stolen_objects
+
 
 
 Write barrier
diff --git a/c4/doc-stmgc.txt b/c4/doc-stmgc.txt
--- a/c4/doc-stmgc.txt
+++ b/c4/doc-stmgc.txt
@@ -304,3 +304,16 @@
 revision numbers of all threads, and theoretically compact each interval
 of numbers down to only one number, but still keep one active revision
 number per thread.
+
+
+Stealing
+--------
+
+This is done by the *stealing thread* in order to gain access to an
+object that is protected by the *foreign thread*.  Stealing is triggered
+when we, the stealing thread, follow a "handle" created by a foreign
+thread.  The handle has a reference to the normal protected/private
+object.  The process depends on the exact state of the protected/private
+copy.  As a general rule, we may carefully read, but not write, to the
+foreign copies during stealing.
+
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -13,11 +13,6 @@
    a transaction inevitable: we then add 1 to it. */
 static revision_t global_cur_time = 2;
 
-/* 'next_locked_value' is incremented by two for every thread that starts.
-   XXX it should be fixed at some point because right now the process will
-   die if we start more than 0x7fff threads. */
-static revision_t next_locked_value = (LOCKED + 1) | 1;
-
 /* a negative odd number that identifies the currently running
    transaction within the thread. */
 __thread revision_t stm_private_rev_num;
@@ -85,6 +80,11 @@
 }
 #endif
 
+static void steal(gcptr P)
+{
+  abort();
+}
+
 gcptr stm_DirectReadBarrier(gcptr G)
 {
   struct tx_descriptor *d = thread_descriptor;
@@ -165,9 +165,9 @@
   return P;
 
  old_to_young:;
-  revision_t target_lock;
-  target_lock = *(revision_t *)(v & ~(HANDLE_BLOCK_SIZE-1));
-  if (target_lock == d->my_lock)
+  revision_t target_descriptor_index;
+  target_descriptor_index = *(revision_t *)(v & ~(HANDLE_BLOCK_SIZE-1));
+  if (target_descriptor_index == d->descriptor_index)
     {
       P = (gcptr)(*(revision_t *)(v - 2));
       assert(!(P->h_tid & GCFLAG_PUBLIC));
@@ -194,6 +194,7 @@
     {
       /* stealing */
       fprintf(stderr, "read_barrier: %p -> stealing %p...", G, (gcptr)v);
+      steal(P);
       abort();
     }
 }
@@ -577,7 +578,7 @@
           "!!!!!!!!!!!!!!!!!!!!!  [%lx] abort %d\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-          "\n", (long)d->my_lock, num);
+          "\n", (long)d->descriptor_index, num);
   if (num != ABRT_MANUAL && d->max_aborts >= 0 && !d->max_aborts--)
     {
       fprintf(stderr, "unexpected abort!\n");
@@ -773,7 +774,7 @@
       handle_block = (revision_t *)
         ((((intptr_t)handle_block) + HANDLE_BLOCK_SIZE-1)
          & ~(HANDLE_BLOCK_SIZE-1));
-      handle_block[0] = d->my_lock;
+      handle_block[0] = d->descriptor_index;
       handle_block[1] = v;
 
       revision_t w = ((revision_t)(handle_block + 1)) + 2;
@@ -947,7 +948,7 @@
                 (XXX statically we should know when we're outside
                 a transaction) */
 
-  fprintf(stderr, "[%lx] inevitable: %s\n", (long)d->my_lock, why);
+  fprintf(stderr, "[%lx] inevitable: %s\n", (long)d->descriptor_index, why);
 
   cur_time = acquire_inev_mutex_and_mark_global_cur_time();
   if (d->start_time != cur_time)
@@ -1071,6 +1072,10 @@
 
 /************************************************************/
 
+struct tx_descriptor *stm_descriptor_array[MAX_THREADS] = {0};
+static revision_t descriptor_array_next = 0;
+static revision_t descriptor_array_lock = 0;
+
 int DescriptorInit(void)
 {
   if (GCFLAG_PREBUILT != PREBUILT_FLAGS)
@@ -1082,33 +1087,39 @@
 
   if (thread_descriptor == NULL)
     {
+      revision_t i;
       struct tx_descriptor *d = stm_malloc(sizeof(struct tx_descriptor));
       memset(d, 0, sizeof(struct tx_descriptor));
+      spinlock_acquire(descriptor_array_lock, 1);
 
-      /* initialize 'my_lock' to be a unique odd number > LOCKED */
-      while (1)
+      i = descriptor_array_next;
+      while (stm_descriptor_array[i] != NULL)
         {
-          d->my_lock = ACCESS_ONCE(next_locked_value);
-          if (d->my_lock > INTPTR_MAX - 2)
+          i++;
+          if (i == MAX_THREADS)
+            i = 0;
+          if (i == descriptor_array_next)
             {
-              /* XXX fix this limitation */
-              fprintf(stderr, "XXX error: too many threads ever created "
+              fprintf(stderr, "error: too many threads at the same time "
                               "in this process");
               abort();
             }
-          if (bool_cas(&next_locked_value, d->my_lock, d->my_lock + 2))
-            break;
         }
+      descriptor_array_next = i;
+      stm_descriptor_array[i] = d;
+      d->descriptor_index = i;
+      d->my_lock = LOCKED + 2 * i;
       assert(d->my_lock & 1);
-      assert(d->my_lock > LOCKED);
+      assert(d->my_lock >= LOCKED);
       stm_private_rev_num = -1;
       d->private_revision_ref = &stm_private_rev_num;
       d->max_aborts = -1;
       thread_descriptor = d;
 
       fprintf(stderr, "[%lx] pthread %lx starting\n",
-              (long)d->my_lock, (long)pthread_self());
+              (long)d->descriptor_index, (long)pthread_self());
 
+      spinlock_release(descriptor_array_lock);
       return 1;
     }
   else
@@ -1121,6 +1132,7 @@
     assert(d != NULL);
     assert(d->active == 0);
 
+    stm_descriptor_array[d->descriptor_index] = NULL;
     thread_descriptor = NULL;
 
     g2l_delete(&d->public_to_private);
@@ -1142,7 +1154,7 @@
         num_spinloops += d->num_spinloops[i];
 
     p += sprintf(p, "[%lx] finishing: %d commits, %d aborts ",
-                 (long)d->my_lock,
+                 (long)d->descriptor_index,
                  d->num_commits,
                  num_aborts);
 
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -11,7 +11,8 @@
 #define _SRCSTM_ET_H
 
 
-#define LOCKED  ((INTPTR_MAX - 0xffff) | 1)
+#define MAX_THREADS         1024
+#define LOCKED              (INTPTR_MAX - 2*(MAX_THREADS-1))
 
 #define WORD                sizeof(gcptr)
 #define HANDLE_BLOCK_SIZE   (2 * WORD)
@@ -103,6 +104,7 @@
 struct tx_descriptor {
   jmp_buf *setjmp_buf;
   revision_t start_time;
+  revision_t descriptor_index;
   revision_t my_lock;
   revision_t collection_lock;
   gcptr *shadowstack;
@@ -127,11 +129,11 @@
   long long longest_abort_info_time;
   struct FXCache recent_reads_cache;
   revision_t *private_revision_ref;
-  struct tx_descriptor *tx_next, *tx_prev;   /* a doubly linked list */
 };
 
 extern __thread struct tx_descriptor *thread_descriptor;
 extern __thread revision_t stm_private_rev_num;
+extern struct tx_descriptor *stm_descriptor_array[];
 
 /************************************************************/
 
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -84,7 +84,7 @@
     gcptr pseudoprebuilt(size_t size, int tid);
     revision_t get_private_rev_num(void);
     revision_t get_start_time(void);
-    revision_t get_my_lock(void);
+    revision_t get_descriptor_index(void);
 
     //gcptr *addr_of_thread_local(void);
     //int in_nursery(gcptr);
@@ -93,7 +93,6 @@
     /* some constants normally private that are useful in the tests */
     #define WORD                     ...
     #define GC_PAGE_SIZE             ...
-    #define LOCKED                   ...
     #define HANDLE_BLOCK_SIZE        ...
     #define GCFLAG_OLD               ...
     #define GCFLAG_VISITED           ...
@@ -207,9 +206,9 @@
         return thread_descriptor->start_time;
     }
 
-    revision_t get_my_lock(void)
+    revision_t get_descriptor_index(void)
     {
-        return thread_descriptor->my_lock;
+        return thread_descriptor->descriptor_index;
     }
 
     /*gcptr *addr_of_thread_local(void)
@@ -551,7 +550,7 @@
 def decode_handle(r):
     assert (r & 3) == 2
     p = r & ~(lib.HANDLE_BLOCK_SIZE-1)
-    my_lock = ffi.cast("revision_t *", p)[0]
-    assert my_lock >= lib.LOCKED
+    dindex = ffi.cast("revision_t *", p)[0]
+    assert 0 <= dindex < 20
     ptr = ffi.cast("gcptr *", r - 2)[0]
-    return ptr, my_lock
+    return ptr, dindex
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -138,7 +138,7 @@
     lib.stm_begin_inevitable_transaction()
     assert classify(p) == "public"
     assert classify(p2) == "protected"
-    assert decode_handle(p.h_revision) == (p2, lib.get_my_lock())
+    assert decode_handle(p.h_revision) == (p2, lib.get_descriptor_index())
     assert lib.rawgetlong(p, 0) == 28971289
     assert lib.rawgetlong(p2, 0) == 1289222
 
@@ -224,14 +224,36 @@
     assert p4 == p2
     assert list_of_read_objects() == [p2]
 
-def test_stealing_protected_without_backup():
+def test_stealing():
     p = palloc(HDR + WORD)
+    plist = [p]
     def f1(r):
-        lib.setlong(p, 0, 2782172)
+        assert (p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE) == 0
+        p1 = lib.stm_write_barrier(p)   # private copy
+        assert p1 != p
+        assert classify(p) == "public"
+        assert classify(p1) == "private"
+        assert p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
+        lib.rawsetlong(p1, 0, 2782172)
         lib.stm_commit_transaction()
         lib.stm_begin_inevitable_transaction()
+        assert classify(p) == "public"
+        assert classify(p1) == "protected"
+        plist.append(p1)
+        # now p's most recent revision is protected
+        assert p.h_revision % 4 == 2    # a handle
         r.set(2)
+        r.wait(3)
+        assert lib.list_stolen_objects() == plist[-2:]
+        p2 = lib.stm_read_barrier(p1)
+        assert p2 == plist[-1]
     def f2(r):
         r.wait(2)
-        assert lib.getlong(p, 0) == 2782172
+        p2 = lib.stm_read_barrier(p)    # steals
+        assert lib.rawgetlong(p2, 0) == 2782172
+        assert p.h_revision == int(ffi.cast("revision_t", p2))
+        assert p2 == lib.stm_read_barrier(p)
+        assert p2 not in plist
+        plist.append(p2)
+        r.set(3)
     run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Sat Jun  8 21:25:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Jun 2013 21:25:56 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Introduce tx_public_descriptor to cope
 with threads ending
Message-ID: <20130608192556.85E891C06B1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r79:91fdce635f8f
Date: 2013-06-08 21:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/91fdce635f8f/

Log:	Introduce tx_public_descriptor to cope with threads ending

diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt
--- a/c4/doc-objects.txt
+++ b/c4/doc-objects.txt
@@ -83,8 +83,7 @@
 - the PRN (private revision number): odd, negative, changes for every
   transaction that commits
 
-- list active_backup_copies =
-  [(private converted from protected, backup copy)]
+- list active_backup_copies = [(private, backup copy)]
 
 - dict public_to_private = {public obj: private copy}
 
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -82,6 +82,13 @@
 
 static void steal(gcptr P)
 {
+  struct tx_public_descriptor *foreign_pd;
+  revision_t target_descriptor_index;
+  revision_t v = ACCESS_ONCE(P->h_revision);
+  if ((v & 3) != 2)
+    return;
+  target_descriptor_index = *(revision_t *)(v & ~(HANDLE_BLOCK_SIZE-1));
+  //foreign_pd = ACCESS_ONCE(stm_descriptor_array[target_descriptor_index]);
   abort();
 }
 
@@ -167,7 +174,7 @@
  old_to_young:;
   revision_t target_descriptor_index;
   target_descriptor_index = *(revision_t *)(v & ~(HANDLE_BLOCK_SIZE-1));
-  if (target_descriptor_index == d->descriptor_index)
+  if (target_descriptor_index == d->public_descriptor_index)
     {
       P = (gcptr)(*(revision_t *)(v - 2));
       assert(!(P->h_tid & GCFLAG_PUBLIC));
@@ -195,7 +202,7 @@
       /* stealing */
       fprintf(stderr, "read_barrier: %p -> stealing %p...", G, (gcptr)v);
       steal(P);
-      abort();
+      goto retry;
     }
 }
 
@@ -331,7 +338,7 @@
       memcpy(B + 1, P + 1, size - sizeof(*B));
     }
   assert(B->h_tid & GCFLAG_BACKUP_COPY);
-  g2l_insert(&d->private_to_backup, P, B);
+  gcptrlist_insert2(&d->public_descriptor->active_backup_copies, P, B);
   P->h_revision = stm_private_rev_num;
   return P;
 }
@@ -394,10 +401,13 @@
 
 gcptr stm_get_backup_copy(gcptr P)
 {
-  struct tx_descriptor *d = thread_descriptor;
-  wlog_t *entry;
-  G2L_FIND(d->private_to_backup, P, entry, return NULL);
-  return entry->val;
+  struct tx_public_descriptor *pd = thread_descriptor->public_descriptor;
+  long i, size = pd->active_backup_copies.size;
+  gcptr *items = pd->active_backup_copies.items;
+  for (i = 0; i < size; i += 2)
+    if (items[i] == P)
+      return items[i + 1];
+  return NULL;
 }
 
 gcptr stm_get_read_obj(long index)
@@ -568,7 +578,7 @@
   }
 
   gcptrlist_clear(&d->list_of_read_objects);
-  g2l_clear(&d->private_to_backup);
+  gcptrlist_clear(&d->public_descriptor->active_backup_copies);
   abort();//stmgc_abort_transaction(d);
 
   fprintf(stderr,
@@ -578,7 +588,7 @@
           "!!!!!!!!!!!!!!!!!!!!!  [%lx] abort %d\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-          "\n", (long)d->descriptor_index, num);
+          "\n", (long)d->public_descriptor_index, num);
   if (num != ABRT_MANUAL && d->max_aborts >= 0 && !d->max_aborts--)
     {
       fprintf(stderr, "unexpected abort!\n");
@@ -632,7 +642,7 @@
     d->start_real_time.tv_nsec = -1;
   }
   assert(d->list_of_read_objects.size == 0);
-  assert(!g2l_any_entry(&d->private_to_backup));
+  assert(d->public_descriptor->active_backup_copies.size == 0);
   assert(!g2l_any_entry(&d->public_to_private));
 
   d->count_reads = 1;
@@ -774,7 +784,7 @@
       handle_block = (revision_t *)
         ((((intptr_t)handle_block) + HANDLE_BLOCK_SIZE-1)
          & ~(HANDLE_BLOCK_SIZE-1));
-      handle_block[0] = d->descriptor_index;
+      handle_block[0] = d->public_descriptor_index;
       handle_block[1] = v;
 
       revision_t w = ((revision_t)(handle_block + 1)) + 2;
@@ -826,18 +836,19 @@
 void TurnPrivateWithBackupToProtected(struct tx_descriptor *d,
                                       revision_t cur_time)
 {
-  wlog_t *item;
-  G2L_LOOP_FORWARD(d->private_to_backup, item)
+  long i, size = d->public_descriptor->active_backup_copies.size;
+  gcptr *items = d->public_descriptor->active_backup_copies.items;
+
+  for (i = 0; i < size; i += 2)
     {
-      gcptr P = item->addr;
-      gcptr B = item->val;
+      gcptr P = items[i];
+      gcptr B = items[i + 1];
       assert(P->h_revision == stm_private_rev_num);
       assert(B->h_tid & GCFLAG_BACKUP_COPY);
       B->h_revision = cur_time;
       P->h_revision = (revision_t)B;
-
-    } G2L_LOOP_END;
-  g2l_clear(&d->private_to_backup);
+    };
+  gcptrlist_clear(&d->public_descriptor->active_backup_copies);
 }
 
 void CommitTransaction(void)
@@ -846,7 +857,7 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
-  spinlock_acquire(d->collection_lock, 'C');  /* committing */
+  spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
   AcquireLocks(d);
 
   if (is_inevitable(d))
@@ -868,10 +879,10 @@
           if (cur_time & 1)
             {                    // there is another inevitable transaction
               CancelLocks(d);
-              spinlock_release(d->collection_lock);
+              spinlock_release(d->public_descriptor->collection_lock);
               inev_mutex_acquire();   // wait until released
               inev_mutex_release();
-              spinlock_acquire(d->collection_lock, 'C');
+              spinlock_acquire(d->public_descriptor->collection_lock, 'C');
               AcquireLocks(d);
               continue;
             }
@@ -907,7 +918,7 @@
 
   UpdateChainHeads(d, cur_time, localrev);
 
-  spinlock_release(d->collection_lock);
+  spinlock_release(d->public_descriptor->collection_lock);
   d->num_commits++;
   d->active = 0;
   stm_stop_sharedlock();
@@ -948,7 +959,8 @@
                 (XXX statically we should know when we're outside
                 a transaction) */
 
-  fprintf(stderr, "[%lx] inevitable: %s\n", (long)d->descriptor_index, why);
+  fprintf(stderr, "[%lx] inevitable: %s\n",
+          (long)d->public_descriptor_index, why);
 
   cur_time = acquire_inev_mutex_and_mark_global_cur_time();
   if (d->start_time != cur_time)
@@ -1072,8 +1084,8 @@
 
 /************************************************************/
 
-struct tx_descriptor *stm_descriptor_array[MAX_THREADS] = {0};
-static revision_t descriptor_array_next = 0;
+struct tx_public_descriptor *stm_descriptor_array[MAX_THREADS] = {0};
+static revision_t descriptor_array_free_list = 0;
 static revision_t descriptor_array_lock = 0;
 
 int DescriptorInit(void)
@@ -1092,22 +1104,30 @@
       memset(d, 0, sizeof(struct tx_descriptor));
       spinlock_acquire(descriptor_array_lock, 1);
 
-      i = descriptor_array_next;
-      while (stm_descriptor_array[i] != NULL)
-        {
-          i++;
-          if (i == MAX_THREADS)
-            i = 0;
-          if (i == descriptor_array_next)
-            {
+      struct tx_public_descriptor *pd;
+      i = descriptor_array_free_list;
+      pd = stm_descriptor_array[i];
+      if (pd != NULL) {
+          /* we are reusing 'pd' */
+          descriptor_array_free_list = pd->free_list_next;
+          assert(descriptor_array_free_list >= 0);
+      }
+      else {
+          /* no item in the free list */
+          descriptor_array_free_list = i + 1;
+          if (descriptor_array_free_list == MAX_THREADS) {
               fprintf(stderr, "error: too many threads at the same time "
                               "in this process");
               abort();
-            }
-        }
-      descriptor_array_next = i;
-      stm_descriptor_array[i] = d;
-      d->descriptor_index = i;
+          }
+          pd = stm_malloc(sizeof(struct tx_public_descriptor));
+          memset(pd, 0, sizeof(struct tx_public_descriptor));
+          stm_descriptor_array[i] = pd;
+      }
+      pd->free_list_next = -1;
+
+      d->public_descriptor = pd;
+      d->public_descriptor_index = i;
       d->my_lock = LOCKED + 2 * i;
       assert(d->my_lock & 1);
       assert(d->my_lock >= LOCKED);
@@ -1117,7 +1137,7 @@
       thread_descriptor = d;
 
       fprintf(stderr, "[%lx] pthread %lx starting\n",
-              (long)d->descriptor_index, (long)pthread_self());
+              (long)d->public_descriptor_index, (long)pthread_self());
 
       spinlock_release(descriptor_array_lock);
       return 1;
@@ -1128,15 +1148,25 @@
 
 void DescriptorDone(void)
 {
+    revision_t i;
     struct tx_descriptor *d = thread_descriptor;
     assert(d != NULL);
     assert(d->active == 0);
 
-    stm_descriptor_array[d->descriptor_index] = NULL;
+    d->public_descriptor->collection_lock = 0;    /* unlock */
+
+    spinlock_acquire(descriptor_array_lock, 1);
+    i = d->public_descriptor_index;
+    assert(stm_descriptor_array[i] == d->public_descriptor);
+    d->public_descriptor->free_list_next = descriptor_array_free_list;
+    descriptor_array_free_list = i;
+    spinlock_release(descriptor_array_lock);
+
     thread_descriptor = NULL;
 
     g2l_delete(&d->public_to_private);
-    g2l_delete(&d->private_to_backup);
+    assert(d->public_descriptor->active_backup_copies.size == 0);
+    gcptrlist_delete(&d->public_descriptor->active_backup_copies);
     gcptrlist_delete(&d->list_of_read_objects);
     gcptrlist_delete(&d->abortinfo);
     free(d->longest_abort_info);
@@ -1145,7 +1175,6 @@
 #endif
 
     int num_aborts = 0, num_spinloops = 0;
-    int i;
     char line[256], *p = line;
 
     for (i=0; i<ABORT_REASONS; i++)
@@ -1154,7 +1183,7 @@
         num_spinloops += d->num_spinloops[i];
 
     p += sprintf(p, "[%lx] finishing: %d commits, %d aborts ",
-                 (long)d->descriptor_index,
+                 (long)d->public_descriptor_index,
                  d->num_commits,
                  num_aborts);
 
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -101,12 +101,25 @@
 #define SPLP_LOCKED_COMMIT        3
 #define SPINLOOP_REASONS      4
 
+/* this struct contains thread-local data that may be occasionally
+ * accessed by a foreign thread and that must stay around after the
+ * thread shuts down.  It is reused the next time a thread starts. */
+struct tx_public_descriptor {
+  revision_t collection_lock;
+  struct GcPtrList stolen_objects;
+  struct GcPtrList active_backup_copies;
+  revision_t free_list_next;
+  /* xxx gcpage data here */
+};
+
+/* this struct contains all thread-local data that is never accessed
+ * by a foreign thread */
 struct tx_descriptor {
+  struct tx_public_descriptor *public_descriptor;
+  revision_t public_descriptor_index;
   jmp_buf *setjmp_buf;
   revision_t start_time;
-  revision_t descriptor_index;
   revision_t my_lock;
-  revision_t collection_lock;
   gcptr *shadowstack;
   gcptr **shadowstack_end_ref;
 
@@ -123,7 +136,6 @@
   unsigned int num_spinloops[SPINLOOP_REASONS];
   struct GcPtrList list_of_read_objects;
   struct GcPtrList abortinfo;
-  struct G2L private_to_backup;
   struct G2L public_to_private;
   char *longest_abort_info;
   long long longest_abort_info_time;
@@ -133,43 +145,10 @@
 
 extern __thread struct tx_descriptor *thread_descriptor;
 extern __thread revision_t stm_private_rev_num;
-extern struct tx_descriptor *stm_descriptor_array[];
+extern struct tx_public_descriptor *stm_descriptor_array[];
 
 /************************************************************/
 
-//#define STM_BARRIER_P2R(P)
-//    (__builtin_expect((((gcptr)(P))->h_tid & GCFLAG_GLOBAL) == 0, 1) ?
-//     (P) : (typeof(P))stm_DirectReadBarrier(P))
-
-//#define STM_BARRIER_G2R(G)
-//    (assert(((gcptr)(G))->h_tid & GCFLAG_GLOBAL),
-//     (typeof(G))stm_DirectReadBarrier(G))
-
-//#define STM_BARRIER_O2R(O)
-//    (__builtin_expect((((gcptr)(O))->h_tid & GCFLAG_POSSIBLY_OUTDATED) == 0,
-//                      1) ?
-//     (O) : (typeof(O))stm_RepeatReadBarrier(O))
-
-//#define STM_READ_BARRIER_P_FROM_R(P, R_container, offset)
-//    (__builtin_expect((((gcptr)(P))->h_tid & GCFLAG_GLOBAL) == 0, 1) ?
-//     (P) : (typeof(P))stm_DirectReadBarrierFromR((P),
-//                                              (R_container),
-//                                              offset))
-
-//#define STM_BARRIER_P2W(P)
-//    (__builtin_expect((((gcptr)(P))->h_tid & GCFLAG_NOT_WRITTEN) == 0, 1) ?
-//     (P) : (typeof(P))stm_WriteBarrier(P))
-
-//#define STM_BARRIER_G2W(G)
-//    (assert(((gcptr)(G))->h_tid & GCFLAG_GLOBAL),
-//     (typeof(G))stm_WriteBarrier(G))
-
-//#define STM_BARRIER_R2W(R)
-//    (__builtin_expect((((gcptr)(R))->h_tid & GCFLAG_NOT_WRITTEN) == 0, 1) ?
-//     (R) : (typeof(R))stm_WriteBarrierFromReady(R))
-
-//#define STM_BARRIER_O2W(R)  STM_BARRIER_R2W(R)   /* same logic works here */
-
 
 void BeginTransaction(jmp_buf *);
 void BeginInevitableTransaction(void);  /* must save roots around this call */
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -208,7 +208,7 @@
 
     revision_t get_descriptor_index(void)
     {
-        return thread_descriptor->descriptor_index;
+        return thread_descriptor->public_descriptor_index;
     }
 
     /*gcptr *addr_of_thread_local(void)

From noreply at buildbot.pypy.org  Sat Jun  8 22:49:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Jun 2013 22:49:45 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130608204945.C962E1C06B1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r80:1456f52e680d
Date: 2013-06-08 22:49 +0200
http://bitbucket.org/pypy/stmgc/changeset/1456f52e680d/

Log:	in-progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -60,38 +60,6 @@
 
 /************************************************************/
 
-#if 0
-static inline gcptr AddInReadSet(struct tx_descriptor *d, gcptr R)
-{
-  fprintf(stderr, "AddInReadSet(%p)\n", R);
-  d->count_reads++;
-  if (!fxcache_add(&d->recent_reads_cache, R)) {
-      /* not in the cache: it may be the first time we see it,
-       * so insert it into the list */
-      gcptrlist_insert(&d->list_of_read_objects, R);
-  }
-      //      break;
-
-      //  case 2:
-      /* already in the cache, and FX_THRESHOLD reached */
-      //      return Localize(d, R);
-      //  }
-  return R;
-}
-#endif
-
-static void steal(gcptr P)
-{
-  struct tx_public_descriptor *foreign_pd;
-  revision_t target_descriptor_index;
-  revision_t v = ACCESS_ONCE(P->h_revision);
-  if ((v & 3) != 2)
-    return;
-  target_descriptor_index = *(revision_t *)(v & ~(HANDLE_BLOCK_SIZE-1));
-  //foreign_pd = ACCESS_ONCE(stm_descriptor_array[target_descriptor_index]);
-  abort();
-}
-
 gcptr stm_DirectReadBarrier(gcptr G)
 {
   struct tx_descriptor *d = thread_descriptor;
@@ -106,26 +74,26 @@
       v = ACCESS_ONCE(P->h_revision);
       if (!(v & 1))  // "is a pointer", i.e.
         {            //      "has a more recent revision"
-          if (v & 2)
-            goto old_to_young;
-          assert(P->h_tid & GCFLAG_PUBLIC);
+          /* if we land on a P in read_barrier_cache: just return it */
+          gcptr P_next = (gcptr)v;
+          if (FXCACHE_AT(P_next) == P_next)
+            {
+              fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P_next);
+              return P_next;
+            }
+
+          if (P->h_tid & GCFLAG_STUB)
+            goto follow_stub;
 
           gcptr P_prev = P;
-          P = (gcptr)v;
-
-          /* if we land on a P in read_barrier_cache: just return it */
-          if (FXCACHE_AT(P) == P)
-            {
-              fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P);
-              return P;
-            }
+          P = P_next;
+          assert(P->h_tid & GCFLAG_PUBLIC);
 
           v = ACCESS_ONCE(P->h_revision);
           if (!(v & 1))  // "is a pointer", i.e.
             {            //      "has a more recent revision"
-              if (v & 2)
-                goto old_to_young;
-              assert(P->h_tid & GCFLAG_PUBLIC);
+              if (P->h_tid & GCFLAG_STUB)
+                goto follow_stub;
 
               /* we update P_prev->h_revision as a shortcut */
               /* XXX check if this really gives a worse performance than only
@@ -171,12 +139,12 @@
   gcptrlist_insert(&d->list_of_read_objects, P);
   return P;
 
- old_to_young:;
-  revision_t target_descriptor_index;
-  target_descriptor_index = *(revision_t *)(v & ~(HANDLE_BLOCK_SIZE-1));
-  if (target_descriptor_index == d->public_descriptor_index)
+ follow_stub:;
+  struct tx_public_descriptor *foreign_pd = STUB_THREAD(P);
+  if (foreign_pd == d->public_descriptor)
     {
-      P = (gcptr)(*(revision_t *)(v - 2));
+      /* same thread */
+      P = (gcptr)P->h_revision;
       assert(!(P->h_tid & GCFLAG_PUBLIC));
       if (P->h_revision == stm_private_rev_num)
         {
@@ -200,8 +168,8 @@
   else
     {
       /* stealing */
-      fprintf(stderr, "read_barrier: %p -> stealing %p...", G, (gcptr)v);
-      steal(P);
+      fprintf(stderr, "read_barrier: %p -> stealing %p...", G, P);
+      stm_steal_stub(P);
       goto retry;
     }
 }
@@ -401,6 +369,8 @@
 
 gcptr stm_get_backup_copy(gcptr P)
 {
+  assert(P->h_revision == stm_private_rev_num);
+
   struct tx_public_descriptor *pd = thread_descriptor->public_descriptor;
   long i, size = pd->active_backup_copies.size;
   gcptr *items = pd->active_backup_copies.items;
@@ -735,7 +705,7 @@
 #endif
 }
 
-static pthread_mutex_t mutex_prebuilt_gcroots = PTHREAD_MUTEX_INITIALIZER;
+//static pthread_mutex_t mutex_prebuilt_gcroots = PTHREAD_MUTEX_INITIALIZER;
 
 static void UpdateChainHeads(struct tx_descriptor *d, revision_t cur_time,
                              revision_t localrev)
@@ -763,6 +733,11 @@
 #endif
       L->h_revision = new_revision;
 
+      gcptr stub = stm_stub_malloc(d->public_descriptor);
+      stub->h_tid = GCFLAG_PUBLIC | GCFLAG_STUB;
+      stub->h_revision = (revision_t)L;
+      item->val = stub;
+
     } G2L_LOOP_END;
 
   smp_wmb(); /* a memory barrier: make sure the new L->h_revisions are visible
@@ -779,22 +754,12 @@
       assert(!(R->h_tid & GCFLAG_STOLEN));
       assert(R->h_revision != localrev);
 
-      /* XXX compactify and don't leak! */
-      revision_t *handle_block = stm_malloc(3 * WORD);
-      handle_block = (revision_t *)
-        ((((intptr_t)handle_block) + HANDLE_BLOCK_SIZE-1)
-         & ~(HANDLE_BLOCK_SIZE-1));
-      handle_block[0] = d->public_descriptor_index;
-      handle_block[1] = v;
-
-      revision_t w = ((revision_t)(handle_block + 1)) + 2;
-
 #ifdef DUMP_EXTRA
-      fprintf(stderr, "%p->h_revision = %p (UpdateChainHeads2)\n",
-              R, (gcptr)w);
+      fprintf(stderr, "%p->h_revision = %p (stub to %p)\n",
+              R, (gcptr)v, (gcptr)item->val->h_revision);
       /*mark*/
 #endif
-      ACCESS_ONCE(R->h_revision) = w;
+      ACCESS_ONCE(R->h_revision) = v;
 
 #if 0
       if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -13,9 +13,7 @@
 
 #define MAX_THREADS         1024
 #define LOCKED              (INTPTR_MAX - 2*(MAX_THREADS-1))
-
 #define WORD                sizeof(gcptr)
-#define HANDLE_BLOCK_SIZE   (2 * WORD)
 
 /* Description of the flags
  * ------------------------
@@ -52,8 +50,9 @@
  * GCFLAG_STOLEN is set of protected objects after we notice that they
  * have been stolen.
  *
- * GCFLAG_STUB is used for debugging: it's set on stub objects made by
- * stealing or by major collections.
+ * GCFLAG_STUB is set on stub objects made by stealing or by major
+ * collections.  It's removed once the stub's protected h_revision
+ * target is stolen and replaced by a regular public object.
  */
 #define GCFLAG_OLD               (STM_FIRST_GCFLAG << 0)
 #define GCFLAG_VISITED           (STM_FIRST_GCFLAG << 1)
@@ -64,7 +63,7 @@
 #define GCFLAG_WRITE_BARRIER     (STM_FIRST_GCFLAG << 6)
 #define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 7)
 #define GCFLAG_STOLEN            (STM_FIRST_GCFLAG << 8)
-#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 9)   /* debugging */
+#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 9)
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
@@ -106,6 +105,8 @@
  * thread shuts down.  It is reused the next time a thread starts. */
 struct tx_public_descriptor {
   revision_t collection_lock;
+  struct stub_block_s *stub_blocks;
+  gcptr stub_free_list;
   struct GcPtrList stolen_objects;
   struct GcPtrList active_backup_copies;
   revision_t free_list_next;
diff --git a/c4/steal.c b/c4/steal.c
new file mode 100644
--- /dev/null
+++ b/c4/steal.c
@@ -0,0 +1,57 @@
+#include "stmimpl.h"
+
+
+#define STUB_PAGE         (4096 - 2*WORD)
+#define STUB_NB_OBJS      ((STUB_BLOCK_SIZE - 2*WORD) /    \
+                             sizeof(struct stm_object_s))
+
+struct stub_block_s {
+    struct tx_public_descriptor *thread;
+    struct stub_block_s *next;
+    struct stm_object_s stubs[STUB_NB_OBJS];
+};
+
+gcptr stm_stub_malloc(struct tx_public_descriptor *pd)
+{
+    gcptr p = pd->stub_free_list;
+    if (p == NULL) {
+        assert(sizeof(struct stub_block_s) == STUB_BLOCK_SIZE);
+
+        char *page = stm_malloc(STUB_PAGE);
+        char *page_end = page + STUB_PAGE;
+        page += (-(revision_t)page) & (STUB_BLOCK_SIZE-1);  /* round up */
+
+        struct stub_block_s *b = (struct stub_block_s *)page;
+        struct stub_block_s *nextb = NULL;
+        gcptr nextp = NULL;
+        int i;
+
+        while (((char *)(b + 1)) <= page_end) {
+            b->thread = pd;
+            b->next = nextb;
+            for (i = 0; i < STUB_NB_OBJS; i++) {
+                b->stubs[i].h_revision = (revision_t)nextp;
+                nextp = &b->stubs[i];
+            }
+            b++;
+        }
+        assert(nextp != NULL);
+        p = nextp;
+    }
+    pd->stub_free_list = (gcptr)p->h_revision;
+    assert(STUB_THREAD(p) == pd);
+    return p;
+}
+
+void stm_steal_stub(gcptr P)
+{
+    abort();
+  struct tx_public_descriptor *foreign_pd;
+  revision_t target_descriptor_index;
+  revision_t v = ACCESS_ONCE(P->h_revision);
+  if ((v & 3) != 2)
+    return;
+  target_descriptor_index = *(revision_t *)(v & ~(STUB_BLOCK_SIZE-1));
+  foreign_pd = stm_descriptor_array[target_descriptor_index];
+  abort();
+}
diff --git a/c4/steal.h b/c4/steal.h
new file mode 100644
--- /dev/null
+++ b/c4/steal.h
@@ -0,0 +1,14 @@
+#ifndef _SRCSTM_STEAL_H
+#define _SRCSTM_STEAL_H
+
+
+#define STUB_BLOCK_SIZE   (16 * WORD)    /* power of two */
+
+#define STUB_THREAD(h)    (*(struct tx_public_descriptor **)           \
+                            (((revision_t)(h)) & ~(STUB_BLOCK_SIZE-1)))
+
+gcptr stm_stub_malloc(struct tx_public_descriptor *);
+void stm_steal_stub(gcptr);
+
+
+#endif
diff --git a/c4/stmimpl.h b/c4/stmimpl.h
--- a/c4/stmimpl.h
+++ b/c4/stmimpl.h
@@ -31,6 +31,7 @@
 #include "lists.h"
 #include "dbgmem.h"
 #include "et.h"
+#include "steal.h"
 #include "stmsync.h"
 
 #endif
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -5,11 +5,11 @@
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 header_files = [os.path.join(parent_dir, _n) for _n in
-                "et.h lists.h "
+                "et.h lists.h steal.h "
                 "stmsync.h dbgmem.h fprintcolor.h "
                 "stmgc.h stmimpl.h atomic_ops.h".split()]
 source_files = [os.path.join(parent_dir, _n) for _n in
-                "et.c lists.c "
+                "et.c lists.c steal.c "
                 "stmsync.c dbgmem.c fprintcolor.c".split()]
 
 _pycache_ = os.path.join(parent_dir, 'test', '__pycache__')
@@ -70,6 +70,7 @@
     void AbortTransaction(int);
     gcptr stm_get_backup_copy(gcptr);
     gcptr stm_get_read_obj(long index);
+    void *STUB_THREAD(gcptr);
 
     gcptr getptr(gcptr, long);
     void setptr(gcptr, long, gcptr);
@@ -84,7 +85,7 @@
     gcptr pseudoprebuilt(size_t size, int tid);
     revision_t get_private_rev_num(void);
     revision_t get_start_time(void);
-    revision_t get_descriptor_index(void);
+    void *my_stub_thread(void);
 
     //gcptr *addr_of_thread_local(void);
     //int in_nursery(gcptr);
@@ -93,7 +94,7 @@
     /* some constants normally private that are useful in the tests */
     #define WORD                     ...
     #define GC_PAGE_SIZE             ...
-    #define HANDLE_BLOCK_SIZE        ...
+    #define STUB_BLOCK_SIZE          ...
     #define GCFLAG_OLD               ...
     #define GCFLAG_VISITED           ...
     #define GCFLAG_PUBLIC            ...
@@ -206,9 +207,9 @@
         return thread_descriptor->start_time;
     }
 
-    revision_t get_descriptor_index(void)
+    void *my_stub_thread(void)
     {
-        return thread_descriptor->public_descriptor_index;
+        return (void *)thread_descriptor->public_descriptor;
     }
 
     /*gcptr *addr_of_thread_local(void)
@@ -526,10 +527,14 @@
     private = p.h_revision == lib.get_private_rev_num()
     public  = (p.h_tid & GCFLAG_PUBLIC) != 0
     backup  = (p.h_tid & GCFLAG_BACKUP_COPY) != 0
+    stub    = (p.h_tid & GCFLAG_STUB) != 0
     assert private + public + backup <= 1
+    assert stub <= public
     if private:
         return "private"
     if public:
+        if stub:
+            return "stub"
         return "public"
     if backup:
         return "backup"
@@ -547,10 +552,4 @@
         index += 1
     return result
 
-def decode_handle(r):
-    assert (r & 3) == 2
-    p = r & ~(lib.HANDLE_BLOCK_SIZE-1)
-    dindex = ffi.cast("revision_t *", p)[0]
-    assert 0 <= dindex < 20
-    ptr = ffi.cast("gcptr *", r - 2)[0]
-    return ptr, dindex
+stub_thread = lib.STUB_THREAD
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -80,10 +80,9 @@
     assert p.h_revision == lib.get_private_rev_num()
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
-    assert lib.stm_get_backup_copy(p) == ffi.NULL
     assert classify(p) == "protected"
     assert classify(pback) == "backup"
-    assert ffi.cast("revision_t *", p.h_revision) == pback
+    assert ffi.cast("gcptr", p.h_revision) == pback
 
 def test_protected_backup_reused():
     p = nalloc(HDR + WORD)
@@ -95,7 +94,6 @@
     assert pback != p
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
-    assert lib.stm_get_backup_copy(p) == ffi.NULL
     assert classify(p) == "protected"
     assert classify(pback) == "backup"
     assert lib.rawgetlong(p, 0) == 927122
@@ -138,7 +136,9 @@
     lib.stm_begin_inevitable_transaction()
     assert classify(p) == "public"
     assert classify(p2) == "protected"
-    assert decode_handle(p.h_revision) == (p2, lib.get_descriptor_index())
+    pstub = ffi.cast("gcptr", p.h_revision)
+    assert classify(pstub) == "stub"
+    assert stub_thread(pstub) == lib.my_stub_thread()
     assert lib.rawgetlong(p, 0) == 28971289
     assert lib.rawgetlong(p2, 0) == 1289222
 
@@ -241,7 +241,7 @@
         assert classify(p1) == "protected"
         plist.append(p1)
         # now p's most recent revision is protected
-        assert p.h_revision % 4 == 2    # a handle
+        assert classify(ffi.cast("gcptr", p.h_revision)) == "stub"
         r.set(2)
         r.wait(3)
         assert lib.list_stolen_objects() == plist[-2:]

From noreply at buildbot.pypy.org  Sat Jun  8 22:57:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Jun 2013 22:57:49 +0200 (CEST)
Subject: [pypy-commit] stmgc default: progress in stealing
Message-ID: <20130608205749.E74E11C06B1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r81:77e624bcbfd2
Date: 2013-06-08 22:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/77e624bcbfd2/

Log:	progress in stealing

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -45,13 +45,23 @@
 
 void stm_steal_stub(gcptr P)
 {
-    abort();
-  struct tx_public_descriptor *foreign_pd;
-  revision_t target_descriptor_index;
-  revision_t v = ACCESS_ONCE(P->h_revision);
-  if ((v & 3) != 2)
-    return;
-  target_descriptor_index = *(revision_t *)(v & ~(STUB_BLOCK_SIZE-1));
-  foreign_pd = stm_descriptor_array[target_descriptor_index];
-  abort();
+    struct tx_public_descriptor *foreign_pd = STUB_THREAD(P);
+
+    spinlock_acquire(foreign_pd->collection_lock, 'S');   /*stealing*/
+
+    if (!(P->h_tid & GCFLAG_STUB))
+        goto done;     /* un-stubbed while we waited for the lock */
+
+    /* XXX check for now that P is a regular protected object */
+    gcptr L = (gcptr)P->h_revision;
+    gcptr Q = stmgc_duplicate(L);
+    Q->h_tid |= GCFLAG_PUBLIC;
+
+    smp_wmb();
+
+    P->h_revision = (revision_t)Q;
+    P->h_tid &= ~GCFLAG_STUB;
+
+ done:
+    spinlock_release(foreign_pd->collection_lock);
 }
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -251,6 +251,7 @@
         r.wait(2)
         p2 = lib.stm_read_barrier(p)    # steals
         assert lib.rawgetlong(p2, 0) == 2782172
+        assert p2 == lib.stm_read_barrier(p)    # short-circuit h_revision
         assert p.h_revision == int(ffi.cast("revision_t", p2))
         assert p2 == lib.stm_read_barrier(p)
         assert p2 not in plist

From noreply at buildbot.pypy.org  Sat Jun  8 23:00:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Jun 2013 23:00:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fix
Message-ID: <20130608210001.318BD1C06B1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r82:1366ee6b9a4d
Date: 2013-06-08 22:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/1366ee6b9a4d/

Log:	fix

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -56,10 +56,10 @@
     gcptr L = (gcptr)P->h_revision;
     gcptr Q = stmgc_duplicate(L);
     Q->h_tid |= GCFLAG_PUBLIC;
+    P->h_revision = (revision_t)Q;
 
     smp_wmb();
 
-    P->h_revision = (revision_t)Q;
     P->h_tid &= ~GCFLAG_STUB;
 
  done:

From noreply at buildbot.pypy.org  Sun Jun  9 10:23:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Jun 2013 10:23:49 +0200 (CEST)
Subject: [pypy-commit] stmgc default: tweaks
Message-ID: <20130609082349.6D5EB1C094A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r83:185610f34659
Date: 2013-06-09 10:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/185610f34659/

Log:	tweaks

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -74,25 +74,18 @@
       v = ACCESS_ONCE(P->h_revision);
       if (!(v & 1))  // "is a pointer", i.e.
         {            //      "has a more recent revision"
-          /* if we land on a P in read_barrier_cache: just return it */
-          gcptr P_next = (gcptr)v;
-          if (FXCACHE_AT(P_next) == P_next)
-            {
-              fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P_next);
-              return P_next;
-            }
-
-          if (P->h_tid & GCFLAG_STUB)
+          if (v & 2)
             goto follow_stub;
 
           gcptr P_prev = P;
-          P = P_next;
+          P = (gcptr)v;
           assert(P->h_tid & GCFLAG_PUBLIC);
 
           v = ACCESS_ONCE(P->h_revision);
+
           if (!(v & 1))  // "is a pointer", i.e.
             {            //      "has a more recent revision"
-              if (P->h_tid & GCFLAG_STUB)
+              if (v & 2)
                 goto follow_stub;
 
               /* we update P_prev->h_revision as a shortcut */
@@ -104,6 +97,13 @@
             }
         }
 
+      /* if we land on a P in read_barrier_cache: just return it */
+      if (FXCACHE_AT(P) == P)
+        {
+          fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P);
+          return P;
+        }
+
       if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
         {
           wlog_t *item;
@@ -144,7 +144,7 @@
   if (foreign_pd == d->public_descriptor)
     {
       /* same thread */
-      P = (gcptr)P->h_revision;
+      P = (gcptr)v;
       assert(!(P->h_tid & GCFLAG_PUBLIC));
       if (P->h_revision == stm_private_rev_num)
         {
@@ -735,7 +735,7 @@
 
       gcptr stub = stm_stub_malloc(d->public_descriptor);
       stub->h_tid = GCFLAG_PUBLIC | GCFLAG_STUB;
-      stub->h_revision = (revision_t)L;
+      stub->h_revision = ((revision_t)L) | 2;
       item->val = stub;
 
     } G2L_LOOP_END;
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -50,9 +50,10 @@
  * GCFLAG_STOLEN is set of protected objects after we notice that they
  * have been stolen.
  *
- * GCFLAG_STUB is set on stub objects made by stealing or by major
- * collections.  It's removed once the stub's protected h_revision
- * target is stolen and replaced by a regular public object.
+ * GCFLAG_STUB is set for debugging on stub objects made by stealing or
+ * by major collections.  'p_stub->h_revision' might be a value
+ * that is == 2 (mod 4): in this case they point to a protected/private
+ * object that belongs to the thread 'STUB_THREAD(p_stub)'.
  */
 #define GCFLAG_OLD               (STM_FIRST_GCFLAG << 0)
 #define GCFLAG_VISITED           (STM_FIRST_GCFLAG << 1)
@@ -63,7 +64,7 @@
 #define GCFLAG_WRITE_BARRIER     (STM_FIRST_GCFLAG << 6)
 #define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 7)
 #define GCFLAG_STOLEN            (STM_FIRST_GCFLAG << 8)
-#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 9)
+#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 9)   /* debugging */
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
@@ -166,6 +167,7 @@
 gcptr _stm_nonrecord_barrier(gcptr, int *);
 gcptr stm_get_backup_copy(gcptr);
 gcptr stm_get_read_obj(long);  /* debugging */
+gcptr stmgc_duplicate(gcptr);
 
 int DescriptorInit(void);
 void DescriptorDone(void);
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -49,18 +49,17 @@
 
     spinlock_acquire(foreign_pd->collection_lock, 'S');   /*stealing*/
 
-    if (!(P->h_tid & GCFLAG_STUB))
+    revision_t v = ACCESS_ONCE(P->h_revision);
+    if ((v & 3) != 2)
         goto done;     /* un-stubbed while we waited for the lock */
 
-    /* XXX check for now that P is a regular protected object */
-    gcptr L = (gcptr)P->h_revision;
+    gcptr L = (gcptr)(v - 2);
     gcptr Q = stmgc_duplicate(L);
     Q->h_tid |= GCFLAG_PUBLIC;
-    P->h_revision = (revision_t)Q;
 
     smp_wmb();
 
-    P->h_tid &= ~GCFLAG_STUB;
+    P->h_revision = (revision_t)Q;
 
  done:
     spinlock_release(foreign_pd->collection_lock);

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:46 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:46 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: try to speedup things
Message-ID: <20130609094146.27BB41C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64827:374e8a9d721a
Date: 2013-06-06 14:19 +0200
http://bitbucket.org/pypy/pypy/changeset/374e8a9d721a/

Log:	try to speedup things

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -49,16 +49,19 @@
         self.i = i2
 
     def getslice(self, start, end):
+        assert start > 0
         assert end > 0
         return self.s[start:end]
 
     def skip_whitespace(self):
-        while not self.eof():
-            ch = self.peek()
+        i = self.i
+        while i < len(self.s):
+            ch = self.s[i]
             if is_whitespace(ch):
-                self.next()
+                i+=1
             else:
                 break
+        self.i = i
 
     @specialize.arg(1)
     def _raise(self, msg, *args):
@@ -240,14 +243,16 @@
 
     def decode_string(self):
         start = self.i
+        i = self.i
         bits = 0
-        while not self.eof():
+        while i < len(self.s):
             # this loop is a fast path for strings which do not contain escape
             # characters
-            ch = self.next()
+            ch = self.s[i]
+            i += 1
             bits |= ord(ch)
             if ch == '"':
-                content_utf8 = self.getslice(start, self.i-1)
+                content_utf8 = self.getslice(start, i-1)
                 if bits & 0x80:
                     # the 8th bit is set, it's an utf8 strnig
                     content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
@@ -255,10 +260,11 @@
                     # ascii only, faster to decode
                     content_unicode = content_utf8.decode('ascii')
                 self.last_type = TYPE_STRING
+                self.i = i
                 return self.space.wrap(content_unicode)
             elif ch == '\\':
-                content_so_far = self.getslice(start, self.i-1)
-                self.unget()
+                content_so_far = self.getslice(start, i-1)
+                self.i = i-1
                 return self.decode_string_escaped(start, content_so_far)
         self._raise("Unterminated string starting at char %d", start)
 

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:47 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:47 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: here are dragons: valgrind shows that
 a good percentage of time was spent in
Message-ID: <20130609094147.852EF1C094A@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64828:c9e7d8554e5f
Date: 2013-06-06 15:33 +0200
http://bitbucket.org/pypy/pypy/changeset/c9e7d8554e5f/

Log:	here are dragons: valgrind shows that a good percentage of time was
	spent in taking the slice + converting it to unicode. Instead, we
	directly create an unicode string by copying the relevant characters
	from the original string, but we need to go to the level of "low
	level helpers" to do that, with llstr&co.

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -19,6 +19,19 @@
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 
+def strslice2unicode_ascii(s, start, end):
+    from rpython.rtyper.annlowlevel import llstr, hlunicode
+    from rpython.rtyper.lltypesystem.rstr import malloc, UNICODE
+    from rpython.rtyper.lltypesystem.lltype import cast_primitive, UniChar
+    length = end-start
+    ll_s = llstr(s)
+    ll_res = malloc(UNICODE, length)
+    ll_res.hash = 0
+    for i in range(length):
+        ch = ll_s.chars[start+i]
+        ll_res.chars[i] = cast_primitive(UniChar, ch)
+    return hlunicode(ll_res)
+
 class JSONDecoder(object):
     def __init__(self, space, s):
         self.space = space
@@ -252,13 +265,13 @@
             i += 1
             bits |= ord(ch)
             if ch == '"':
-                content_utf8 = self.getslice(start, i-1)
                 if bits & 0x80:
                     # the 8th bit is set, it's an utf8 strnig
+                    content_utf8 = self.getslice(start, i-1)
                     content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
                 else:
-                    # ascii only, faster to decode
-                    content_unicode = content_utf8.decode('ascii')
+                    # ascii only, fast path
+                    content_unicode = strslice2unicode_ascii(self.s, start, i-1)
                 self.last_type = TYPE_STRING
                 self.i = i
                 return self.space.wrap(content_unicode)

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:48 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:48 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: speed up parse_digits by storing
	self.i only at the end
Message-ID: <20130609094148.D7CA51C0EB9@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64829:16e72115b986
Date: 2013-06-06 16:28 +0200
http://bitbucket.org/pypy/pypy/changeset/16e72115b986/

Log:	speed up parse_digits by storing self.i only at the end

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -178,16 +178,18 @@
         "Parse a sequence of digits as a decimal number. No sign allowed"
         intval = 0
         count = 0
-        while not self.eof():
-            ch = self.peek()
+        i = self.i
+        while i < len(self.s):
+            ch = self.s[i]
             if ch.isdigit():
                 intval = intval*10 + ord(ch)-ord('0')
                 count += 1
-                self.next()
+                i += 1
             else:
                 break
         if count == 0:
-            self._raise("Expected digit at char %d", self.i)
+            self._raise("Expected digit at char %d", i)
+        self.i = i
         return intval, count
         
     def decode_array(self):
@@ -252,8 +254,6 @@
                             ch, self.i)
         self._raise("Unterminated object starting at char %d", start)
 
-
-
     def decode_string(self):
         start = self.i
         i = self.i

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:50 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:50 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: call math.pow only when it's really
 needed. Believe it or not, it saves another 5% on my benchmark
Message-ID: <20130609094150.1D06F1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64830:e851f2e812ec
Date: 2013-06-06 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/e851f2e812ec/

Log:	call math.pow only when it's really needed. Believe it or not, it
	saves another 5% on my benchmark

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -160,7 +160,8 @@
         if is_float:
             # build the float
             floatval = intval + frcval
-            floatval = floatval * math.pow(10, exp)
+            if exp != 0:
+                floatval = floatval * math.pow(10, exp)
             return self.space.wrap(floatval)
         else:
             return self.space.wrap(intval)

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:51 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:51 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: rename JSONDecoder.i to .pos,
 just because it's a better name
Message-ID: <20130609094151.4B9FF1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64831:ee4fe752ea20
Date: 2013-06-06 17:22 +0200
http://bitbucket.org/pypy/pypy/changeset/ee4fe752ea20/

Log:	rename JSONDecoder.i to .pos, just because it's a better name

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -36,14 +36,14 @@
     def __init__(self, space, s):
         self.space = space
         self.s = s
-        self.i = 0
+        self.pos = 0
         self.last_type = TYPE_UNKNOWN
 
     def eof(self):
-        return self.i == len(self.s)
+        return self.pos == len(self.s)
 
     def peek(self):
-        return self.s[self.i]
+        return self.s[self.pos]
 
     def peek_maybe(self):
         if self.eof():
@@ -53,13 +53,13 @@
 
     def next(self):
         ch = self.peek()
-        self.i += 1
+        self.pos += 1
         return ch
 
     def unget(self):
-        i2 = self.i - 1
-        assert i2 > 0 # so that we can use self.i as slice start
-        self.i = i2
+        i2 = self.pos - 1
+        assert i2 > 0 # so that we can use self.pos as slice start
+        self.pos = i2
 
     def getslice(self, start, end):
         assert start > 0
@@ -67,14 +67,14 @@
         return self.s[start:end]
 
     def skip_whitespace(self):
-        i = self.i
+        i = self.pos
         while i < len(self.s):
             ch = self.s[i]
             if is_whitespace(ch):
                 i+=1
             else:
                 break
-        self.i = i
+        self.pos = i
 
     @specialize.arg(1)
     def _raise(self, msg, *args):
@@ -105,35 +105,35 @@
             return self.decode_false()
         else:
             self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
-                        ch, self.i)
+                        ch, self.pos)
 
     def decode_null(self):
         N = len('ull')
-        if (self.i+N <= len(self.s) and
+        if (self.pos+N <= len(self.s) and
             self.next() == 'u' and
             self.next() == 'l' and
             self.next() == 'l'):
             return self.space.w_None
-        self._raise("Error when decoding null at char %d", self.i)
+        self._raise("Error when decoding null at char %d", self.pos)
 
     def decode_true(self):
         N = len('rue')
-        if (self.i+N <= len(self.s) and
+        if (self.pos+N <= len(self.s) and
             self.next() == 'r' and
             self.next() == 'u' and
             self.next() == 'e'):
             return self.space.w_True
-        self._raise("Error when decoding true at char %d", self.i)
+        self._raise("Error when decoding true at char %d", self.pos)
 
     def decode_false(self):
         N = len('alse')
-        if (self.i+N <= len(self.s) and
+        if (self.pos+N <= len(self.s) and
             self.next() == 'a' and
             self.next() == 'l' and
             self.next() == 's' and
             self.next() == 'e'):
             return self.space.w_False
-        self._raise("Error when decoding false at char %d", self.i)
+        self._raise("Error when decoding false at char %d", self.pos)
 
     def decode_numeric(self):
         intval = self.parse_integer()
@@ -179,7 +179,7 @@
         "Parse a sequence of digits as a decimal number. No sign allowed"
         intval = 0
         count = 0
-        i = self.i
+        i = self.pos
         while i < len(self.s):
             ch = self.s[i]
             if ch.isdigit():
@@ -190,11 +190,11 @@
                 break
         if count == 0:
             self._raise("Expected digit at char %d", i)
-        self.i = i
+        self.pos = i
         return intval, count
         
     def decode_array(self):
-        start = self.i
+        start = self.pos
         w_list = self.space.newlist([])
         self.skip_whitespace()
         while not self.eof():
@@ -214,12 +214,12 @@
                 pass
             else:
                 self._raise("Unexpected '%s' when decoding array (char %d)",
-                            ch, self.i)
+                            ch, self.pos)
         self._raise("Unterminated array starting at char %d", start)
 
 
     def decode_object(self):
-        start = self.i
+        start = self.pos
         w_dict = self.space.newdict()
         while not self.eof():
             ch = self.peek()
@@ -237,7 +237,7 @@
                 break
             ch = self.next()
             if ch != ':':
-                self._raise("No ':' found at char %d", self.i)
+                self._raise("No ':' found at char %d", self.pos)
             self.skip_whitespace()
             #
             w_value = self.decode_any()
@@ -252,12 +252,12 @@
                 pass
             else:
                 self._raise("Unexpected '%s' when decoding object (char %d)",
-                            ch, self.i)
+                            ch, self.pos)
         self._raise("Unterminated object starting at char %d", start)
 
     def decode_string(self):
-        start = self.i
-        i = self.i
+        start = self.pos
+        i = self.pos
         bits = 0
         while i < len(self.s):
             # this loop is a fast path for strings which do not contain escape
@@ -274,11 +274,11 @@
                     # ascii only, fast path
                     content_unicode = strslice2unicode_ascii(self.s, start, i-1)
                 self.last_type = TYPE_STRING
-                self.i = i
+                self.pos = i
                 return self.space.wrap(content_unicode)
             elif ch == '\\':
                 content_so_far = self.getslice(start, i-1)
-                self.i = i-1
+                self.pos = i-1
                 return self.decode_string_escaped(start, content_so_far)
         self._raise("Unterminated string starting at char %d", start)
 
@@ -314,16 +314,16 @@
         elif ch == 'u':
             return self.decode_escape_sequence_unicode(builder)
         else:
-            self._raise("Invalid \\escape: %s (char %d)", ch, self.i-1)
+            self._raise("Invalid \\escape: %s (char %d)", ch, self.pos-1)
 
     def decode_escape_sequence_unicode(self, builder):
         # at this point we are just after the 'u' of the \u1234 sequence.
-        hexdigits = self.getslice(self.i, self.i+4)
-        self.i += 4
+        hexdigits = self.getslice(self.pos, self.pos+4)
+        self.pos += 4
         try:
             uchr = unichr(int(hexdigits, 16))
         except ValueError:
-            self._raise("Invalid \uXXXX escape (char %d)", self.i-1)
+            self._raise("Invalid \uXXXX escape (char %d)", self.pos-1)
             return # help the annotator to know that we'll never go beyond
                    # this point
         #
@@ -338,7 +338,7 @@
     w_res = decoder.decode_any()
     decoder.skip_whitespace()
     if not decoder.eof():
-        start = decoder.i
+        start = decoder.pos
         end = len(decoder.s)
         raise operationerrfmt(space.w_ValueError, "Extra data: char %d - %d", start, end)
     return w_res
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -4,7 +4,7 @@
 
 def test_skip_whitespace():
     dec = JSONDecoder('fake space', '   hello   ')
-    assert dec.i == 0
+    assert dec.pos == 0
     dec.skip_whitespace()
     assert dec.next() == 'h'
     assert dec.next() == 'e'

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:52 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:52 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add a comment to this function,
 and give it a better name
Message-ID: <20130609094152.841EC1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64832:286ff8c96fd4
Date: 2013-06-06 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/286ff8c96fd4/

Log:	add a comment to this function, and give it a better name

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -16,10 +16,17 @@
         return 0.0
     return x * NEG_POW_10[exp]
 
-TYPE_UNKNOWN = 0
-TYPE_STRING = 1
+def strslice2unicode_latin1(s, start, end):
+    """
+    Convert s[start:end] to unicode. s is supposed to be an RPython string
+    encoded in latin-1, which means that the numeric value of each char is the
+    same as the corresponding unicode code point.
 
-def strslice2unicode_ascii(s, start, end):
+    Internally it's implemented at the level of low-level helpers, to avoid
+    the extra copy we would need if we take the actual slice first.
+    
+    No bound checking is done, use carefully.
+    """
     from rpython.rtyper.annlowlevel import llstr, hlunicode
     from rpython.rtyper.lltypesystem.rstr import malloc, UNICODE
     from rpython.rtyper.lltypesystem.lltype import cast_primitive, UniChar
@@ -32,6 +39,8 @@
         ll_res.chars[i] = cast_primitive(UniChar, ch)
     return hlunicode(ll_res)
 
+TYPE_UNKNOWN = 0
+TYPE_STRING = 1
 class JSONDecoder(object):
     def __init__(self, space, s):
         self.space = space
@@ -271,8 +280,10 @@
                     content_utf8 = self.getslice(start, i-1)
                     content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
                 else:
-                    # ascii only, fast path
-                    content_unicode = strslice2unicode_ascii(self.s, start, i-1)
+                    # ascii only, fast path (ascii is a strict subset of
+                    # latin1, and we already checked that all the chars are <
+                    # 128)
+                    content_unicode = strslice2unicode_latin1(self.s, start, i-1)
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:55 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:55 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: start to refactor the decoder to avoid
 continuously updating .pos
Message-ID: <20130609094155.E4E1D1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64833:a3f22cd8a330
Date: 2013-06-06 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/a3f22cd8a330/

Log:	start to refactor the decoder to avoid continuously updating .pos

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -75,22 +75,21 @@
         assert end > 0
         return self.s[start:end]
 
-    def skip_whitespace(self):
-        i = self.pos
+    def skip_whitespace(self, i):
         while i < len(self.s):
             ch = self.s[i]
             if is_whitespace(ch):
                 i+=1
             else:
                 break
-        self.pos = i
+        return i
 
     @specialize.arg(1)
     def _raise(self, msg, *args):
         raise operationerrfmt(self.space.w_ValueError, msg, *args)
 
     def decode_any(self):
-        self.skip_whitespace()
+        self.pos = self.skip_whitespace(self.pos)
         ch = self.peek()
         if ch == '"':
             self.next()
@@ -203,21 +202,25 @@
         return intval, count
         
     def decode_array(self):
+        w_list = self.space.newlist([])
         start = self.pos
-        w_list = self.space.newlist([])
-        self.skip_whitespace()
-        while not self.eof():
-            ch = self.peek()
+        i = self.skip_whitespace(start)
+        while i < len(self.s):
+            ch = self.s[i]
             if ch == ']':
-                self.next()
+                self.pos = i+1
                 return w_list
+            self.pos = i
             w_item = self.decode_any()
+            i = self.pos
             self.space.call_method(w_list, 'append', w_item)
-            self.skip_whitespace()
-            if self.eof():
+            i = self.skip_whitespace(i)
+            if i == len(self.s):
                 break
-            ch = self.next()
+            ch = self.s[i]
+            i += 1
             if ch == ']':
+                self.pos = i
                 return w_list
             elif ch == ',':
                 pass
@@ -241,17 +244,17 @@
             w_name = self.decode_any()
             if self.last_type != TYPE_STRING:
                 self._raise("Key name must be string for object starting at char %d", start)
-            self.skip_whitespace()
+            self.pos = self.skip_whitespace(self.pos)
             if self.eof():
                 break
             ch = self.next()
             if ch != ':':
                 self._raise("No ':' found at char %d", self.pos)
-            self.skip_whitespace()
+            self.pos = self.skip_whitespace(self.pos)
             #
             w_value = self.decode_any()
             self.space.setitem(w_dict, w_name, w_value)
-            self.skip_whitespace()
+            self.pos = self.skip_whitespace(self.pos)
             if self.eof():
                 break
             ch = self.next()
@@ -347,7 +350,7 @@
 def loads(space, s):
     decoder = JSONDecoder(space, s)
     w_res = decoder.decode_any()
-    decoder.skip_whitespace()
+    decoder.pos = decoder.skip_whitespace(decoder.pos)
     if not decoder.eof():
         start = decoder.pos
         end = len(decoder.s)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -3,16 +3,12 @@
 from pypy.module._fastjson.interp_decoder import JSONDecoder
 
 def test_skip_whitespace():
-    dec = JSONDecoder('fake space', '   hello   ')
+    s = '   hello   '
+    dec = JSONDecoder('fake space', s)
     assert dec.pos == 0
-    dec.skip_whitespace()
-    assert dec.next() == 'h'
-    assert dec.next() == 'e'
-    assert dec.next() == 'l'
-    assert dec.next() == 'l'
-    assert dec.next() == 'o'
-    dec.skip_whitespace()
-    assert dec.eof()
+    assert dec.skip_whitespace(0) == 3
+    assert dec.skip_whitespace(3) == 3
+    assert dec.skip_whitespace(8) == len(s)
 
     
 

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:57 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:57 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: put a \0 sentinel at the end of the
 string: the cost of the string copy is neligible but this let us to avoid
 lots of eof() checks during the parsing. Also,
 relax the dependency on self.pos,
 and explicitly pass the current index around
Message-ID: <20130609094157.B19101C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64834:f642cad5d507
Date: 2013-06-07 01:13 +0200
http://bitbucket.org/pypy/pypy/changeset/f642cad5d507/

Log:	put a \0 sentinel at the end of the string: the cost of the string
	copy is neligible but this let us to avoid lots of eof() checks
	during the parsing. Also, relax the dependency on self.pos, and
	explicitly pass the current index around

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import unicodehelper
+from rpython.rtyper.annlowlevel import llstr, hlunicode
 
 def is_whitespace(ch):
     return ch == ' ' or ch == '\t' or ch == '\r' or ch == '\n'
@@ -88,62 +89,54 @@
     def _raise(self, msg, *args):
         raise operationerrfmt(self.space.w_ValueError, msg, *args)
 
-    def decode_any(self):
-        self.pos = self.skip_whitespace(self.pos)
-        ch = self.peek()
+    def decode_any(self, i):
+        i = self.skip_whitespace(i)
+        ch = self.s[i]
         if ch == '"':
-            self.next()
-            return self.decode_string()
+            return self.decode_string(i+1)
         elif ch.isdigit() or ch == '-':
-            return self.decode_numeric()
+            return self.decode_numeric(i)
         elif ch == '[':
-            self.next()
-            return self.decode_array()
+            return self.decode_array(i+1)
         elif ch == '{':
-            self.next()
-            return self.decode_object()
+            return self.decode_object(i+1)
         elif ch == 'n':
-            self.next()
-            return self.decode_null()
+            return self.decode_null(i+1)
         elif ch == 't':
-            self.next()
-            return self.decode_true()
+            return self.decode_true(i+1)
         elif ch == 'f':
-            self.next()
-            return self.decode_false()
+            return self.decode_false(i+1)
         else:
             self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
                         ch, self.pos)
 
-    def decode_null(self):
-        N = len('ull')
-        if (self.pos+N <= len(self.s) and
-            self.next() == 'u' and
-            self.next() == 'l' and
-            self.next() == 'l'):
+    def decode_null(self, i):
+        if (self.s[i]   == 'u' and
+            self.s[i+1] == 'l' and
+            self.s[i+2] == 'l'):
+            self.pos = i+3
             return self.space.w_None
-        self._raise("Error when decoding null at char %d", self.pos)
+        self._raise("Error when decoding null at char %d", i)
 
-    def decode_true(self):
-        N = len('rue')
-        if (self.pos+N <= len(self.s) and
-            self.next() == 'r' and
-            self.next() == 'u' and
-            self.next() == 'e'):
+    def decode_true(self, i):
+        if (self.s[i]   == 'r' and
+            self.s[i+1] == 'u' and
+            self.s[i+2] == 'e'):
+            self.pos = i+3
             return self.space.w_True
-        self._raise("Error when decoding true at char %d", self.pos)
+        self._raise("Error when decoding true at char %d", i)
 
-    def decode_false(self):
-        N = len('alse')
-        if (self.pos+N <= len(self.s) and
-            self.next() == 'a' and
-            self.next() == 'l' and
-            self.next() == 's' and
-            self.next() == 'e'):
+    def decode_false(self, i):
+        if (self.s[i]   == 'a' and
+            self.s[i+1] == 'l' and
+            self.s[i+2] == 's' and
+            self.s[i+3] == 'e'):
+            self.pos = i+4
             return self.space.w_False
-        self._raise("Error when decoding false at char %d", self.pos)
+        self._raise("Error when decoding false at char %d", i)
 
-    def decode_numeric(self):
+    def decode_numeric(self, i):
+        self.pos = i
         intval = self.parse_integer()
         #
         is_float = False
@@ -201,22 +194,20 @@
         self.pos = i
         return intval, count
         
-    def decode_array(self):
+    def decode_array(self, i):
         w_list = self.space.newlist([])
-        start = self.pos
+        start = i
+        count = 0
         i = self.skip_whitespace(start)
         while i < len(self.s):
             ch = self.s[i]
             if ch == ']':
                 self.pos = i+1
                 return w_list
-            self.pos = i
-            w_item = self.decode_any()
+            w_item = self.decode_any(i)
             i = self.pos
             self.space.call_method(w_list, 'append', w_item)
             i = self.skip_whitespace(i)
-            if i == len(self.s):
-                break
             ch = self.s[i]
             i += 1
             if ch == ']':
@@ -230,35 +221,34 @@
         self._raise("Unterminated array starting at char %d", start)
 
 
-    def decode_object(self):
-        start = self.pos
+    def decode_object(self, i):
+        start = i
         w_dict = self.space.newdict()
-        while not self.eof():
-            ch = self.peek()
+        while i < len(self.s):
+            ch = self.s[i]
             if ch == '}':
-                self.next()
+                self.pos = i+1
                 return w_dict
             #
             # parse a key: value
             self.last_type = TYPE_UNKNOWN
-            w_name = self.decode_any()
+            w_name = self.decode_any(i)
             if self.last_type != TYPE_STRING:
                 self._raise("Key name must be string for object starting at char %d", start)
-            self.pos = self.skip_whitespace(self.pos)
-            if self.eof():
-                break
-            ch = self.next()
+            i = self.skip_whitespace(self.pos)
+            ch = self.s[i]
             if ch != ':':
-                self._raise("No ':' found at char %d", self.pos)
-            self.pos = self.skip_whitespace(self.pos)
+                self._raise("No ':' found at char %d", i)
+            i += 1
+            i = self.skip_whitespace(i)
             #
-            w_value = self.decode_any()
+            w_value = self.decode_any(i)
             self.space.setitem(w_dict, w_name, w_value)
-            self.pos = self.skip_whitespace(self.pos)
-            if self.eof():
-                break
-            ch = self.next()
+            i = self.skip_whitespace(self.pos)
+            ch = self.s[i]
+            i += 1
             if ch == '}':
+                self.pos = i
                 return w_dict
             elif ch == ',':
                 pass
@@ -267,9 +257,8 @@
                             ch, self.pos)
         self._raise("Unterminated object starting at char %d", start)
 
-    def decode_string(self):
-        start = self.pos
-        i = self.pos
+    def decode_string(self, i):
+        start = i
         bits = 0
         while i < len(self.s):
             # this loop is a fast path for strings which do not contain escape
@@ -348,11 +337,13 @@
 
 @unwrap_spec(s=str)
 def loads(space, s):
+    # the '\0' serves as a sentinel, so that we can avoid the bound check
+    s = s + '\0'
     decoder = JSONDecoder(space, s)
-    w_res = decoder.decode_any()
-    decoder.pos = decoder.skip_whitespace(decoder.pos)
-    if not decoder.eof():
-        start = decoder.pos
-        end = len(decoder.s)
+    w_res = decoder.decode_any(0)
+    i = decoder.skip_whitespace(decoder.pos)
+    if s[i] != '\0':
+        start = i
+        end = len(s) - 1
         raise operationerrfmt(space.w_ValueError, "Extra data: char %d - %d", start, end)
     return w_res

From noreply at buildbot.pypy.org  Sun Jun  9 11:41:58 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:41:58 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: apparently,
 checking for digits at last gives another small speedup
Message-ID: <20130609094158.EF8FB1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64835:fdc8c71594f3
Date: 2013-06-07 11:12 +0200
http://bitbucket.org/pypy/pypy/changeset/fdc8c71594f3/

Log:	apparently, checking for digits at last gives another small speedup

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -94,8 +94,6 @@
         ch = self.s[i]
         if ch == '"':
             return self.decode_string(i+1)
-        elif ch.isdigit() or ch == '-':
-            return self.decode_numeric(i)
         elif ch == '[':
             return self.decode_array(i+1)
         elif ch == '{':
@@ -106,6 +104,8 @@
             return self.decode_true(i+1)
         elif ch == 'f':
             return self.decode_false(i+1)
+        elif ch.isdigit() or ch == '-':
+            return self.decode_numeric(i)
         else:
             self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
                         ch, self.pos)

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:00 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:00 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: more drangons: RPython is not smart
 enough to remove bound checking everywhere we are interested in. Instead,
 we access directly to the underlying char array:
Message-ID: <20130609094200.2CDB61C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64836:0cf42778c9db
Date: 2013-06-07 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/0cf42778c9db/

Log:	more drangons: RPython is not smart enough to remove bound checking
	everywhere we are interested in. Instead, we access directly to the
	underlying char array:

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -46,14 +46,16 @@
     def __init__(self, space, s):
         self.space = space
         self.s = s
+        self.ll_chars = llstr(s).chars
+        self.length = len(self.s)
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
 
     def eof(self):
-        return self.pos == len(self.s)
+        return self.pos == self.length
 
     def peek(self):
-        return self.s[self.pos]
+        return self.ll_chars[self.pos]
 
     def peek_maybe(self):
         if self.eof():
@@ -77,8 +79,8 @@
         return self.s[start:end]
 
     def skip_whitespace(self, i):
-        while i < len(self.s):
-            ch = self.s[i]
+        while i < self.length:
+            ch = self.ll_chars[i]
             if is_whitespace(ch):
                 i+=1
             else:
@@ -91,7 +93,7 @@
 
     def decode_any(self, i):
         i = self.skip_whitespace(i)
-        ch = self.s[i]
+        ch = self.ll_chars[i]
         if ch == '"':
             return self.decode_string(i+1)
         elif ch == '[':
@@ -111,26 +113,26 @@
                         ch, self.pos)
 
     def decode_null(self, i):
-        if (self.s[i]   == 'u' and
-            self.s[i+1] == 'l' and
-            self.s[i+2] == 'l'):
+        if (self.ll_chars[i]   == 'u' and
+            self.ll_chars[i+1] == 'l' and
+            self.ll_chars[i+2] == 'l'):
             self.pos = i+3
             return self.space.w_None
         self._raise("Error when decoding null at char %d", i)
 
     def decode_true(self, i):
-        if (self.s[i]   == 'r' and
-            self.s[i+1] == 'u' and
-            self.s[i+2] == 'e'):
+        if (self.ll_chars[i]   == 'r' and
+            self.ll_chars[i+1] == 'u' and
+            self.ll_chars[i+2] == 'e'):
             self.pos = i+3
             return self.space.w_True
         self._raise("Error when decoding true at char %d", i)
 
     def decode_false(self, i):
-        if (self.s[i]   == 'a' and
-            self.s[i+1] == 'l' and
-            self.s[i+2] == 's' and
-            self.s[i+3] == 'e'):
+        if (self.ll_chars[i]   == 'a' and
+            self.ll_chars[i+1] == 'l' and
+            self.ll_chars[i+2] == 's' and
+            self.ll_chars[i+3] == 'e'):
             self.pos = i+4
             return self.space.w_False
         self._raise("Error when decoding false at char %d", i)
@@ -181,8 +183,8 @@
         intval = 0
         count = 0
         i = self.pos
-        while i < len(self.s):
-            ch = self.s[i]
+        while i < self.length:
+            ch = self.ll_chars[i]
             if ch.isdigit():
                 intval = intval*10 + ord(ch)-ord('0')
                 count += 1
@@ -199,8 +201,8 @@
         start = i
         count = 0
         i = self.skip_whitespace(start)
-        while i < len(self.s):
-            ch = self.s[i]
+        while i < self.length:
+            ch = self.ll_chars[i]
             if ch == ']':
                 self.pos = i+1
                 return w_list
@@ -208,7 +210,7 @@
             i = self.pos
             self.space.call_method(w_list, 'append', w_item)
             i = self.skip_whitespace(i)
-            ch = self.s[i]
+            ch = self.ll_chars[i]
             i += 1
             if ch == ']':
                 self.pos = i
@@ -224,8 +226,8 @@
     def decode_object(self, i):
         start = i
         w_dict = self.space.newdict()
-        while i < len(self.s):
-            ch = self.s[i]
+        while i < self.length:
+            ch = self.ll_chars[i]
             if ch == '}':
                 self.pos = i+1
                 return w_dict
@@ -236,7 +238,7 @@
             if self.last_type != TYPE_STRING:
                 self._raise("Key name must be string for object starting at char %d", start)
             i = self.skip_whitespace(self.pos)
-            ch = self.s[i]
+            ch = self.ll_chars[i]
             if ch != ':':
                 self._raise("No ':' found at char %d", i)
             i += 1
@@ -245,7 +247,7 @@
             w_value = self.decode_any(i)
             self.space.setitem(w_dict, w_name, w_value)
             i = self.skip_whitespace(self.pos)
-            ch = self.s[i]
+            ch = self.ll_chars[i]
             i += 1
             if ch == '}':
                 self.pos = i
@@ -260,10 +262,10 @@
     def decode_string(self, i):
         start = i
         bits = 0
-        while i < len(self.s):
+        while i < self.length:
             # this loop is a fast path for strings which do not contain escape
             # characters
-            ch = self.s[i]
+            ch = self.ll_chars[i]
             i += 1
             bits |= ord(ch)
             if ch == '"':

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:01 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:01 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: there is no need to check for the
 bound, as there is the sentinel at the end of the string anyway. Morever,
 avoid to keep a separate counter
Message-ID: <20130609094201.51C5E1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64837:d0a3ae2d5a74
Date: 2013-06-07 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/d0a3ae2d5a74/

Log:	there is no need to check for the bound, as there is the sentinel at
	the end of the string anyway. Morever, avoid to keep a separate
	counter

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -182,15 +182,15 @@
         "Parse a sequence of digits as a decimal number. No sign allowed"
         intval = 0
         count = 0
-        i = self.pos
-        while i < self.length:
+        start = i = self.pos
+        while True:
             ch = self.ll_chars[i]
             if ch.isdigit():
                 intval = intval*10 + ord(ch)-ord('0')
-                count += 1
                 i += 1
             else:
                 break
+        count = i - start
         if count == 0:
             self._raise("Expected digit at char %d", i)
         self.pos = i

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:02 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:02 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: wow, avoiding self.pos,
 peek_maybe() and friends gives a speedup of ~30%\!
Message-ID: <20130609094202.7C7F71C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64838:24a2398ade5b
Date: 2013-06-07 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/24a2398ade5b/

Log:	wow, avoiding self.pos, peek_maybe() and friends gives a speedup of
	~30%\!

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -138,8 +138,7 @@
         self._raise("Error when decoding false at char %d", i)
 
     def decode_numeric(self, i):
-        self.pos = i
-        intval = self.parse_integer()
+        i, intval = self.parse_integer(i)
         #
         is_float = False
         exp = 0
@@ -147,19 +146,18 @@
         frccount = 0
         #
         # check for the optional fractional part
-        ch = self.peek_maybe()
+        ch = self.ll_chars[i]
         if ch == '.':
             is_float = True
-            self.next()
-            frcval, frccount = self.parse_digits()
+            i, frcval, frccount = self.parse_digits(i+1)
             frcval = neg_pow_10(frcval, frccount)
-            ch = self.peek_maybe()
+            ch = self.ll_chars[i]
         # check for the optional exponent part
         if ch == 'E' or ch == 'e':
             is_float = True
-            self.next()
-            exp = self.parse_integer()
+            i, exp = self.parse_integer(i+1)
         #
+        self.pos = i
         if is_float:
             # build the float
             floatval = intval + frcval
@@ -169,20 +167,19 @@
         else:
             return self.space.wrap(intval)
 
-    def parse_integer(self):
+    def parse_integer(self, i):
         "Parse a decimal number with an optional minus sign"
         sign = 1
-        if self.peek_maybe() == '-':
+        if self.ll_chars[i] == '-':
             sign = -1
-            self.next()
-        intval, _ = self.parse_digits()
-        return sign * intval
+            i += 1
+        i, intval, _ = self.parse_digits(i)
+        return i, sign * intval
 
-    def parse_digits(self):
+    def parse_digits(self, i):
         "Parse a sequence of digits as a decimal number. No sign allowed"
         intval = 0
-        count = 0
-        start = i = self.pos
+        start = i 
         while True:
             ch = self.ll_chars[i]
             if ch.isdigit():
@@ -193,8 +190,7 @@
         count = i - start
         if count == 0:
             self._raise("Expected digit at char %d", i)
-        self.pos = i
-        return intval, count
+        return i, intval, count
         
     def decode_array(self, i):
         w_list = self.space.newlist([])

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:03 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:03 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: kill the last references to peek, next,
 eof and peek_maybe, they are no longer used
Message-ID: <20130609094203.B5A7F1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64839:022385d2f170
Date: 2013-06-07 18:19 +0200
http://bitbucket.org/pypy/pypy/changeset/022385d2f170/

Log:	kill the last references to peek, next, eof and peek_maybe, they are
	no longer used

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -51,28 +51,6 @@
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
 
-    def eof(self):
-        return self.pos == self.length
-
-    def peek(self):
-        return self.ll_chars[self.pos]
-
-    def peek_maybe(self):
-        if self.eof():
-            return '\0'
-        else:
-            return self.peek()
-
-    def next(self):
-        ch = self.peek()
-        self.pos += 1
-        return ch
-
-    def unget(self):
-        i2 = self.pos - 1
-        assert i2 > 0 # so that we can use self.pos as slice start
-        self.pos = i2
-
     def getslice(self, start, end):
         assert start > 0
         assert end > 0
@@ -287,23 +265,27 @@
     def decode_string_escaped(self, start, content_so_far):
         builder = StringBuilder(len(content_so_far)*2) # just an estimate
         builder.append(content_so_far)
-        while not self.eof():
-            ch = self.next()
+        i = self.pos
+        while True:
+            ch = self.ll_chars[i]
+            i += 1
             if ch == '"':
                 content_utf8 = builder.build()
                 content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
                 self.last_type = TYPE_STRING
+                self.pos = i
                 return self.space.wrap(content_unicode)
             elif ch == '\\':
-                self.decode_escape_sequence(builder)
+                i = self.decode_escape_sequence(i, builder)
+            elif ch == '\0':
+                self._raise("Unterminated string starting at char %d", start)
             else:
                 builder.append_multiple_char(ch, 1) # we should implement append_char
-        #
-        self._raise("Unterminated string starting at char %d", start)
 
-    def decode_escape_sequence(self, builder):
+    def decode_escape_sequence(self, i, builder):
+        ch = self.ll_chars[i]
+        i += 1
         put = builder.append_multiple_char
-        ch = self.next()
         if ch == '\\':  put('\\', 1)
         elif ch == '"': put('"' , 1)
         elif ch == '/': put('/' , 1)
@@ -313,24 +295,26 @@
         elif ch == 'r': put('\r', 1)
         elif ch == 't': put('\t', 1)
         elif ch == 'u':
-            return self.decode_escape_sequence_unicode(builder)
+            return self.decode_escape_sequence_unicode(i, builder)
         else:
             self._raise("Invalid \\escape: %s (char %d)", ch, self.pos-1)
+        return i
 
-    def decode_escape_sequence_unicode(self, builder):
+    def decode_escape_sequence_unicode(self, i, builder):
         # at this point we are just after the 'u' of the \u1234 sequence.
-        hexdigits = self.getslice(self.pos, self.pos+4)
-        self.pos += 4
+        start = i
+        i += 4
+        hexdigits = self.getslice(start, i)
         try:
             uchr = unichr(int(hexdigits, 16))
         except ValueError:
-            self._raise("Invalid \uXXXX escape (char %d)", self.pos-1)
+            self._raise("Invalid \uXXXX escape (char %d)", i-1)
             return # help the annotator to know that we'll never go beyond
                    # this point
         #
         utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
         builder.append(utf8_ch)
-
+        return i
 
 
 @unwrap_spec(s=str)

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:04 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:04 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: remove one more bound check (safely!
	:))
Message-ID: <20130609094204.E44EC1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64840:b31803bdd48c
Date: 2013-06-07 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/b31803bdd48c/

Log:	remove one more bound check (safely! :))

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -46,8 +46,10 @@
     def __init__(self, space, s):
         self.space = space
         self.s = s
-        self.ll_chars = llstr(s).chars
-        self.length = len(self.s)
+        # we put a sentinel at the end so that we never have to check for the
+        # "end of string" condition
+        self.ll_chars = llstr(s+'\0').chars
+        self.length = len(s)
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
 
@@ -57,7 +59,7 @@
         return self.s[start:end]
 
     def skip_whitespace(self, i):
-        while i < self.length:
+        while True:
             ch = self.ll_chars[i]
             if is_whitespace(ch):
                 i+=1
@@ -319,12 +321,10 @@
 
 @unwrap_spec(s=str)
 def loads(space, s):
-    # the '\0' serves as a sentinel, so that we can avoid the bound check
-    s = s + '\0'
     decoder = JSONDecoder(space, s)
     w_res = decoder.decode_any(0)
     i = decoder.skip_whitespace(decoder.pos)
-    if s[i] != '\0':
+    if i < len(s):
         start = i
         end = len(s) - 1
         raise operationerrfmt(space.w_ValueError, "Extra data: char %d - %d", start, end)

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:06 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:06 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: kill one more end-of-string check
Message-ID: <20130609094206.297AC1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64841:2a2c1967e1c1
Date: 2013-06-07 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/2a2c1967e1c1/

Log:	kill one more end-of-string check

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -177,7 +177,7 @@
         start = i
         count = 0
         i = self.skip_whitespace(start)
-        while i < self.length:
+        while True:
             ch = self.ll_chars[i]
             if ch == ']':
                 self.pos = i+1
@@ -193,10 +193,11 @@
                 return w_list
             elif ch == ',':
                 pass
+            elif ch == '\0':
+                self._raise("Unterminated array starting at char %d", start)
             else:
                 self._raise("Unexpected '%s' when decoding array (char %d)",
                             ch, self.pos)
-        self._raise("Unterminated array starting at char %d", start)
 
 
     def decode_object(self, i):

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:07 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:07 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: kill yet another more end-of-string
	check
Message-ID: <20130609094207.4DDE21C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64842:27044fc0ceb8
Date: 2013-06-07 18:41 +0200
http://bitbucket.org/pypy/pypy/changeset/27044fc0ceb8/

Log:	kill yet another more end-of-string check

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -203,7 +203,7 @@
     def decode_object(self, i):
         start = i
         w_dict = self.space.newdict()
-        while i < self.length:
+        while True:
             ch = self.ll_chars[i]
             if ch == '}':
                 self.pos = i+1
@@ -231,10 +231,12 @@
                 return w_dict
             elif ch == ',':
                 pass
+            elif ch == '\0':
+                self._raise("Unterminated object starting at char %d", start)
             else:
                 self._raise("Unexpected '%s' when decoding object (char %d)",
                             ch, self.pos)
-        self._raise("Unterminated object starting at char %d", start)
+
 
     def decode_string(self, i):
         start = i

From noreply at buildbot.pypy.org  Sun Jun  9 11:42:08 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun,  9 Jun 2013 11:42:08 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: remove the last end-of-string check
Message-ID: <20130609094208.7671C1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64843:2a7d18573294
Date: 2013-06-07 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/2a7d18573294/

Log:	remove the last end-of-string check

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -49,7 +49,6 @@
         # we put a sentinel at the end so that we never have to check for the
         # "end of string" condition
         self.ll_chars = llstr(s+'\0').chars
-        self.length = len(s)
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
 
@@ -241,7 +240,7 @@
     def decode_string(self, i):
         start = i
         bits = 0
-        while i < self.length:
+        while True:
             # this loop is a fast path for strings which do not contain escape
             # characters
             ch = self.ll_chars[i]
@@ -264,7 +263,8 @@
                 content_so_far = self.getslice(start, i-1)
                 self.pos = i-1
                 return self.decode_string_escaped(start, content_so_far)
-        self._raise("Unterminated string starting at char %d", start)
+            elif ch == '\0':
+                self._raise("Unterminated string starting at char %d", start)
 
 
     def decode_string_escaped(self, start, content_so_far):

From noreply at buildbot.pypy.org  Sun Jun  9 16:07:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Jun 2013 16:07:09 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress, maybe?  messy :-(
Message-ID: <20130609140709.B35A01C07E1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r84:d2ad521c9ac9
Date: 2013-06-09 16:07 +0200
http://bitbucket.org/pypy/stmgc/changeset/d2ad521c9ac9/

Log:	in-progress, maybe? messy :-(

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -66,6 +66,13 @@
   gcptr P = G;
   revision_t v;
 
+  if (UNLIKELY(d->public_descriptor->stolen_objects.size > 0))
+    {
+      spinlock_acquire(d->public_descriptor->collection_lock, 'N');
+      stm_normalize_stolen_objects(d->public_descriptor);
+      spinlock_release(d->public_descriptor->collection_lock);
+    }
+
   if (P->h_tid & GCFLAG_PUBLIC)
     {
       /* follow the chained list of h_revision's as long as they are
@@ -144,7 +151,7 @@
   if (foreign_pd == d->public_descriptor)
     {
       /* same thread */
-      P = (gcptr)v;
+      P = (gcptr)(v - 2);
       assert(!(P->h_tid & GCFLAG_PUBLIC));
       if (P->h_revision == stm_private_rev_num)
         {
@@ -306,7 +313,14 @@
       memcpy(B + 1, P + 1, size - sizeof(*B));
     }
   assert(B->h_tid & GCFLAG_BACKUP_COPY);
-  gcptrlist_insert2(&d->public_descriptor->active_backup_copies, P, B);
+
+  gcptrlist_locked_insert2(&d->public_descriptor->active_backup_copies, P, B,
+                           &d->public_descriptor->collection_lock);
+
+  smp_wmb();   /* guarantees that stm_steal_stub() will see the list
+                  up to the (P, B) pair in case it goes the path
+                  h_revision == *foreign_pd->private_revision_ref */
+
   P->h_revision = stm_private_rev_num;
   return P;
 }
@@ -328,7 +342,6 @@
  not_found:;
   gcptr L = stmgc_duplicate(R);
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
-  assert(!(L->h_tid & GCFLAG_STOLEN));
   assert(!(L->h_tid & GCFLAG_STUB));
   L->h_tid &= ~(GCFLAG_OLD               |
                 GCFLAG_VISITED           |
@@ -367,16 +380,19 @@
   return W;
 }
 
-gcptr stm_get_backup_copy(gcptr P)
+gcptr stm_get_backup_copy(long index)
 {
-  assert(P->h_revision == stm_private_rev_num);
+  struct tx_public_descriptor *pd = thread_descriptor->public_descriptor;
+  if (index < gcptrlist_size(&pd->active_backup_copies))
+    return pd->active_backup_copies.items[index];
+  return NULL;
+}
 
+gcptr stm_get_stolen_obj(long index)
+{
   struct tx_public_descriptor *pd = thread_descriptor->public_descriptor;
-  long i, size = pd->active_backup_copies.size;
-  gcptr *items = pd->active_backup_copies.items;
-  for (i = 0; i < size; i += 2)
-    if (items[i] == P)
-      return items[i + 1];
+  if (index < gcptrlist_size(&pd->stolen_objects))
+    return pd->stolen_objects.items[index];
   return NULL;
 }
 
@@ -549,7 +565,9 @@
 
   gcptrlist_clear(&d->list_of_read_objects);
   gcptrlist_clear(&d->public_descriptor->active_backup_copies);
-  abort();//stmgc_abort_transaction(d);
+  abort();
+  d->public_descriptor->stolen_objects;//XXX clean up
+  //stmgc_abort_transaction(d);
 
   fprintf(stderr,
           "\n"
@@ -612,7 +630,6 @@
     d->start_real_time.tv_nsec = -1;
   }
   assert(d->list_of_read_objects.size == 0);
-  assert(d->public_descriptor->active_backup_copies.size == 0);
   assert(!g2l_any_entry(&d->public_to_private));
 
   d->count_reads = 1;
@@ -724,7 +741,6 @@
       assert(!(L->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
       assert(!(L->h_tid & GCFLAG_PREBUILT_ORIGINAL));
       assert(!(L->h_tid & GCFLAG_NURSERY_MOVED));
-      assert(!(L->h_tid & GCFLAG_STOLEN));
       assert(L->h_revision != localrev);   /* modified by AcquireLocks() */
 
 #ifdef DUMP_EXTRA
@@ -751,7 +767,6 @@
       assert(R->h_tid & GCFLAG_PUBLIC);
       assert(R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
       assert(!(R->h_tid & GCFLAG_NURSERY_MOVED));
-      assert(!(R->h_tid & GCFLAG_STOLEN));
       assert(R->h_revision != localrev);
 
 #ifdef DUMP_EXTRA
@@ -801,19 +816,21 @@
 void TurnPrivateWithBackupToProtected(struct tx_descriptor *d,
                                       revision_t cur_time)
 {
-  long i, size = d->public_descriptor->active_backup_copies.size;
-  gcptr *items = d->public_descriptor->active_backup_copies.items;
+  struct tx_public_descriptor *pd = d->public_descriptor;
+  long i, size = pd->active_backup_copies.size;
+  gcptr *items = pd->active_backup_copies.items;
 
   for (i = 0; i < size; i += 2)
     {
       gcptr P = items[i];
       gcptr B = items[i + 1];
+      assert(B->h_tid & GCFLAG_BACKUP_COPY);
+      assert(!(B->h_tid & GCFLAG_PUBLIC));
       assert(P->h_revision == stm_private_rev_num);
-      assert(B->h_tid & GCFLAG_BACKUP_COPY);
       B->h_revision = cur_time;
       P->h_revision = (revision_t)B;
     };
-  gcptrlist_clear(&d->public_descriptor->active_backup_copies);
+  gcptrlist_clear(&pd->active_backup_copies);
 }
 
 void CommitTransaction(void)
@@ -823,6 +840,9 @@
   assert(d->active >= 1);
 
   spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
+  if (d->public_descriptor->stolen_objects.size)
+    stm_normalize_stolen_objects(d->public_descriptor);
+
   AcquireLocks(d);
 
   if (is_inevitable(d))
@@ -879,7 +899,7 @@
   assert(newrev & 1);
   ACCESS_ONCE(stm_private_rev_num) = newrev;
   fprintf(stderr, "%p: stm_local_revision = %ld\n", d, (long)newrev);
-  assert(d->private_revision_ref = &stm_private_rev_num);
+  assert(d->public_descriptor->private_revision_ref = &stm_private_rev_num);
 
   UpdateChainHeads(d, cur_time, localrev);
 
@@ -1097,7 +1117,7 @@
       assert(d->my_lock & 1);
       assert(d->my_lock >= LOCKED);
       stm_private_rev_num = -1;
-      d->private_revision_ref = &stm_private_rev_num;
+      pd->private_revision_ref = &stm_private_rev_num;
       d->max_aborts = -1;
       thread_descriptor = d;
 
@@ -1113,12 +1133,15 @@
 
 void DescriptorDone(void)
 {
+    static revision_t no_private_revision = 8;
     revision_t i;
     struct tx_descriptor *d = thread_descriptor;
     assert(d != NULL);
     assert(d->active == 0);
 
-    d->public_descriptor->collection_lock = 0;    /* unlock */
+    spinlock_acquire(d->public_descriptor->collection_lock, 'D');  /*done*/
+    d->public_descriptor->private_revision_ref = &no_private_revision;
+    spinlock_release(d->public_descriptor->collection_lock);
 
     spinlock_acquire(descriptor_array_lock, 1);
     i = d->public_descriptor_index;
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -31,6 +31,7 @@
  * GCFLAG_PUBLIC is set on public objects.
  *
  * GCFLAG_BACKUP_COPY means the object is a (protected) backup copy.
+ * For debugging.
  *
  * GCFLAG_PUBLIC_TO_PRIVATE is added to a *public* object that has got a
  * *private* copy.  It is sticky, reset only at the next major collection.
@@ -47,9 +48,6 @@
  *
  * GCFLAG_NURSERY_MOVED is used temporarily during minor collections.
  *
- * GCFLAG_STOLEN is set of protected objects after we notice that they
- * have been stolen.
- *
  * GCFLAG_STUB is set for debugging on stub objects made by stealing or
  * by major collections.  'p_stub->h_revision' might be a value
  * that is == 2 (mod 4): in this case they point to a protected/private
@@ -59,12 +57,11 @@
 #define GCFLAG_VISITED           (STM_FIRST_GCFLAG << 1)
 #define GCFLAG_PUBLIC            (STM_FIRST_GCFLAG << 2)
 #define GCFLAG_PREBUILT_ORIGINAL (STM_FIRST_GCFLAG << 3)
-#define GCFLAG_BACKUP_COPY       (STM_FIRST_GCFLAG << 4)
-#define GCFLAG_PUBLIC_TO_PRIVATE (STM_FIRST_GCFLAG << 5)
-#define GCFLAG_WRITE_BARRIER     (STM_FIRST_GCFLAG << 6)
-#define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 7)
-#define GCFLAG_STOLEN            (STM_FIRST_GCFLAG << 8)
-#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 9)   /* debugging */
+#define GCFLAG_PUBLIC_TO_PRIVATE (STM_FIRST_GCFLAG << 4)
+#define GCFLAG_WRITE_BARRIER     (STM_FIRST_GCFLAG << 5)
+#define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 6)
+#define GCFLAG_BACKUP_COPY       (STM_FIRST_GCFLAG << 7)   /* debugging */
+#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 8)   /* debugging */
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
@@ -108,8 +105,9 @@
   revision_t collection_lock;
   struct stub_block_s *stub_blocks;
   gcptr stub_free_list;
-  struct GcPtrList stolen_objects;
-  struct GcPtrList active_backup_copies;
+  struct GcPtrList active_backup_copies;  /* (P,B) where P=private, B=backup */
+  struct GcPtrList stolen_objects;      /* (P,Q) where P=priv/prot, Q=public */
+  revision_t *private_revision_ref;
   revision_t free_list_next;
   /* xxx gcpage data here */
 };
@@ -142,7 +140,6 @@
   char *longest_abort_info;
   long long longest_abort_info_time;
   struct FXCache recent_reads_cache;
-  revision_t *private_revision_ref;
 };
 
 extern __thread struct tx_descriptor *thread_descriptor;
@@ -165,7 +162,8 @@
 gcptr stm_RepeatReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
 gcptr _stm_nonrecord_barrier(gcptr, int *);
-gcptr stm_get_backup_copy(gcptr);
+gcptr stm_get_backup_copy(long);  /* debugging */
+gcptr stm_get_stolen_obj(long);  /* debugging */
 gcptr stm_get_read_obj(long);  /* debugging */
 gcptr stmgc_duplicate(gcptr);
 
diff --git a/c4/lists.c b/c4/lists.c
--- a/c4/lists.c
+++ b/c4/lists.c
@@ -171,6 +171,23 @@
   gcptrlist->size = i + 2;
 }
 
+void gcptrlist_locked_insert2(struct GcPtrList *gcptrlist, gcptr newitem1,
+                              gcptr newitem2, revision_t *lock)
+{
+  gcptr *items;
+  long i = gcptrlist->size;
+  if (UNLIKELY((gcptrlist->alloc - i) < 2)) 
+    {
+      spinlock_acquire(*lock, 'I');
+      _gcptrlist_grow(gcptrlist);
+      spinlock_release(*lock);
+    }
+  items = gcptrlist->items;
+  items[i+0] = newitem1;
+  items[i+1] = newitem2;
+  gcptrlist->size = i + 2;
+}
+
 void gcptrlist_insert3(struct GcPtrList *gcptrlist, gcptr newitem1,
                        gcptr newitem2, gcptr newitem3)
 {
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -164,6 +164,9 @@
 void gcptrlist_merge(struct GcPtrList *, struct GcPtrList *gcptrlist_source);
 void gcptrlist_move(struct GcPtrList *, struct GcPtrList *gcptrlist_source);
 
+void gcptrlist_locked_insert2(struct GcPtrList *gcptrlist, gcptr newitem1,
+                              gcptr newitem2, revision_t *lock);
+
 /************************************************************/
 
 /* The fxcache_xx functions implement a fixed-size set of gcptr's.
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -53,9 +53,29 @@
     if ((v & 3) != 2)
         goto done;     /* un-stubbed while we waited for the lock */
 
-    gcptr L = (gcptr)(v - 2);
-    gcptr Q = stmgc_duplicate(L);
+    gcptr Q, L = (gcptr)(v - 2);
+    revision_t w = ACCESS_ONCE(L->h_revision);
+
+    if (w == *foreign_pd->private_revision_ref) {
+        /* The stub points to a private object L.  Because it cannot point
+           to "really private" objects, it must mean that L used to be
+           a protected object, and it has an attached backed copy.
+           XXX find a way to optimize this search, maybe */
+        long i;
+        gcptr *items = foreign_pd->active_backup_copies.items;
+        /* we must find L as the first item of a pair in the list.  We
+           cannot rely on how big the list is here, but we know that
+           it will not be resized while we hold collection_lock. */
+        for (i = 0; items[i] != L; i += 2)
+            ;
+        L = items[i + 1];
+        assert(L->h_tid & GCFLAG_BACKUP_COPY);
+    }
+    /* duplicate L */
+    Q = stmgc_duplicate(L);  XXX RACE
+    Q->h_tid &= ~GCFLAG_BACKUP_COPY;
     Q->h_tid |= GCFLAG_PUBLIC;
+    gcptrlist_insert2(&foreign_pd->stolen_objects, L, Q);
 
     smp_wmb();
 
@@ -64,3 +84,16 @@
  done:
     spinlock_release(foreign_pd->collection_lock);
 }
+
+void stm_normalize_stolen_objects(struct tx_public_descriptor *pd)
+{
+    long i, size = pd->stolen_objects.size;
+    gcptr *items = pd->stolen_objects.items;
+    for (i = 0; i < size; i += 2) {
+        gcptr L = items[i];
+        gcptr Q = items[i + 1];
+        if (L->h_revision == stm_private_rev_num) {
+            
+        }
+    }
+}
diff --git a/c4/steal.h b/c4/steal.h
--- a/c4/steal.h
+++ b/c4/steal.h
@@ -9,6 +9,8 @@
 
 gcptr stm_stub_malloc(struct tx_public_descriptor *);
 void stm_steal_stub(gcptr);
+gcptr stm_get_stolen_obj(long index);   /* debugging */
+void stm_normalize_stolen_objects(struct tx_public_descriptor *);
 
 
 #endif
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -68,7 +68,8 @@
     void stm_start_sharedlock(void);
     void stm_stop_sharedlock(void);
     void AbortTransaction(int);
-    gcptr stm_get_backup_copy(gcptr);
+    gcptr stm_get_backup_copy(long index);
+    gcptr stm_get_stolen_obj(long index);
     gcptr stm_get_read_obj(long index);
     void *STUB_THREAD(gcptr);
 
@@ -103,7 +104,6 @@
     #define GCFLAG_PUBLIC_TO_PRIVATE ...
     #define GCFLAG_WRITE_BARRIER     ...
     #define GCFLAG_NURSERY_MOVED     ...
-    #define GCFLAG_STOLEN            ...
     #define GCFLAG_STUB              ...
     #define ABRT_MANUAL              ...
     //typedef struct { ...; } page_header_t;
@@ -552,4 +552,23 @@
         index += 1
     return result
 
+def _list2dict(getter):
+    result = {}
+    index = 0
+    while 1:
+        p = getter(index)
+        if p == ffi.NULL:
+            break
+        q = getter(index + 1)
+        assert q != ffi.NULL
+        result[p] = q
+        index += 2
+    return result
+
+def backup_copies():
+    return _list2dict(lib.stm_get_backup_copy)
+
+def stolen_objs():
+    return _list2dict(lib.stm_get_stolen_obj)
+
 stub_thread = lib.STUB_THREAD
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -60,7 +60,7 @@
     org_r = p.h_revision
     lib.setlong(p, 0, 927122)
     assert p.h_revision == lib.get_private_rev_num()
-    pback = lib.stm_get_backup_copy(p)
+    pback = backup_copies()[p]
     assert pback and pback != p
     assert pback.h_revision == org_r
     assert pback.h_tid == p.h_tid | GCFLAG_BACKUP_COPY
@@ -75,7 +75,7 @@
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
     lib.setlong(p, 0, 927122)
-    pback = lib.stm_get_backup_copy(p)
+    pback = backup_copies()[p]
     assert pback != p
     assert p.h_revision == lib.get_private_rev_num()
     lib.stm_commit_transaction()
@@ -90,7 +90,7 @@
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
     lib.setlong(p, 0, 927122)
-    pback = lib.stm_get_backup_copy(p)
+    pback = backup_copies()[p]
     assert pback != p
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
@@ -100,7 +100,7 @@
     assert lib.rawgetlong(pback, 0) == 78927812    # but should not be used
     lib.setlong(p, 0, 43891)
     assert p.h_revision == lib.get_private_rev_num()
-    assert pback == lib.stm_get_backup_copy(p)
+    assert pback == backup_copies()[p]
     assert lib.rawgetlong(p, 0) == 43891
     assert lib.rawgetlong(pback, 0) == 927122
 
@@ -239,14 +239,16 @@
         lib.stm_begin_inevitable_transaction()
         assert classify(p) == "public"
         assert classify(p1) == "protected"
-        plist.append(p1)
-        # now p's most recent revision is protected
+        plist.append(p1)     # now p's most recent revision is protected
         assert classify(ffi.cast("gcptr", p.h_revision)) == "stub"
         r.set(2)
         r.wait(3)
-        assert lib.list_stolen_objects() == plist[-2:]
-        p2 = lib.stm_read_barrier(p1)
-        assert p2 == plist[-1]
+        d = stolen_objs()
+        assert len(d) == 1
+        assert d.keys() == [p1]
+        [p2] = d.values()
+        assert lib.stm_read_barrier(p) == p2
+        assert lib.stm_read_barrier(p1) == p2
     def f2(r):
         r.wait(2)
         p2 = lib.stm_read_barrier(p)    # steals
@@ -255,6 +257,7 @@
         assert p.h_revision == int(ffi.cast("revision_t", p2))
         assert p2 == lib.stm_read_barrier(p)
         assert p2 not in plist
+        assert classify(p2) == "public"
         plist.append(p2)
         r.set(3)
     run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Sun Jun  9 18:25:56 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun,  9 Jun 2013 18:25:56 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: added two missed optimizations
Message-ID: <20130609162556.C1DD11C094A@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r4974:6460bc5634c4
Date: 2013-06-09 09:24 -0700
http://bitbucket.org/pypy/extradoc/changeset/6460bc5634c4/

Log:	added two missed optimizations

diff --git a/planning/jit.txt b/planning/jit.txt
--- a/planning/jit.txt
+++ b/planning/jit.txt
@@ -77,6 +77,11 @@
 - calling string equality does not automatically promote the argument to
   a constant.
 
+- i0 = int_add_ovf(9223372036854775807, 1)
+  guard_overflow()
+
+- p0 = call_pure(ConstClass(something), ConstPtr(2))
+  guard_exception(SomeException)
 
 PYTHON EXAMPLES
 ---------------

From noreply at buildbot.pypy.org  Sun Jun  9 19:02:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Jun 2013 19:02:14 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Update the document with a simpler
 version which is a better starting point.
Message-ID: <20130609170214.E49721C094A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85:6187b6951c91
Date: 2013-06-09 19:02 +0200
http://bitbucket.org/pypy/stmgc/changeset/6187b6951c91/

Log:	Update the document with a simpler version which is a better
	starting point.

diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt
--- a/c4/doc-objects.txt
+++ b/c4/doc-objects.txt
@@ -27,17 +27,17 @@
 
   Private freshly created
              \                Private, with backup
-              \                ^           |  ^
-               \              /     commit |  |
-         commit \     modify /             |  |
-                 \          /              |  | modify
-                  V        /               |  |
-           Protected, no backup            V  |
-                ^    ^              Protected, with backup
-               /     |       gc       |
-       commit /      `----------------'
-             /
-            /
+              \                ^  /
+        commit \              /  / commit
+                \     modify /  /
+                 \          /  /
+                  V        /  V
+                   Protected
+                  ^
+                 /
+         commit /
+               /
+              /
     Private copy of
     a public obj
 
@@ -48,32 +48,27 @@
 
 Private objects:
 - freshly created                                         PRN
-- converted from a protected obj                          PRN
+- converted from a protected obj                     ptr to backup
 - private copy of a public obj                            PRN
 
 Protected objects:
 - converted from fresh private obj                     (old PRN)
-- converted from a private obj with backup           ptr to backup
-- converted from a private obj from public                GT
+- converted from a private obj                            GT
+
+Backup copy:
 - backup copy of a private obj                    original h_revision
-- backup copy still attached to a protected               GT
-- original obj after GC killed the backup                 GT
 
 Public objects:
 - prebuilt object, never modified                          1
 - other public object, never modified                     GT
-- outdated, has a protected copy              HANDLE to prot/priv copy
-- outdated, target stolen              ptr to a more recent public copy
+- outdated                             ptr to a more recent public copy
 
-Public stubs:
-- from stealing: like outdated public objects
-- from major GC: like outdated public objects with target stolen
+Public stubs (have also a ref to one thread):
+- from stealing                            ptr (maybe to priv/prot) | 2
 
 
 PRN = Private revision number (negative odd number)
 GT = Global time (positive odd number)
-HANDLE = Reference to a prot/priv copy and its thread
-         (positive even number, such that: handle % 4 == 2)
 
 
 
@@ -83,54 +78,67 @@
 - the PRN (private revision number): odd, negative, changes for every
   transaction that commits
 
-- list active_backup_copies = [(private, backup copy)]
+- list private_from_protected = [private obj converted from protected]
 
 - dict public_to_private = {public obj: private copy}
 
 - list read_set containing the objects in the read set, with possibly
   some duplicates (but hopefully not too many)
 
-- list stolen_objects = [(priv/prot object, public copy)]
+- collection_lock: a thread-local lock that is acquired to change
+  the status of private/protected objects
 
 
-Kind of object copy            distinguishing feature
+
+Kind of object copy          distinguishing feature
 -------------------------------------------------------------------
 
-Any private object             h_revision == PRN
-Private with a backup          in active_backup_copies
-Backup copy                    GCFLAG_BACKUP_COPY
-Any public object              GCFLAG_PUBLIC
-Any protected object           h_revision != PRN && !GCFLAG_PUBLIC
-Stubs                          GCFLAG_STUB
+Any private object           h_revision == PRN or GCFLAG_PRIVATE_FROM_PROTECTED
+Private with a backup        GCFLAG_PRIVATE_FROM_PROTECTED
+Backup copy                  GCFLAG_BACKUP_COPY (flag for debugging)
+Any public object            GCFLAG_PUBLIC
+Stubs                        GCFLAG_STUB (flag for debugging)
 
 A public object that might \
-be key in public_to_private    has additionally GCFLAG_PUBLIC_TO_PRIVATE
+be key in public_to_private  has additionally GCFLAG_PUBLIC_TO_PRIVATE
 
 
 
 Read barrier
 -----------------------------------------
 
-Inline check: if P in read_barrier_cache, we don't call the slow path.
+Inline check: if h_revision == PRN or if P in read_barrier_cache,
+              we don't call the slow path.
 Slow path:
 
-    if h_revision == PRN, just add P to read_barrier_cache and return
+    if GCFLAG_PRIVATE_FROM_PROTECTED:
+
+        check P->h_revision->h_revision: if a pointer, then it means
+        the backup copy has been stolen into a public object and then
+        modified by some other thread.  Abort.
+
+        add P to 'read_barrier_cache' and return
 
     if GCFLAG_PUBLIC:
 
         follow the chained list of h_revision's as long as they are
         regular pointers
 
-        if it ends with an odd revision number, check that it's older
-        than start_time; extend the start timestamp if not
+        if it ends with h_revision % 4 == 2:
+            then we're in a stub
 
-        if it ends with a handle (L, Thread):
-
-            if Thread is the current thread: set P = L
+            if Thread is the current thread: follow P = h_revision - 2
 
             else: do stealing and restart the read barrier
 
-        if we land on a P in read_barrier_cache: return
+        if we land on a P in read_barrier_cache:
+            return P
+
+        if P has GCFLAG_PUBLIC_TO_PRIVATE and is in 'public_to_private':
+            return the private object
+
+        if it ends with an odd revision number, check that it's older
+        than start_time; extend the start timestamp if not
 
     add P to 'read_set'
 
@@ -143,29 +151,26 @@
 private objects which might be a majority, vs. making the inline check
 larger).
 
-Handles are stored for example in a global list, and the actual handle
-encodes an index in the list.  Every entry in the list is a pointer to a
-prot/priv object --- excepted once every N positions, where it is a
-thread descriptor giving the thread to which all N-1 following pointers
-belong.  The pair (L, Thread) is thus `(list[H], list[H rounded down to
-a multiple of N])`.
+Stub objects are public, always outdated (with h_revision a pointer) and
+contain only a header; additionally they have a thread descriptor that
+tells to which thread the h_revision object is a protected/private
+object of.
 
-Stealing of an object copy L is done with the "collection lock" of
-the target Thread.  The target would also acquire its own lock in
-when doing some operations, like a minor collection, which can't
-occur in parallel with stealing.
+Stealing of an object copy L is done with the "collection lock" of the
+target Thread.  The target would also acquire its own lock in when doing
+some operations, like a minor collection or a write barrier on a
+protected object, which can't occur in parallel with stealing.
 
 Once we have the lock, stealing is:
 
     if the situation changed while we were waiting for the lock, return
 
-    if L has got a backup copy, turn it public;
-    else L must be protected, and we make a public copy of it
+    if L has GCFLAG_PRIVATE_FROM_PROTECTED:
+        set L = L->h_revision (the backup copy)
 
-    update the original P->h_revision to point directly to the new
-    public copy
+    change L from protected to public, i.e. add GCFLAG_PUBLIC
 
-    add (P, new public copy) to stolen_objects
+    update the original P->h_revision to point directly to L
 
 
 
@@ -174,18 +179,15 @@
 
 The write barrier works for both STM purposes and for GC purposes.
 
-Inline check: if h_revision == PRN && !GCFLAG_WRITE_BARRIER, we're done.
+Inline check: if h_revision == PRN or GCFLAG_PRIVATE_FROM_PROTECTED, we're done.
 Slow path:
 
     R = read_barrier(P)     # always do a full read_barrier first
 
-    if h_revision == PRN:
+    if h_revision == PRN or GCFLAG_PRIVATE_FROM_PROTECTED:
+        return R
 
-        GC only: remove GCFLAG_WRITE_BARRIER, add R to the GC list of
-        modified old objects to trace at the next minor collection,
-        and return R
-
-    elif GCFLAG_PUBLIC:
+    if GCFLAG_PUBLIC:
 
         add the flag GCFLAG_PUBLIC_TO_PRIVATE to R, if needed
 
@@ -193,19 +195,50 @@
 
         add {R: L} in 'public_to_private'
 
+        remove R from read_barrier_cache
+
         return L
 
-    else:    # protected object
+    # else, R is a protected object
+    with collection_lock:
 
-        if h_revision is not a pointer:
+        allocate a backup copy and copy the object into the backup copy
 
-            allocate a backup copy, and attach it to h_revision
+        change R->h_revision to be the backup copy
+        
+        set GCFLAG_PRIVATE_FROM_PROTECTED on R
 
-        copy the object into the backup copy
-
-        change h_revision to be PRN (i.e. turn private)
-
-        if GCFLAG_WRITE_BARRIER: remove it, add R to the GC list of
-        modified old objects to trace at the next minor collection
+        add R in 'private_from_protected'
 
         return R
+
+
+
+Commit-time change of flags
+---------------------------
+
+(This occurs during commit, when we have got the collection_lock.)
+
+public_to_private:
+
+    write GT into the private object
+
+    make a stub with h_revision = private object | 2
+
+    after a CPU write barrier, make the public h_revision to point to the stub
+
+private_from_protected:
+
+    get the backup B from P->h_revision
+
+    set P->h_revision to GT
+
+    if B has GCFLAG_PUBLIC: it has been stolen
+
+        if it has been modified: conflict, abort transaction
+
+        B->h_revision = P
+
+    else:
+        possibly free B now, it's not used any more
+
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -289,12 +289,21 @@
   return L;
 }
 
+static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R);
+
 static gcptr LocalizeProtected(struct tx_descriptor *d, gcptr P)
 {
   gcptr B;
+  spinlock_acquire(d->public_descriptor->collection_lock, 'L');
+
+  if (P->h_tid & GCFLAG_PUBLIC)
+    {
+      /* became PUBLIC while waiting for the collection_lock */
+      spinlock_release(d->public_descriptor->collection_lock);
+      return LocalizePublic(d, P);
+    }
 
   assert(P->h_revision != stm_private_rev_num);
-  assert(!(P->h_tid & GCFLAG_PUBLIC));
   assert(!(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
   assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(P->h_tid & GCFLAG_STUB));
@@ -307,21 +316,17 @@
     }
   else
     {
-      size_t size = stmcb_size(P);
       B = (gcptr)P->h_revision;
       assert(B->h_tid & GCFLAG_BACKUP_COPY);
+      size_t size = stmcb_size(P);
       memcpy(B + 1, P + 1, size - sizeof(*B));
     }
   assert(B->h_tid & GCFLAG_BACKUP_COPY);
 
-  gcptrlist_locked_insert2(&d->public_descriptor->active_backup_copies, P, B,
-                           &d->public_descriptor->collection_lock);
+  gcptrlist_insert2(&d->public_descriptor->active_backup_copies, P, B);
+  P->h_revision = stm_private_rev_num;
 
-  smp_wmb();   /* guarantees that stm_steal_stub() will see the list
-                  up to the (P, B) pair in case it goes the path
-                  h_revision == *foreign_pd->private_revision_ref */
-
-  P->h_revision = stm_private_rev_num;
+  spinlock_release(d->public_descriptor->collection_lock);
   return P;
 }
 
@@ -368,6 +373,7 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
+ retry:
   P = stm_read_barrier(P);
 
   if (P->h_tid & GCFLAG_PUBLIC)
diff --git a/c4/lists.c b/c4/lists.c
--- a/c4/lists.c
+++ b/c4/lists.c
@@ -171,23 +171,6 @@
   gcptrlist->size = i + 2;
 }
 
-void gcptrlist_locked_insert2(struct GcPtrList *gcptrlist, gcptr newitem1,
-                              gcptr newitem2, revision_t *lock)
-{
-  gcptr *items;
-  long i = gcptrlist->size;
-  if (UNLIKELY((gcptrlist->alloc - i) < 2)) 
-    {
-      spinlock_acquire(*lock, 'I');
-      _gcptrlist_grow(gcptrlist);
-      spinlock_release(*lock);
-    }
-  items = gcptrlist->items;
-  items[i+0] = newitem1;
-  items[i+1] = newitem2;
-  gcptrlist->size = i + 2;
-}
-
 void gcptrlist_insert3(struct GcPtrList *gcptrlist, gcptr newitem1,
                        gcptr newitem2, gcptr newitem3)
 {
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -164,9 +164,6 @@
 void gcptrlist_merge(struct GcPtrList *, struct GcPtrList *gcptrlist_source);
 void gcptrlist_move(struct GcPtrList *, struct GcPtrList *gcptrlist_source);
 
-void gcptrlist_locked_insert2(struct GcPtrList *gcptrlist, gcptr newitem1,
-                              gcptr newitem2, revision_t *lock);
-
 /************************************************************/
 
 /* The fxcache_xx functions implement a fixed-size set of gcptr's.
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -53,33 +53,41 @@
     if ((v & 3) != 2)
         goto done;     /* un-stubbed while we waited for the lock */
 
-    gcptr Q, L = (gcptr)(v - 2);
-    revision_t w = ACCESS_ONCE(L->h_revision);
+    gcptr L = (gcptr)(v - 2);
+    revision_t w = L->h_revision;
 
     if (w == *foreign_pd->private_revision_ref) {
         /* The stub points to a private object L.  Because it cannot point
            to "really private" objects, it must mean that L used to be
            a protected object, and it has an attached backed copy.
            XXX find a way to optimize this search, maybe */
-        long i;
+        long i, size = foreign_pd->active_backup_copies.size;
         gcptr *items = foreign_pd->active_backup_copies.items;
-        /* we must find L as the first item of a pair in the list.  We
-           cannot rely on how big the list is here, but we know that
-           it will not be resized while we hold collection_lock. */
-        for (i = 0; items[i] != L; i += 2)
-            ;
+        for (i = size - 2; ; i -= 2) {
+            assert(i >= 0);
+            if (items[i] == L)
+                break;
+        }
         L = items[i + 1];
         assert(L->h_tid & GCFLAG_BACKUP_COPY);
+        L->h_tid &= ~GCFLAG_BACKUP_COPY;
     }
-    /* duplicate L */
-    Q = stmgc_duplicate(L);  XXX RACE
-    Q->h_tid &= ~GCFLAG_BACKUP_COPY;
-    Q->h_tid |= GCFLAG_PUBLIC;
-    gcptrlist_insert2(&foreign_pd->stolen_objects, L, Q);
+    else if (L->h_tid & GCFLAG_PUBLIC) {
+        /* The stub already points to a public object */
+        goto unstub;
+    }
+    else if (!(w & 1)) {
+        /* The stub points to a protected object L which has a backup
+           copy attached.  Forget the backup copy. */
+        w = ((gcptr)w)->h_revision;
+        assert(w & 1);
+        L->h_revision = w;
+    }
+    /* turn L into a public object */
+    L->h_tid |= GCFLAG_PUBLIC;
 
-    smp_wmb();
-
-    P->h_revision = (revision_t)Q;
+ unstub:
+    P->h_revision = (revision_t)L;
 
  done:
     spinlock_release(foreign_pd->collection_lock);

From noreply at buildbot.pypy.org  Sun Jun  9 19:07:13 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Jun 2013 19:07:13 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add stealing in the diagram
Message-ID: <20130609170713.0658A1C094A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86:d9e066ac3a1f
Date: 2013-06-09 19:07 +0200
http://bitbucket.org/pypy/stmgc/changeset/d9e066ac3a1f/

Log:	Add stealing in the diagram

diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt
--- a/c4/doc-objects.txt
+++ b/c4/doc-objects.txt
@@ -33,12 +33,12 @@
                  \          /  /
                   V        /  V
                    Protected
-                  ^
-                 /
-         commit /
-               /
-              /
-    Private copy of
+                  ^        \         Backup of a private
+                 /          \         /
+         commit /            \ steal /
+               /              \     /
+              /                V   V
+    Private copy of           Public
     a public obj
 
 

From noreply at buildbot.pypy.org  Sun Jun  9 21:05:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Jun 2013 21:05:35 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Implement most of the document,
 found a few issues to work on
Message-ID: <20130609190535.13C981C1001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87:391c68e20cf5
Date: 2013-06-09 21:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/391c68e20cf5/

Log:	Implement most of the document, found a few issues to work on

diff --git a/README.txt b/README.txt
--- a/README.txt
+++ b/README.txt
@@ -7,13 +7,13 @@
 This is a C library that combines a GC with STM capabilities.
 It is meant to be a general library that can be used in C programs.
 
-The library interface is in "c3/stmgc.h".
+The library interface is in "c4/stmgc.h".
 
-The file "c3/doc-stmgc.txt" contains a high-level overview followed by
-more detailled explanations.
+The file "c4/doc-objects.txt" contains some low-level explanations.
 
-A demo program can be found in "c3/demo1.c", but the code so far is
-outdated (it doesn't follow what c3/doc-stmgc describes).
+Run tests with "py.test".
+
+A demo program will be found in "c4/demo1.c" (not there yet).
 It can be built with "make debug-demo1" or "make build-demo1".
 
 The plan is to use this C code directly with PyPy, and not write
diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt
--- a/c4/doc-objects.txt
+++ b/c4/doc-objects.txt
@@ -219,13 +219,6 @@
 
 (This occurs during commit, when we have got the collection_lock.)
 
-public_to_private:
-
-    write GT into the private object
-
-    make a stub with h_revision = private object | 2
-
-    after a CPU write barrier, make the public h_revision to point to the stub
 
 private_from_protected:
 
@@ -233,6 +226,8 @@
 
     set P->h_revision to GT
 
+    remove GCFLAG_PRIVATE_FROM_PROTECTED from P
+
     if B has GCFLAG_PUBLIC: it has been stolen
 
         if it has been modified: conflict, abort transaction
@@ -242,3 +237,11 @@
     else:
         possibly free B now, it's not used any more
 
+
+public_to_private:
+
+    write GT into the private object
+
+    make a stub with h_revision = private object | 2
+
+    after a CPU write barrier, make the public h_revision to point to the stub
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -66,11 +66,16 @@
   gcptr P = G;
   revision_t v;
 
-  if (UNLIKELY(d->public_descriptor->stolen_objects.size > 0))
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
-      spinlock_acquire(d->public_descriptor->collection_lock, 'N');
-      stm_normalize_stolen_objects(d->public_descriptor);
-      spinlock_release(d->public_descriptor->collection_lock);
+    private_from_protected:
+      /* check P->h_revision->h_revision: if a pointer, then it means
+         the backup copy has been stolen into a public object and then
+         modified by some other thread.  Abort. */
+      if (!(((gcptr)P->h_revision)->h_revision & 1))
+        AbortTransaction(ABRT_STOLEN_MODIFIED);
+
+      goto add_in_recent_reads_cache;
     }
 
   if (P->h_tid & GCFLAG_PUBLIC)
@@ -142,8 +147,10 @@
     }
 
  register_in_list_of_read_objects:
+  gcptrlist_insert(&d->list_of_read_objects, P);
+
+ add_in_recent_reads_cache:
   fxcache_add(&d->recent_reads_cache, P);
-  gcptrlist_insert(&d->list_of_read_objects, P);
   return P;
 
  follow_stub:;
@@ -159,6 +166,12 @@
                   "private\n", G, P);
           return P;
         }
+      else if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+        {
+          fprintf(stderr, "read_barrier: %p -> %p handle "
+                  "private_from_protected\n", G, P);
+          goto private_from_protected;
+        }
       else if (FXCACHE_AT(P) == P)
         {
           fprintf(stderr, "read_barrier: %p -> %p handle "
@@ -307,24 +320,15 @@
   assert(!(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
   assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(P->h_tid & GCFLAG_STUB));
+  assert(!(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
 
-  if (P->h_revision & 1)
-    {
-      /* does not have a backup yet */
-      B = stmgc_duplicate(P);
-      B->h_tid |= GCFLAG_BACKUP_COPY;
-    }
-  else
-    {
-      B = (gcptr)P->h_revision;
-      assert(B->h_tid & GCFLAG_BACKUP_COPY);
-      size_t size = stmcb_size(P);
-      memcpy(B + 1, P + 1, size - sizeof(*B));
-    }
-  assert(B->h_tid & GCFLAG_BACKUP_COPY);
+  B = stmgc_duplicate(P);
+  B->h_tid |= GCFLAG_BACKUP_COPY;
 
-  gcptrlist_insert2(&d->public_descriptor->active_backup_copies, P, B);
-  P->h_revision = stm_private_rev_num;
+  P->h_tid |= GCFLAG_PRIVATE_FROM_PROTECTED;
+  P->h_revision = (revision_t)B;
+
+  gcptrlist_insert(&d->private_from_protected, P);
 
   spinlock_release(d->public_descriptor->collection_lock);
   return P;
@@ -333,21 +337,20 @@
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
   assert(R->h_tid & GCFLAG_PUBLIC);
-  if (R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
-    {
-      wlog_t *entry;
-      gcptr L;
-      G2L_FIND(d->public_to_private, R, entry, goto not_found);
-      L = entry->val;
-      assert(L->h_revision == stm_private_rev_num);   /* private object */
-      return L;
-    }
+
+#ifdef _GC_DEBUG
+  wlog_t *entry;
+  G2L_FIND(d->public_to_private, R, entry, goto not_found);
+  assert(!"R is already in public_to_private");
+ not_found:
+#endif
+
   R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
- not_found:;
   gcptr L = stmgc_duplicate(R);
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(L->h_tid & GCFLAG_STUB));
+  assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
   L->h_tid &= ~(GCFLAG_OLD               |
                 GCFLAG_VISITED           |
                 GCFLAG_PUBLIC            |
@@ -369,36 +372,31 @@
 
 gcptr stm_WriteBarrier(gcptr P)
 {
-  gcptr W;
+  gcptr R, W;
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
- retry:
-  P = stm_read_barrier(P);
+  R = stm_read_barrier(P);
 
-  if (P->h_tid & GCFLAG_PUBLIC)
-    W = LocalizePublic(d, P);
+  if (R->h_revision == stm_private_rev_num ||
+      (R->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED))
+    return R;
+
+  if (R->h_tid & GCFLAG_PUBLIC)
+    W = LocalizePublic(d, R);
   else
-    W = LocalizeProtected(d, P);
+    W = LocalizeProtected(d, R);
 
-  fprintf(stderr, "write_barrier: %p -> %p\n", P, W);
+  fprintf(stderr, "write_barrier: %p -> %p -> %p\n", P, R, W);
 
   return W;
 }
 
-gcptr stm_get_backup_copy(long index)
+gcptr stm_get_private_from_protected(long index)
 {
-  struct tx_public_descriptor *pd = thread_descriptor->public_descriptor;
-  if (index < gcptrlist_size(&pd->active_backup_copies))
-    return pd->active_backup_copies.items[index];
-  return NULL;
-}
-
-gcptr stm_get_stolen_obj(long index)
-{
-  struct tx_public_descriptor *pd = thread_descriptor->public_descriptor;
-  if (index < gcptrlist_size(&pd->stolen_objects))
-    return pd->stolen_objects.items[index];
+  struct tx_descriptor *d = thread_descriptor;
+  if (index < gcptrlist_size(&d->private_from_protected))
+    return d->private_from_protected.items[index];
   return NULL;
 }
 
@@ -570,9 +568,9 @@
   }
 
   gcptrlist_clear(&d->list_of_read_objects);
-  gcptrlist_clear(&d->public_descriptor->active_backup_copies);
   abort();
-  d->public_descriptor->stolen_objects;//XXX clean up
+  gcptrlist_clear(&d->private_from_protected);  //XXX clean up
+  abort();
   //stmgc_abort_transaction(d);
 
   fprintf(stderr,
@@ -636,14 +634,15 @@
     d->start_real_time.tv_nsec = -1;
   }
   assert(d->list_of_read_objects.size == 0);
+  assert(d->private_from_protected.size == 0);
   assert(!g2l_any_entry(&d->public_to_private));
 
   d->count_reads = 1;
   fxcache_clear(&d->recent_reads_cache);
 #if 0
   gcptrlist_clear(&d->undolog);
+  gcptrlist_clear(&d->abortinfo);
 #endif
-  gcptrlist_clear(&d->abortinfo);
 }
 
 void BeginTransaction(jmp_buf* buf)
@@ -819,24 +818,41 @@
 }
 #endif
 
-void TurnPrivateWithBackupToProtected(struct tx_descriptor *d,
-                                      revision_t cur_time)
+void CommitPrivateFromProtected(struct tx_descriptor *d, revision_t cur_time)
 {
-  struct tx_public_descriptor *pd = d->public_descriptor;
-  long i, size = pd->active_backup_copies.size;
-  gcptr *items = pd->active_backup_copies.items;
+  long i, size = d->private_from_protected.size;
+  gcptr *items = d->private_from_protected.items;
 
-  for (i = 0; i < size; i += 2)
+  for (i = 0; i < size; i++)
     {
       gcptr P = items[i];
-      gcptr B = items[i + 1];
-      assert(B->h_tid & GCFLAG_BACKUP_COPY);
-      assert(!(B->h_tid & GCFLAG_PUBLIC));
-      assert(P->h_revision == stm_private_rev_num);
-      B->h_revision = cur_time;
-      P->h_revision = (revision_t)B;
+
+      assert(!(P->h_revision & 1));   // "is a pointer"
+      gcptr B = (gcptr)P->h_revision;
+
+      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+      P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
+      P->h_revision = cur_time;
+
+      if (B->h_tid & GCFLAG_PUBLIC)
+        {
+          /* B was stolen */
+          while (1)
+            {
+              revision_t v = ACCESS_ONCE(B->h_revision);
+              if (!(v & 1))    // "is a pointer", i.e. "was modified"
+                AbortTransaction(ABRT_STOLEN_MODIFIED);
+
+              if (bool_cas(&B->h_revision, v, (revision_t)P))
+                break;
+            }
+        }
+      else
+        {
+          //stm_free(B);
+        }
     };
-  gcptrlist_clear(&pd->active_backup_copies);
+  gcptrlist_clear(&d->private_from_protected);
 }
 
 void CommitTransaction(void)
@@ -846,9 +862,6 @@
   assert(d->active >= 1);
 
   spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
-  if (d->public_descriptor->stolen_objects.size)
-    stm_normalize_stolen_objects(d->public_descriptor);
-
   AcquireLocks(d);
 
   if (is_inevitable(d))
@@ -885,6 +898,8 @@
         if (!ValidateDuringTransaction(d, 1))
           AbortTransaction(ABRT_VALIDATE_COMMIT);
     }
+  CommitPrivateFromProtected(d, cur_time);
+
   /* we cannot abort any more from here */
   d->setjmp_buf = NULL;
   gcptrlist_clear(&d->list_of_read_objects);
@@ -895,8 +910,6 @@
           "*************************************\n",
           (long)cur_time);
 
-  TurnPrivateWithBackupToProtected(d, cur_time);
-
   revision_t localrev = stm_private_rev_num;
   //UpdateProtectedChainHeads(d, cur_time, localrev);
   //smp_wmb();
@@ -1159,12 +1172,12 @@
     thread_descriptor = NULL;
 
     g2l_delete(&d->public_to_private);
-    assert(d->public_descriptor->active_backup_copies.size == 0);
-    gcptrlist_delete(&d->public_descriptor->active_backup_copies);
+    assert(d->private_from_protected.size == 0);
+    gcptrlist_delete(&d->private_from_protected);
     gcptrlist_delete(&d->list_of_read_objects);
+#if 0
     gcptrlist_delete(&d->abortinfo);
     free(d->longest_abort_info);
-#if 0
     gcptrlist_delete(&d->undolog);
 #endif
 
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -62,6 +62,7 @@
 #define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 6)
 #define GCFLAG_BACKUP_COPY       (STM_FIRST_GCFLAG << 7)   /* debugging */
 #define GCFLAG_STUB              (STM_FIRST_GCFLAG << 8)   /* debugging */
+#define GCFLAG_PRIVATE_FROM_PROTECTED (STM_FIRST_GCFLAG << 9)
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
@@ -73,24 +74,25 @@
                          "VISITED",           \
                          "PUBLIC",            \
                          "PREBUILT_ORIGINAL", \
-                         "BACKUP_COPY",       \
                          "PUBLIC_TO_PRIVATE", \
                          "WRITE_BARRIER",     \
                          "NURSERY_MOVED",     \
-                         "STOLEN",            \
+                         "BACKUP_COPY",       \
                          "STUB",              \
+                         "PRIVATE_FROM_PROTECTED", \
                          NULL }
 
 /************************************************************/
 
 #define ABRT_MANUAL               0
 #define ABRT_COMMIT               1
-#define ABRT_VALIDATE_INFLIGHT    2
-#define ABRT_VALIDATE_COMMIT      3
-#define ABRT_VALIDATE_INEV        4
-#define ABRT_COLLECT_MINOR        5
-#define ABRT_COLLECT_MAJOR        6
-#define ABORT_REASONS         7
+#define ABRT_STOLEN_MODIFIED      2
+#define ABRT_VALIDATE_INFLIGHT    3
+#define ABRT_VALIDATE_COMMIT      4
+#define ABRT_VALIDATE_INEV        5
+#define ABRT_COLLECT_MINOR        6
+#define ABRT_COLLECT_MAJOR        7
+#define ABORT_REASONS         8
 
 #define SPLP_ABORT                0
 #define SPLP_LOCKED_INFLIGHT      1
@@ -105,8 +107,6 @@
   revision_t collection_lock;
   struct stub_block_s *stub_blocks;
   gcptr stub_free_list;
-  struct GcPtrList active_backup_copies;  /* (P,B) where P=private, B=backup */
-  struct GcPtrList stolen_objects;      /* (P,Q) where P=priv/prot, Q=public */
   revision_t *private_revision_ref;
   revision_t free_list_next;
   /* xxx gcpage data here */
@@ -135,7 +135,8 @@
   unsigned int num_aborts[ABORT_REASONS];
   unsigned int num_spinloops[SPINLOOP_REASONS];
   struct GcPtrList list_of_read_objects;
-  struct GcPtrList abortinfo;
+  //struct GcPtrList abortinfo;
+  struct GcPtrList private_from_protected;
   struct G2L public_to_private;
   char *longest_abort_info;
   long long longest_abort_info_time;
@@ -162,8 +163,7 @@
 gcptr stm_RepeatReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
 gcptr _stm_nonrecord_barrier(gcptr, int *);
-gcptr stm_get_backup_copy(long);  /* debugging */
-gcptr stm_get_stolen_obj(long);  /* debugging */
+gcptr stm_get_private_from_protected(long);  /* debugging */
 gcptr stm_get_read_obj(long);  /* debugging */
 gcptr stmgc_duplicate(gcptr);
 
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -54,54 +54,21 @@
         goto done;     /* un-stubbed while we waited for the lock */
 
     gcptr L = (gcptr)(v - 2);
-    revision_t w = L->h_revision;
 
-    if (w == *foreign_pd->private_revision_ref) {
-        /* The stub points to a private object L.  Because it cannot point
-           to "really private" objects, it must mean that L used to be
-           a protected object, and it has an attached backed copy.
-           XXX find a way to optimize this search, maybe */
-        long i, size = foreign_pd->active_backup_copies.size;
-        gcptr *items = foreign_pd->active_backup_copies.items;
-        for (i = size - 2; ; i -= 2) {
-            assert(i >= 0);
-            if (items[i] == L)
-                break;
-        }
-        L = items[i + 1];
-        assert(L->h_tid & GCFLAG_BACKUP_COPY);
+    if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+        L = (gcptr)L->h_revision;     /* the backup copy */
         L->h_tid &= ~GCFLAG_BACKUP_COPY;
     }
-    else if (L->h_tid & GCFLAG_PUBLIC) {
-        /* The stub already points to a public object */
-        goto unstub;
-    }
-    else if (!(w & 1)) {
-        /* The stub points to a protected object L which has a backup
-           copy attached.  Forget the backup copy. */
-        w = ((gcptr)w)->h_revision;
-        assert(w & 1);
-        L->h_revision = w;
-    }
-    /* turn L into a public object */
+
+    /* change L from protected to public */
     L->h_tid |= GCFLAG_PUBLIC;
 
- unstub:
+    smp_wmb();      /* the following update must occur "after" the flag
+                       GCFLAG_PUBLIC was added, for other threads */
+
+    /* update the original P->h_revision to point directly to L */
     P->h_revision = (revision_t)L;
 
  done:
     spinlock_release(foreign_pd->collection_lock);
 }
-
-void stm_normalize_stolen_objects(struct tx_public_descriptor *pd)
-{
-    long i, size = pd->stolen_objects.size;
-    gcptr *items = pd->stolen_objects.items;
-    for (i = 0; i < size; i += 2) {
-        gcptr L = items[i];
-        gcptr Q = items[i + 1];
-        if (L->h_revision == stm_private_rev_num) {
-            
-        }
-    }
-}
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -87,8 +87,8 @@
 gcptr stm_write_barrier(gcptr obj)
 {
     /* XXX inline in the caller */
-    if (UNLIKELY(((obj->h_tid & GCFLAG_WRITE_BARRIER) != 0) |
-                 (obj->h_revision != stm_private_rev_num)))
+    if (UNLIKELY((obj->h_revision != stm_private_rev_num) &
+                 ((obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) == 0)))
         obj = stm_WriteBarrier(obj);
     return obj;
 }
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -68,8 +68,7 @@
     void stm_start_sharedlock(void);
     void stm_stop_sharedlock(void);
     void AbortTransaction(int);
-    gcptr stm_get_backup_copy(long index);
-    gcptr stm_get_stolen_obj(long index);
+    gcptr stm_get_private_from_protected(long index);
     gcptr stm_get_read_obj(long index);
     void *STUB_THREAD(gcptr);
 
@@ -105,6 +104,7 @@
     #define GCFLAG_WRITE_BARRIER     ...
     #define GCFLAG_NURSERY_MOVED     ...
     #define GCFLAG_STUB              ...
+    #define GCFLAG_PRIVATE_FROM_PROTECTED  ...
     #define ABRT_MANUAL              ...
     //typedef struct { ...; } page_header_t;
 ''')
@@ -524,7 +524,8 @@
     lib.AbortTransaction(lib.ABRT_MANUAL)
 
 def classify(p):
-    private = p.h_revision == lib.get_private_rev_num()
+    private = (p.h_revision == lib.get_private_rev_num() or
+               (p.h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) != 0)
     public  = (p.h_tid & GCFLAG_PUBLIC) != 0
     backup  = (p.h_tid & GCFLAG_BACKUP_COPY) != 0
     stub    = (p.h_tid & GCFLAG_STUB) != 0
@@ -541,34 +542,26 @@
     else:
         return "protected"
 
-def list_of_read_objects():
+def _get_full_list(getter):
     result = []
     index = 0
     while 1:
-        p = lib.stm_get_read_obj(index)
+        p = getter(index)
         if p == ffi.NULL:
             break
         result.append(p)
         index += 1
     return result
 
-def _list2dict(getter):
-    result = {}
-    index = 0
-    while 1:
-        p = getter(index)
-        if p == ffi.NULL:
-            break
-        q = getter(index + 1)
-        assert q != ffi.NULL
-        result[p] = q
-        index += 2
-    return result
+def list_of_read_objects():
+    return _get_full_list(lib.stm_get_read_obj)
 
-def backup_copies():
-    return _list2dict(lib.stm_get_backup_copy)
-
-def stolen_objs():
-    return _list2dict(lib.stm_get_stolen_obj)
+def list_of_private_from_protected():
+    return _get_full_list(lib.stm_get_private_from_protected)
 
 stub_thread = lib.STUB_THREAD
+
+def follow_revision(p):
+    r = p.h_revision
+    assert (r % 4) == 0
+    return ffi.cast("gcptr", r)
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -49,8 +49,10 @@
     assert classify(p) == "protected"
     p2 = lib.stm_write_barrier(p)
     assert p2 == p       # does not move
-    assert p.h_revision == r2
     assert classify(p) == "private"
+    pback = follow_revision(p)
+    assert classify(pback) == "backup"
+    assert list_of_private_from_protected() == [p]
 
 def test_get_backup_copy():
     p = nalloc(HDR + WORD)
@@ -58,52 +60,19 @@
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
     org_r = p.h_revision
+    assert classify(p) == "protected"
     lib.setlong(p, 0, 927122)
-    assert p.h_revision == lib.get_private_rev_num()
-    pback = backup_copies()[p]
+    assert classify(p) == "private"
+    pback = follow_revision(p)
     assert pback and pback != p
     assert pback.h_revision == org_r
-    assert pback.h_tid == p.h_tid | GCFLAG_BACKUP_COPY
+    assert pback.h_tid == ((p.h_tid & ~GCFLAG_PRIVATE_FROM_PROTECTED) |
+                           GCFLAG_BACKUP_COPY)
     assert lib.rawgetlong(pback, 0) == 78927812
     assert lib.rawgetlong(p, 0) == 927122
     assert classify(p) == "private"
     assert classify(pback) == "backup"
 
-def test_protected_with_backup():
-    p = nalloc(HDR + WORD)
-    lib.setlong(p, 0, 78927812)
-    lib.stm_commit_transaction()
-    lib.stm_begin_inevitable_transaction()
-    lib.setlong(p, 0, 927122)
-    pback = backup_copies()[p]
-    assert pback != p
-    assert p.h_revision == lib.get_private_rev_num()
-    lib.stm_commit_transaction()
-    lib.stm_begin_inevitable_transaction()
-    assert classify(p) == "protected"
-    assert classify(pback) == "backup"
-    assert ffi.cast("gcptr", p.h_revision) == pback
-
-def test_protected_backup_reused():
-    p = nalloc(HDR + WORD)
-    lib.setlong(p, 0, 78927812)
-    lib.stm_commit_transaction()
-    lib.stm_begin_inevitable_transaction()
-    lib.setlong(p, 0, 927122)
-    pback = backup_copies()[p]
-    assert pback != p
-    lib.stm_commit_transaction()
-    lib.stm_begin_inevitable_transaction()
-    assert classify(p) == "protected"
-    assert classify(pback) == "backup"
-    assert lib.rawgetlong(p, 0) == 927122
-    assert lib.rawgetlong(pback, 0) == 78927812    # but should not be used
-    lib.setlong(p, 0, 43891)
-    assert p.h_revision == lib.get_private_rev_num()
-    assert pback == backup_copies()[p]
-    assert lib.rawgetlong(p, 0) == 43891
-    assert lib.rawgetlong(pback, 0) == 927122
-
 def test_prebuilt_is_public():
     p = palloc(HDR)
     assert p.h_revision == 1
@@ -240,15 +209,13 @@
         assert classify(p) == "public"
         assert classify(p1) == "protected"
         plist.append(p1)     # now p's most recent revision is protected
-        assert classify(ffi.cast("gcptr", p.h_revision)) == "stub"
+        assert classify(follow_revision(p)) == "stub"
+        assert p1.h_revision & 1
         r.set(2)
         r.wait(3)
-        d = stolen_objs()
-        assert len(d) == 1
-        assert d.keys() == [p1]
-        [p2] = d.values()
-        assert lib.stm_read_barrier(p) == p2
-        assert lib.stm_read_barrier(p1) == p2
+        assert classify(p1) == "public"
+        assert lib.stm_read_barrier(p) == p1
+        assert lib.stm_read_barrier(p1) == p1
     def f2(r):
         r.wait(2)
         p2 = lib.stm_read_barrier(p)    # steals
@@ -256,8 +223,61 @@
         assert p2 == lib.stm_read_barrier(p)    # short-circuit h_revision
         assert p.h_revision == int(ffi.cast("revision_t", p2))
         assert p2 == lib.stm_read_barrier(p)
-        assert p2 not in plist
+        assert p2 == plist[-1]
         assert classify(p2) == "public"
-        plist.append(p2)
         r.set(3)
     run_parallel(f1, f2)
+
+def test_stealing_while_modifying():
+    py.test.skip("in-progress")
+    p = palloc(HDR + WORD)
+
+    def f1(r):
+        p1 = lib.stm_write_barrier(p)   # private copy
+        assert classify(p) == "public"
+        assert classify(p1) == "private"
+        lib.rawsetlong(p1, 0, 2782172)
+
+        def cb(c):
+            assert c == 0
+            assert classify(p) == "public"
+            assert classify(p1) == "protected"
+            assert classify(follow_revision(p)) == "stub"
+            p2 = lib.stm_write_barrier(p)
+            assert p2 == p1
+            lib.rawsetlong(p2, 0, -451112)
+            pback = follow_revision(p1)
+            assert classify(p1) == "private"
+            assert classify(pback) == "backup"
+            assert lib.stm_read_barrier(p) == p1
+            assert lib.stm_read_barrier(p1) == p1
+            assert pback.h_revision & 1
+            r.wait_while_in_parallel()
+            assert classify(p1) == "private"
+            assert classify(pback) == "public"
+            assert pback.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
+            assert lib.stm_read_barrier(p) == p1
+            assert lib.stm_read_barrier(p1) == p1
+            assert pback.h_revision & 1
+        perform_transaction(cb)
+
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        assert classify(p1) == "protected"
+        assert classify(pback) == "public"
+        assert classify(follow_revision(pback)) == "stub"
+        assert follow_revision(pback).h_revision == (
+            ffi.cast("revision_t", p1) | 2)
+
+    def f2(r):
+        def cb(c):
+            assert c == 0
+            r.enter_in_parallel()
+            p2 = lib.stm_read_barrier(p)    # steals
+            assert lib.rawgetlong(p2, 0) == 2782172
+            assert p2 == lib.stm_read_barrier(p)
+            assert classify(p2) == "public"
+            r.leave_in_parallel()
+        perform_transaction(cb)
+
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Sun Jun  9 22:23:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Jun 2013 22:23:21 +0200 (CEST)
Subject: [pypy-commit] stmgc default: progress
Message-ID: <20130609202321.903931C1001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88:38cc005153e5
Date: 2013-06-09 22:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/38cc005153e5/

Log:	progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -30,6 +30,11 @@
 {
   return thread_descriptor;
 }
+static int is_private(gcptr P)
+{
+  return (P->h_revision == stm_private_rev_num) ||
+    (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+}
 
 /************************************************************/
 
@@ -119,6 +124,7 @@
       if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
         {
           wlog_t *item;
+        retry_public_to_private:;
           G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
 
           P = item->val;
@@ -126,8 +132,14 @@
           assert(P->h_revision == stm_private_rev_num);
           fprintf(stderr, "read_barrier: %p -> %p public_to_private\n", G, P);
           return P;
+
+        no_private_obj:
+          if (d->public_descriptor->stolen_objects.size > 0)
+            {
+              stm_normalize_stolen_objects(d);
+              goto retry_public_to_private;
+            }
         }
-    no_private_obj:
 
       if (UNLIKELY(v > d->start_time))   // object too recent?
         {
@@ -188,7 +200,7 @@
   else
     {
       /* stealing */
-      fprintf(stderr, "read_barrier: %p -> stealing %p...", G, P);
+      fprintf(stderr, "read_barrier: %p -> stealing %p...\n  ", G, P);
       stm_steal_stub(P);
       goto retry;
     }
@@ -378,8 +390,7 @@
 
   R = stm_read_barrier(P);
 
-  if (R->h_revision == stm_private_rev_num ||
-      (R->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED))
+  if (is_private(R))
     return R;
 
   if (R->h_tid & GCFLAG_PUBLIC)
@@ -419,11 +430,14 @@
 static _Bool ValidateDuringTransaction(struct tx_descriptor *d,
                                        _Bool during_commit)
 {
-  abort();
-#if 0
   long i, size = d->list_of_read_objects.size;
   gcptr *items = d->list_of_read_objects.items;
 
+  if (size == 0)
+    return 1;
+  abort();
+
+#if 0
   for (i=0; i<size; i++)
     {
       gcptr R = items[i];
@@ -636,6 +650,7 @@
   assert(d->list_of_read_objects.size == 0);
   assert(d->private_from_protected.size == 0);
   assert(!g2l_any_entry(&d->public_to_private));
+  assert(d->public_descriptor->stolen_objects.size == 0);
 
   d->count_reads = 1;
   fxcache_clear(&d->recent_reads_cache);
@@ -668,7 +683,7 @@
       gcptr R = item->addr;
       revision_t v;
     retry:
-      assert(R->h_tid & GCFLAG_OLD);
+      assert(R->h_tid & GCFLAG_PUBLIC);
       v = ACCESS_ONCE(R->h_revision);
       if (!(v & 1))            // "is a pointer", i.e.
         {                      //   "has a more recent revision"
@@ -861,6 +876,9 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
+  if (d->public_descriptor->stolen_objects.size != 0)
+    stm_normalize_stolen_objects(d);
+
   spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
   AcquireLocks(d);
 
@@ -918,7 +936,7 @@
   assert(newrev & 1);
   ACCESS_ONCE(stm_private_rev_num) = newrev;
   fprintf(stderr, "%p: stm_local_revision = %ld\n", d, (long)newrev);
-  assert(d->public_descriptor->private_revision_ref = &stm_private_rev_num);
+  assert(d->private_revision_ref = &stm_private_rev_num);
 
   UpdateChainHeads(d, cur_time, localrev);
 
@@ -1136,8 +1154,9 @@
       assert(d->my_lock & 1);
       assert(d->my_lock >= LOCKED);
       stm_private_rev_num = -1;
-      pd->private_revision_ref = &stm_private_rev_num;
+      d->private_revision_ref = &stm_private_rev_num;
       d->max_aborts = -1;
+      pd->descriptor = d;
       thread_descriptor = d;
 
       fprintf(stderr, "[%lx] pthread %lx starting\n",
@@ -1152,15 +1171,12 @@
 
 void DescriptorDone(void)
 {
-    static revision_t no_private_revision = 8;
     revision_t i;
     struct tx_descriptor *d = thread_descriptor;
     assert(d != NULL);
     assert(d->active == 0);
-
-    spinlock_acquire(d->public_descriptor->collection_lock, 'D');  /*done*/
-    d->public_descriptor->private_revision_ref = &no_private_revision;
-    spinlock_release(d->public_descriptor->collection_lock);
+    assert(d->public_descriptor->stolen_objects.size == 0);
+    gcptrlist_delete(&d->public_descriptor->stolen_objects);
 
     spinlock_acquire(descriptor_array_lock, 1);
     i = d->public_descriptor_index;
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -105,9 +105,10 @@
  * thread shuts down.  It is reused the next time a thread starts. */
 struct tx_public_descriptor {
   revision_t collection_lock;
+  struct tx_descriptor *descriptor;
   struct stub_block_s *stub_blocks;
   gcptr stub_free_list;
-  revision_t *private_revision_ref;
+  struct GcPtrList stolen_objects;
   revision_t free_list_next;
   /* xxx gcpage data here */
 };
@@ -140,6 +141,7 @@
   struct G2L public_to_private;
   char *longest_abort_info;
   long long longest_abort_info_time;
+  revision_t *private_revision_ref;
   struct FXCache recent_reads_cache;
 };
 
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -56,8 +56,16 @@
     gcptr L = (gcptr)(v - 2);
 
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
-        L = (gcptr)L->h_revision;     /* the backup copy */
-        L->h_tid &= ~GCFLAG_BACKUP_COPY;
+        gcptr B = (gcptr)L->h_revision;     /* the backup copy */
+        B->h_tid &= ~GCFLAG_BACKUP_COPY;
+        L->h_revision = *foreign_pd->descriptor->private_revision_ref;
+
+        /* add {B: L} in 'public_to_private', but lazily, because we don't
+           want to walk over the feet of the foreign thread */
+        B->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
+        gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
+
+        L = B;
     }
 
     /* change L from protected to public */
@@ -72,3 +80,20 @@
  done:
     spinlock_release(foreign_pd->collection_lock);
 }
+
+void stm_normalize_stolen_objects(struct tx_descriptor *d)
+{
+    spinlock_acquire(d->public_descriptor->collection_lock, 'N');
+
+    long i, size = d->public_descriptor->stolen_objects.size;
+    gcptr *items = d->public_descriptor->stolen_objects.items;
+
+    for (i = 0; i < size; i += 2) {
+        gcptr B = items[i];
+        gcptr L = items[i + 1];
+        g2l_insert(&d->public_to_private, B, L);
+    }
+    gcptrlist_clear(&d->public_descriptor->stolen_objects);
+
+    spinlock_release(d->public_descriptor->collection_lock);
+}
diff --git a/c4/steal.h b/c4/steal.h
--- a/c4/steal.h
+++ b/c4/steal.h
@@ -10,7 +10,7 @@
 gcptr stm_stub_malloc(struct tx_public_descriptor *);
 void stm_steal_stub(gcptr);
 gcptr stm_get_stolen_obj(long index);   /* debugging */
-void stm_normalize_stolen_objects(struct tx_public_descriptor *);
+void stm_normalize_stolen_objects(struct tx_descriptor *);
 
 
 #endif
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -229,7 +229,6 @@
     run_parallel(f1, f2)
 
 def test_stealing_while_modifying():
-    py.test.skip("in-progress")
     p = palloc(HDR + WORD)
 
     def f1(r):
@@ -258,6 +257,7 @@
             assert pback.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
             assert lib.stm_read_barrier(p) == p1
             assert lib.stm_read_barrier(p1) == p1
+            assert lib.stm_read_barrier(pback) == p1
             assert pback.h_revision & 1
         perform_transaction(cb)
 
@@ -273,6 +273,8 @@
         def cb(c):
             assert c == 0
             r.enter_in_parallel()
+            lib.stm_commit_transaction()
+            lib.stm_begin_inevitable_transaction()
             p2 = lib.stm_read_barrier(p)    # steals
             assert lib.rawgetlong(p2, 0) == 2782172
             assert p2 == lib.stm_read_barrier(p)

From noreply at buildbot.pypy.org  Mon Jun 10 09:26:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Jun 2013 09:26:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Getting somewhere
Message-ID: <20130610072625.98AF01C10DD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89:b63ed074b565
Date: 2013-06-10 09:26 +0200
http://bitbucket.org/pypy/stmgc/changeset/b63ed074b565/

Log:	Getting somewhere

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -129,14 +129,16 @@
 
           P = item->val;
           assert(!(P->h_tid & GCFLAG_PUBLIC));
-          assert(P->h_revision == stm_private_rev_num);
+          assert(is_private(P));
           fprintf(stderr, "read_barrier: %p -> %p public_to_private\n", G, P);
           return P;
 
         no_private_obj:
           if (d->public_descriptor->stolen_objects.size > 0)
             {
+              spinlock_acquire(d->public_descriptor->collection_lock, 'N');
               stm_normalize_stolen_objects(d);
+              spinlock_release(d->public_descriptor->collection_lock);
               goto retry_public_to_private;
             }
         }
@@ -433,11 +435,6 @@
   long i, size = d->list_of_read_objects.size;
   gcptr *items = d->list_of_read_objects.items;
 
-  if (size == 0)
-    return 1;
-  abort();
-
-#if 0
   for (i=0; i<size; i++)
     {
       gcptr R = items[i];
@@ -446,18 +443,7 @@
       v = ACCESS_ONCE(R->h_revision);
       if (!(v & 1))               // "is a pointer", i.e.
         {                         //   "has a more recent revision"
-          /* ... unless it's a protected-to-private link */
-          if (((gcptr)v)->h_revision == stm_local_revision)
-            continue;
-          /* ... or unless it is a GCFLAG_STOLEN object */
-          if (R->h_tid & GCFLAG_STOLEN)
-            {
-              assert(is_young(R));
-              assert(!is_young((gcptr)v));
-              R = (gcptr)v;
-              goto retry;
-            }
-          return 0;               // really has a more recent revision
+          return 0;
         }
       if (v >= LOCKED)            // locked
         {
@@ -475,7 +461,6 @@
         }
     }
   return 1;
-#endif
 }
 
 static void ValidateNow(struct tx_descriptor *d)
@@ -703,8 +688,11 @@
         goto retry;
 
       gcptr L = item->val;
-      assert(L->h_revision == stm_private_rev_num);
+      assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED ?
+             L->h_revision == (revision_t)R :
+             L->h_revision == stm_private_rev_num);
       assert(v != stm_private_rev_num);
+      assert(v & 1);
       L->h_revision = v;   /* store temporarily this value here */
 
     } G2L_LOOP_END;
@@ -712,8 +700,6 @@
 
 static void CancelLocks(struct tx_descriptor *d)
 {
-  abort();
-#if 0
   revision_t my_lock = d->my_lock;
   wlog_t *item;
 
@@ -724,13 +710,20 @@
     {
       gcptr R = item->addr;
       gcptr L = item->val;
-      revision_t v = L->h_revision;
-      if (v == stm_local_revision)
+      revision_t expected, v = L->h_revision;
+
+      if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+        expected = (revision_t)R;
+      else
+        expected = stm_private_rev_num;
+
+      if (v == expected)
         {
           assert(R->h_revision != my_lock);
           break;    /* done */
         }
-      L->h_revision = stm_local_revision;
+
+      L->h_revision = expected;
 
 #ifdef DUMP_EXTRA
       fprintf(stderr, "%p->h_revision = %p (CancelLocks)\n", R, (gcptr)v);
@@ -739,7 +732,6 @@
       ACCESS_ONCE(R->h_revision) = v;
 
     } G2L_LOOP_END;
-#endif
 }
 
 //static pthread_mutex_t mutex_prebuilt_gcroots = PTHREAD_MUTEX_INITIALIZER;
@@ -812,27 +804,6 @@
   g2l_clear(&d->public_to_private);
 }
 
-#if 0
-void UpdateProtectedChainHeads(struct tx_descriptor *d, revision_t cur_time,
-                               revision_t localrev)
-{
-  revision_t new_revision = cur_time + 1;     // make an odd number
-  assert(new_revision & 1);
-
-  long i, size = d->protected_with_private_copy.size;
-  gcptr *items = d->protected_with_private_copy.items;
-  for (i = 0; i < size; i++)
-    {
-      gcptr R = items[i];
-      if (R->h_tid & GCFLAG_STOLEN)       /* ignore stolen objects */
-        continue;
-      gcptr L = (gcptr)R->h_revision;
-      assert(L->h_revision == localrev);
-      L->h_revision = new_revision;
-    }
-}
-#endif
-
 void CommitPrivateFromProtected(struct tx_descriptor *d, revision_t cur_time)
 {
   long i, size = d->private_from_protected.size;
@@ -841,12 +812,19 @@
   for (i = 0; i < size; i++)
     {
       gcptr P = items[i];
-
-      assert(!(P->h_revision & 1));   // "is a pointer"
-      gcptr B = (gcptr)P->h_revision;
-
       assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
       P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
+
+      if (P->h_revision & 1)   // "is not a pointer"
+        {
+          /* This case occurs when a GCFLAG_PRIVATE_FROM_PROTECTED object
+             is stolen: it ends up as a value in 'public_to_private'.
+             Its h_revision is then mangled by AcquireLocks(). */
+          assert(P->h_revision != stm_private_rev_num);
+          continue;
+        }
+
+      gcptr B = (gcptr)P->h_revision;
       P->h_revision = cur_time;
 
       if (B->h_tid & GCFLAG_PUBLIC)
@@ -876,10 +854,10 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
+  spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
   if (d->public_descriptor->stolen_objects.size != 0)
     stm_normalize_stolen_objects(d);
 
-  spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
   AcquireLocks(d);
 
   if (is_inevitable(d))
@@ -1219,7 +1197,7 @@
                      d->num_spinloops[i]);
 
     p += sprintf(p, "]\n");
-    fwrite(line, 1, p - line, stderr);
+    fprintf(stderr, "%s", line);
 
     stm_free(d, sizeof(struct tx_descriptor));
 }
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -58,13 +58,14 @@
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
-        L->h_revision = *foreign_pd->descriptor->private_revision_ref;
 
         /* add {B: L} in 'public_to_private', but lazily, because we don't
            want to walk over the feet of the foreign thread */
         B->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
         gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
 
+        fprintf(stderr, "stolen: %p -> %p - - -> %p\n", P, B, L);
+
         L = B;
     }
 
@@ -83,17 +84,15 @@
 
 void stm_normalize_stolen_objects(struct tx_descriptor *d)
 {
-    spinlock_acquire(d->public_descriptor->collection_lock, 'N');
-
     long i, size = d->public_descriptor->stolen_objects.size;
     gcptr *items = d->public_descriptor->stolen_objects.items;
 
     for (i = 0; i < size; i += 2) {
         gcptr B = items[i];
         gcptr L = items[i + 1];
+
+        assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
         g2l_insert(&d->public_to_private, B, L);
     }
     gcptrlist_clear(&d->public_descriptor->stolen_objects);
-
-    spinlock_release(d->public_descriptor->collection_lock);
 }
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -236,6 +236,7 @@
         assert classify(p) == "public"
         assert classify(p1) == "private"
         lib.rawsetlong(p1, 0, 2782172)
+        pback_ = []
 
         def cb(c):
             assert c == 0
@@ -246,6 +247,7 @@
             assert p2 == p1
             lib.rawsetlong(p2, 0, -451112)
             pback = follow_revision(p1)
+            pback_.append(pback)
             assert classify(p1) == "private"
             assert classify(pback) == "backup"
             assert lib.stm_read_barrier(p) == p1
@@ -263,11 +265,12 @@
 
         lib.stm_commit_transaction()
         lib.stm_begin_inevitable_transaction()
+        [pback] = pback_
         assert classify(p1) == "protected"
         assert classify(pback) == "public"
         assert classify(follow_revision(pback)) == "stub"
         assert follow_revision(pback).h_revision == (
-            ffi.cast("revision_t", p1) | 2)
+            int(ffi.cast("revision_t", p1)) | 2)
 
     def f2(r):
         def cb(c):

From noreply at buildbot.pypy.org  Mon Jun 10 09:49:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Jun 2013 09:49:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Aborting,
 including aborting GCFLAG_PRIVATE_FROM_PROTECTED
Message-ID: <20130610074938.3B11B1C10DD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90:50f4a1c6ad65
Date: 2013-06-10 09:49 +0200
http://bitbucket.org/pypy/stmgc/changeset/50f4a1c6ad65/

Log:	Aborting, including aborting GCFLAG_PRIVATE_FROM_PROTECTED

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -494,6 +494,10 @@
   struct timespec now;
   long long elapsed_time;
 
+  /* acquire the lock, but don't double-acquire it if already committing */
+  if (d->public_descriptor->collection_lock != 'C')
+    spinlock_acquire(d->public_descriptor->collection_lock, 'C');
+
   assert(d->active != 0);
   assert(!is_inevitable(d));
   assert(num < ABORT_REASONS);
@@ -566,11 +570,13 @@
       d->reads_size_limit_nonatomic = limit;
   }
 
+  AbortPrivateFromProtected(d);
   gcptrlist_clear(&d->list_of_read_objects);
-  abort();
-  gcptrlist_clear(&d->private_from_protected);  //XXX clean up
-  abort();
-  //stmgc_abort_transaction(d);
+  g2l_clear(&d->public_to_private);
+  gcptrlist_clear(&d->public_descriptor->stolen_objects);
+
+  /* release the lock */
+  spinlock_release(d->public_descriptor->collection_lock);
 
   fprintf(stderr,
           "\n"
@@ -848,6 +854,35 @@
   gcptrlist_clear(&d->private_from_protected);
 }
 
+void AbortPrivateFromProtected(struct tx_descriptor *d)
+{
+  long i, size = d->private_from_protected.size;
+  gcptr *items = d->private_from_protected.items;
+
+  for (i = 0; i < size; i++)
+    {
+      gcptr P = items[i];
+      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+      assert(!(P->h_revision & 1));   // "is a pointer"
+
+      gcptr B = (gcptr)P->h_revision;
+      if (B->h_tid & GCFLAG_PUBLIC)
+        {
+          assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
+          P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
+          P->h_tid |= GCFLAG_PUBLIC;
+          /* P becomes a public outdated object */
+        }
+      else
+        {
+          assert(B->h_tid & GCFLAG_BACKUP_COPY);
+          memcpy(P, B, stmcb_size(P));
+          P->h_tid &= ~GCFLAG_BACKUP_COPY;
+        }
+    };
+  gcptrlist_clear(&d->private_from_protected);
+}
+
 void CommitTransaction(void)
 {   /* must save roots around this call */
   revision_t cur_time;
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -228,7 +228,7 @@
         r.set(3)
     run_parallel(f1, f2)
 
-def test_stealing_while_modifying():
+def test_stealing_while_modifying(aborting=False):
     p = palloc(HDR + WORD)
 
     def f1(r):
@@ -239,7 +239,15 @@
         pback_ = []
 
         def cb(c):
-            assert c == 0
+            if c != 0:
+                assert aborting
+                [pback] = pback_
+                assert classify(p) == "public"
+                assert classify(p1) == "public"
+                assert classify(pback) == "public"
+                assert lib.stm_read_barrier(p) == pback
+                assert lib.stm_read_barrier(p1) == pback
+                return
             assert classify(p) == "public"
             assert classify(p1) == "protected"
             assert classify(follow_revision(p)) == "stub"
@@ -261,16 +269,24 @@
             assert lib.stm_read_barrier(p1) == p1
             assert lib.stm_read_barrier(pback) == p1
             assert pback.h_revision & 1
+            if aborting:
+                abort_and_retry()
         perform_transaction(cb)
 
         lib.stm_commit_transaction()
         lib.stm_begin_inevitable_transaction()
         [pback] = pback_
-        assert classify(p1) == "protected"
-        assert classify(pback) == "public"
-        assert classify(follow_revision(pback)) == "stub"
-        assert follow_revision(pback).h_revision == (
-            int(ffi.cast("revision_t", p1)) | 2)
+        if aborting:
+            assert classify(p1) == "public"
+            assert classify(pback) == "public"
+            assert pback.h_revision & 1
+            assert p1.h_revision == int(ffi.cast("revision_t", pback))
+        else:
+            assert classify(p1) == "protected"
+            assert classify(pback) == "public"
+            assert classify(follow_revision(pback)) == "stub"
+            assert follow_revision(pback).h_revision == (
+                int(ffi.cast("revision_t", p1)) | 2)
 
     def f2(r):
         def cb(c):
@@ -286,3 +302,22 @@
         perform_transaction(cb)
 
     run_parallel(f1, f2)
+
+def test_abort_private_from_protected():
+    p = nalloc(HDR + WORD)
+    lib.setlong(p, 0, 897987)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    #
+    def cb(c):
+        assert classify(p) == "protected"
+        assert lib.getlong(p, 0) == 897987
+        if c == 0:
+            lib.setlong(p, 0, -38383)
+            assert lib.getlong(p, 0) == -38383
+            assert classify(p) == "private"
+            abort_and_retry()
+    perform_transaction(cb)
+
+def test_abort_stealing_while_modifying():
+    test_stealing_while_modifying(aborting=True)

From noreply at buildbot.pypy.org  Mon Jun 10 09:54:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Jun 2013 09:54:36 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Missing forward declaration
Message-ID: <20130610075436.848681C10DD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91:fc5246585adc
Date: 2013-06-10 09:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/fc5246585adc/

Log:	Missing forward declaration

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -487,6 +487,8 @@
                               int abort_reason, char *output);
 #endif
 
+void AbortPrivateFromProtected(struct tx_descriptor *d);
+
 void AbortTransaction(int num)
 {
   struct tx_descriptor *d = thread_descriptor;

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:38 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:38 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added some more primitives,
 some of the are stubbed
Message-ID: <20130610170038.0A6331C1068@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r436:7917efb6170d
Date: 2013-06-03 13:17 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/7917efb6170d/

Log:	added some more primitives, some of the are stubbed

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -67,7 +67,7 @@
                 else:
                     return result
             except error.PrimitiveFailedError:
-                IProxy.success_flag = False
+                IProxy.failed()
                 if mapping[result_type] is sqInt:
                     return 0
                 elif mapping[result_type] is sqDouble:
@@ -77,6 +77,7 @@
                 else:
                     raise NotImplementedError(
                         "InterpreterProxy: unknown result_type %s" % (result_type, ))
+        wrapped.func_name = "wrapped_ipf_" + func.func_name
         functions.append(("c_" + func.func_name, f_ptr, wrapped))
         return wrapped
     return decorator
@@ -132,10 +133,7 @@
 def stackFloatValue(offset):
     s_frame = IProxy.s_frame
     f = s_frame.peek(offset)
-    if isinstance(f, model.W_Float):
-        return f.value
-    else:
-        raise ProxyFunctionFailed
+    return IProxy.space.unwrap_float(f)
 
 @expose_on_virtual_machine_proxy([int], int)
 def stackIntegerValue(offset):
@@ -191,23 +189,96 @@
 def fetchClassOf(w_object):
     w_class = w_object.getclass(IProxy.space)
     return w_class
-#     sqInt  (*fetchClassOf)(sqInt oop);
-#     double (*fetchFloatofObject)(sqInt fieldIndex, sqInt objectPointer);
-#     sqInt  (*fetchIntegerofObject)(sqInt fieldIndex, sqInt objectPointer);
-#     sqInt  (*fetchPointerofObject)(sqInt fieldIndex, sqInt oop);
-#     sqInt  (*obsoleteDontUseThisFetchWordofObject)(sqInt fieldFieldIndex, sqInt oop);
-#     void  *(*firstFixedField)(sqInt oop);
-#     void  *(*firstIndexableField)(sqInt oop);
-#     sqInt  (*literalofMethod)(sqInt offset, sqInt methodPointer);
-#     sqInt  (*literalCountOf)(sqInt methodPointer);
-#     sqInt  (*methodArgumentCount)(void);
-#     sqInt  (*methodPrimitiveIndex)(void);
-#     sqInt  (*primitiveIndexOf)(sqInt methodPointer);
-#     sqInt  (*sizeOfSTArrayFromCPrimitive)(void *cPtr);
-#     sqInt  (*slotSizeOf)(sqInt oop);
-#     sqInt  (*stObjectat)(sqInt array, sqInt fieldIndex);
-#     sqInt  (*stObjectatput)(sqInt array, sqInt fieldIndex, sqInt value);
-#     sqInt  (*stSizeOf)(sqInt oop);
+
+ at expose_on_virtual_machine_proxy([int, oop], float)
+def fetchFloatofObject(fieldIndex, w_object):
+    space = IProxy.space
+    w_float = w_object.fetch(space, fieldIndex)
+    return space.unwrap_float(w_float)
+
+ at expose_on_virtual_machine_proxy([int, oop], int)
+def fetchIntegerofObject(fieldIndex, w_object):
+    space = IProxy.space
+    w_int = w_object.fetch(space, fieldIndex)
+    return space.unwrap_int(w_int)
+
+ at expose_on_virtual_machine_proxy([int, oop], oop)
+def fetchPointerofObject(fieldIndex, w_object):
+    return w_object.fetch(IProxy.space, fieldIndex)
+
+ at expose_on_virtual_machine_proxy([int, oop], int)
+def obsoleteDontUseThisFetchWordofObject(fieldIndex, w_object):
+    # XXX: correctness?
+    space = IProxy.space
+    w_int = w_object.fetch(space, fieldIndex)
+    return space.unwrap_uint(w_int)
+
+ at expose_on_virtual_machine_proxy([oop], list)
+def firstFixedField(w_object):
+    # return a list with oops (?) of w_objects instVars
+    raise NotImplementedError
+
+ at expose_on_virtual_machine_proxy([oop], list)
+def firstIndexableField(w_object):
+    # return a list with values (?) of w_objects variable-parts
+    raise NotImplementedError
+
+ at expose_on_virtual_machine_proxy([int, oop], oop)
+def literalofMethod(offset, w_method):
+    if isinstance(w_method, model.W_CompiledMethod):
+        return w_method.literalat0(offset)
+    else:
+        raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], int)
+def literalCountOf(w_method):
+    if isinstance(w_method, model.W_CompiledMethod):
+        return w_method.getliteralsize()
+    else:
+        raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], int)
+def methodArgumentCount():
+    return IProxy.argcount
+
+ at expose_on_virtual_machine_proxy([], int)
+def methodPrimitiveIndex():
+    return IProxy.s_method.primitive()
+
+ at expose_on_virtual_machine_proxy([oop], int)
+def primitiveIndexOf(w_method):
+    if isinstance(w_method, model.W_CompiledMethod):
+        return w_method.as_compiledmethod_get_shadow().primitive()
+    else:
+        raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([list], int)
+def sizeOfSTArrayFromCPrimitive(c_array):
+    raise NotImplementedError
+
+ at expose_on_virtual_machine_proxy([oop], int)
+def slotSizeOf(w_object):
+    return w_object.size()
+
+ at expose_on_virtual_machine_proxy([oop, int], oop)
+def stObjectat(w_object, n0):
+    from spyvm.primitives import assert_valid_index
+    space = IProxy.space
+    n0 = assert_valid_index(space, n0, w_object)
+    return w_object.at0(space, n0)
+
+ at expose_on_virtual_machine_proxy([oop, int, oop], int)
+def stObjectatput(w_object, n0, w_value):
+    from spyvm.primitives import assert_valid_index
+    space = IProxy.space
+    n0 = assert_valid_index(space, n0, w_object)
+    w_object.atput0(space, n0, w_value)
+    return 0 # XXX: check return value
+
+ at expose_on_virtual_machine_proxy([oop], int)
+def stSizeOf(w_object):
+    return w_object.primsize(IProxy.space)
+
 #     sqInt  (*storeIntegerofObjectwithValue)(sqInt fieldIndex, sqInt oop, sqInt integer);
 #     sqInt  (*storePointerofObjectwithValue)(sqInt fieldIndex, sqInt oop, sqInt valuePointer);
 
@@ -259,8 +330,21 @@
 
 #     /* InterpreterProxy methodsFor: 'instance creation' */
 
-#     sqInt (*clone)(sqInt oop);
-#     sqInt (*instantiateClassindexableSize)(sqInt classPointer, sqInt size);
+ at expose_on_virtual_machine_proxy([oop], oop)
+def clone(w_object):
+    return w_object.clone(IProxy.space)
+
+ at expose_on_virtual_machine_proxy([oop, int], oop)
+def instantiateClassindexableSize(w_class, varsize):
+    s_class = w_class.as_class_get_shadow(IProxy.space)
+    return s_class.new(varsize)
+
+# @expose_on_virtual_machine_proxy([int, int], oop)
+# def makePointwithxValueyValue(x, y):
+#     space = IProxy.space
+#     w_x = space.wrap_int(x)
+#     w_y = space.wrap_int(y)
+
 #     sqInt (*makePointwithxValueyValue)(sqInt xValue, sqInt yValue);
 #     sqInt (*popRemappableOop)(void);
 #     sqInt (*pushRemappableOop)(sqInt oop);
@@ -271,13 +355,36 @@
 #     sqInt (*byteSwapped)(sqInt w);
 #     sqInt (*failed)(void);
 #     sqInt (*fullDisplayUpdate)(void);
-#     sqInt (*fullGC)(void);
-#     sqInt (*incrementalGC)(void);
-#     sqInt (*primitiveFail)(void);
+ at expose_on_virtual_machine_proxy([], int)
+def fullGC():
+    # XXX: how to invoke gc?
+    return 0
+ at expose_on_virtual_machine_proxy([], int)
+def incrementalGC():
+    # XXX: how to invoke gc?
+    return 0
+
+ at expose_on_virtual_machine_proxy([], int)
+def primitiveFail():
+    raise ProxyFunctionFailed
+
 #     sqInt (*showDisplayBitsLeftTopRightBottom)(sqInt aForm, sqInt l, sqInt t, sqInt r, sqInt b);
 #     sqInt (*signalSemaphoreWithIndex)(sqInt semaIndex);
-#     sqInt (*success)(sqInt aBoolean);
-#     sqInt (*superclassOf)(sqInt classPointer);
+
+ at expose_on_virtual_machine_proxy([bool], int)
+def success(aBoolean):
+    if aBoolean:
+        return 0
+    else:
+        raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], oop)
+def superclassOf(w_class):
+    s_superclass = w_class.as_class_get_shadow(IProxy.space).s_superclass()
+    if s_superclass is not None:
+        return s_superclass.w_self()
+    else:
+        return IProxy.space.w_nil
 
 #     /* InterpreterProxy methodsFor: 'compiler' */
 
@@ -471,6 +578,7 @@
         self.argcount = 0
         self.s_method = None
         self.success_flag = True
+        self.fail_reason = 0
 
     def call(self, signature, interp, s_frame, argcount, s_method):
         self.interp = interp
@@ -481,6 +589,8 @@
         try:
             print "Hello World..."
             raise error.Exit("External Call")
+            if not self.success_flag:
+                raise error.PrimitiveFailedError
         finally:
             self.reset()
 
@@ -490,4 +600,8 @@
     def object_to_oop(self, oop):
         return 0
 
+    def failed(self, reason=1):
+        self.success_flag = False
+        self.fail_reason = reason
+
 IProxy = _InterpreterProxy()

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:43 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:43 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added storing into object to
	interpreter proxy
Message-ID: <20130610170043.849171C1380@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r441:ca33279eb6d4
Date: 2013-06-10 18:56 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/ca33279eb6d4/

Log:	added storing into object to interpreter proxy

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -280,8 +280,23 @@
 def stSizeOf(w_object):
     return w_object.primsize(IProxy.space)
 
-#     sqInt  (*storeIntegerofObjectwithValue)(sqInt fieldIndex, sqInt oop, sqInt integer);
-#     sqInt  (*storePointerofObjectwithValue)(sqInt fieldIndex, sqInt oop, sqInt valuePointer);
+ at expose_on_virtual_machine_proxy([int, oop, int], oop)
+def storeIntegerofObjectwithValue(n0, w_object, a):
+    if w_object.size() > n0:
+        space = IProxy.space
+        w_object.store(space, n0, space.wrap_int(a))
+        return space.wrap_int(a)
+    else:
+        raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, oop, oop], oop)
+def storePointerofObjectwithValue(n0, w_object, w_value):
+    if w_object.size() > n0:
+        w_object.store(IProxy.space, n0, w_value)
+        return w_value
+    else:
+        IProxy.failed()
+        return w_value
 
 #     /* InterpreterProxy methodsFor: 'testing' */
 

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:39 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:39 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed some type-issues by
 branching in interpreter proxy
Message-ID: <20130610170039.276EF1C10DD@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r437:d94a8bfa757f
Date: 2013-06-10 13:46 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/d94a8bfa757f/

Log:	fixed some type-issues by branching in interpreter proxy added maps
	for oop-to-object mapping

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -226,7 +226,7 @@
 @expose_on_virtual_machine_proxy([int, oop], oop)
 def literalofMethod(offset, w_method):
     if isinstance(w_method, model.W_CompiledMethod):
-        return w_method.literalat0(offset)
+        return w_method.literalat0(IProxy.space, offset)
     else:
         raise ProxyFunctionFailed
 
@@ -267,13 +267,13 @@
     n0 = assert_valid_index(space, n0, w_object)
     return w_object.at0(space, n0)
 
- at expose_on_virtual_machine_proxy([oop, int, oop], int)
+ at expose_on_virtual_machine_proxy([oop, int, oop], oop)
 def stObjectatput(w_object, n0, w_value):
     from spyvm.primitives import assert_valid_index
     space = IProxy.space
     n0 = assert_valid_index(space, n0, w_object)
     w_object.atput0(space, n0, w_value)
-    return 0 # XXX: check return value
+    return w_value
 
 @expose_on_virtual_machine_proxy([oop], int)
 def stSizeOf(w_object):
@@ -336,6 +336,8 @@
 
 @expose_on_virtual_machine_proxy([oop, int], oop)
 def instantiateClassindexableSize(w_class, varsize):
+    if not isinstance(w_class, model.W_PointersObject):
+        raise error.PrimitiveFailedError
     s_class = w_class.as_class_get_shadow(IProxy.space)
     return s_class.new(varsize)
 
@@ -380,6 +382,8 @@
 
 @expose_on_virtual_machine_proxy([oop], oop)
 def superclassOf(w_class):
+    if not isinstance(w_class, model.W_PointersObject):
+        raise error.PrimitiveFailedError
     s_superclass = w_class.as_class_get_shadow(IProxy.space).s_superclass()
     if s_superclass is not None:
         return s_superclass.w_self()
@@ -570,6 +574,9 @@
     def __init__(self):
         self.vm_proxy = lltype.nullptr(VMPtr.TO)
         self.vm_initialized = False
+        self._next_oop = 0
+        self.oop_map = {}
+        self.object_map = {}
         self.reset()
 
     def reset(self):
@@ -587,18 +594,33 @@
         self.s_method = s_method
         self.space = interp.space
         try:
-            print "Hello World..."
-            raise error.Exit("External Call")
+            # Load the correct DLL
+            self.success_flag = False
+            # call the correct function in it...
             if not self.success_flag:
                 raise error.PrimitiveFailedError
         finally:
             self.reset()
 
     def oop_to_object(self, oop):
-        return self.interp.space.w_nil
+        try:
+            return self.oop_map[oop]
+        except KeyError:
+            raise ProxyFunctionFailed
 
-    def object_to_oop(self, oop):
-        return 0
+    def object_to_oop(self, w_object):
+        try:
+            return self.object_map[w_object]
+        except KeyError:
+            new_index = self.next_oop()
+            self.oop_map[new_index] = w_object
+            self.object_map[w_object] = new_index
+            return new_index
+
+    def next_oop(self):
+        next_oop = self._next_oop
+        self._next_oop = next_oop + 1
+        return next_oop
 
     def failed(self, reason=1):
         self.success_flag = False

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:45 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:45 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added misc category functions
Message-ID: <20130610170045.AAD891C13E5@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r442:127ac7fc16ff
Date: 2013-06-10 18:57 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/127ac7fc16ff/

Log:	added misc category functions

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -473,15 +473,21 @@
     s_class = w_class.as_class_get_shadow(IProxy.space)
     return s_class.new(varsize)
 
-# @expose_on_virtual_machine_proxy([int, int], oop)
-# def makePointwithxValueyValue(x, y):
-#     space = IProxy.space
-#     w_x = space.wrap_int(x)
-#     w_y = space.wrap_int(y)
+ at expose_on_virtual_machine_proxy([int, int], oop)
+def makePointwithxValueyValue(x, y):
+    space = IProxy.space
+    w_point = space.w_Point.as_class_get_shadow(space).new()
+    w_point.store(space, 0, space.wrap_int(x))
+    w_point.store(space, 1, space.wrap_int(y))
+    return w_point
 
-#     sqInt (*makePointwithxValueyValue)(sqInt xValue, sqInt yValue);
-#     sqInt (*popRemappableOop)(void);
-#     sqInt (*pushRemappableOop)(sqInt oop);
+ at expose_on_virtual_machine_proxy([], oop)
+def popRemappableOop():
+    return IProxy.pop_remappable()
+
+ at expose_on_virtual_machine_proxy([oop], oop)
+def pushRemappableOop(w_object):
+    return IProxy.push_remappable(w_object)
 
 #     /* InterpreterProxy methodsFor: 'other' */
 
@@ -709,6 +715,7 @@
         self._next_oop = 0
         self.oop_map = {}
         self.object_map = {}
+        self.remappable_objects = []
         self.reset()
 
     def reset(self):
@@ -716,7 +723,6 @@
         self.s_frame = None
         self.argcount = 0
         self.s_method = None
-        self.success_flag = True
         self.fail_reason = 0
 
     def call(self, signature, interp, s_frame, argcount, s_method):
@@ -725,15 +731,20 @@
         self.argcount = argcount
         self.s_method = s_method
         self.space = interp.space
+        # ensure that space.w_nil gets the first possible oop
+        self.object_to_oop(self.space.w_nil)
         try:
             # Load the correct DLL
-            self.success_flag = False
+            self.failed()
             # call the correct function in it...
-            if not self.success_flag:
+            if not self.fail_reason == 0:
                 raise error.PrimitiveFailedError
         finally:
             self.reset()
 
+    def failed(self, reason=1):
+        self.fail_reason = reason
+
     def oop_to_object(self, oop):
         try:
             return self.oop_map[oop]
@@ -754,8 +765,15 @@
         self._next_oop = next_oop + 1
         return next_oop
 
-    def failed(self, reason=1):
-        self.success_flag = False
-        self.fail_reason = reason
+    def pop_remappable(self):
+        try:
+            return self.remappable_objects.pop()
+        except IndexError:
+            self.failed()
+            return self.space.w_nil
+
+    def push_remappable(self, w_object):
+        self.remappable_objects.append(w_object)
+        return w_object
 
 IProxy = _InterpreterProxy()

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:40 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:40 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added constant returning
 functions to interpreter-proxy
Message-ID: <20130610170040.3AA131C1190@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r438:a5e3526739ba
Date: 2013-06-10 18:53 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/a5e3526739ba/

Log:	added constant returning functions to interpreter-proxy added
	bitmap-class to constants

diff --git a/spyvm/constants.py b/spyvm/constants.py
--- a/spyvm/constants.py
+++ b/spyvm/constants.py
@@ -107,7 +107,7 @@
 
 # XXX more missing?
 classes_in_special_object_table = {
-#    "Bitmap" : SO_BITMAP_CLASS,
+    "Bitmap" : SO_BITMAP_CLASS,
     "SmallInteger" : SO_SMALLINTEGER_CLASS,
     "String" : SO_STRING_CLASS,
     "Array" : SO_ARRAY_CLASS,
diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -309,24 +309,71 @@
 
 #     /* InterpreterProxy methodsFor: 'special objects' */
 
-#     sqInt (*characterTable)(void);
-#     sqInt (*displayObject)(void);
-#     sqInt (*falseObject)(void);
-#     sqInt (*nilObject)(void);
-#     sqInt (*trueObject)(void);
+ at expose_on_virtual_machine_proxy([], oop)
+def characterTable():
+    return IProxy.space.w_charactertable
+
+ at expose_on_virtual_machine_proxy([], oop)
+def displayObject():
+    return IProxy.space.objtable['w_display']
+
+ at expose_on_virtual_machine_proxy([], oop)
+def falseObject():
+    return IProxy.space.w_false
+
+ at expose_on_virtual_machine_proxy([], oop)
+def nilObject():
+    return IProxy.space.w_nil
+
+ at expose_on_virtual_machine_proxy([], oop)
+def trueObject():
+    return IProxy.space.w_true
 
 #     /* InterpreterProxy methodsFor: 'special classes' */
 
-#     sqInt (*classArray)(void);
-#     sqInt (*classBitmap)(void);
-#     sqInt (*classByteArray)(void);
-#     sqInt (*classCharacter)(void);
-#     sqInt (*classFloat)(void);
-#     sqInt (*classLargePositiveInteger)(void);
-#     sqInt (*classPoint)(void);
-#     sqInt (*classSemaphore)(void);
-#     sqInt (*classSmallInteger)(void);
-#     sqInt (*classString)(void);
+# Can't generate these, because the IProxy-field-access can only be done after
+# first "call"-call
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classArray():
+    return IProxy.space.w_Array
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classBitmap():
+    return IProxy.space.w_Bitmap
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classByteArray():
+    return IProxy.space.w_ByteArray
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classCharacter():
+    return IProxy.space.w_Character
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classFloat():
+    return IProxy.space.w_Float
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classLargePositiveInteger():
+    return IProxy.space.w_LargePositiveInteger
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classPoint():
+    return IProxy.space.w_Point
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classSemaphore():
+    return IProxy.space.w_Semaphore
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classSmallInteger():
+    return IProxy.space.w_SmallInteger
+
+ at expose_on_virtual_machine_proxy([], oop)
+def classString():
+    return IProxy.space.w_String
+
 
 #     /* InterpreterProxy methodsFor: 'instance creation' */
 
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -38,7 +38,8 @@
 
     def instsize(self, space):
         """Return the size of the object reserved for instance variables.
-        Only returns something non-zero for W_PointersObjects"""
+        Only returns something non-zero for W_PointersObjects, W_Floats, and
+        W_LargePositiveInteger1Words"""
         return 0
 
     def varsize(self, space):

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:46 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:46 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added remaining other
	functions
Message-ID: <20130610170046.E443D1C1068@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r443:2a52dc328e17
Date: 2013-06-10 18:58 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/2a52dc328e17/

Log:	added remaining other functions

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -491,10 +491,30 @@
 
 #     /* InterpreterProxy methodsFor: 'other' */
 
-#     sqInt (*becomewith)(sqInt array1, sqInt array2);
-#     sqInt (*byteSwapped)(sqInt w);
-#     sqInt (*failed)(void);
-#     sqInt (*fullDisplayUpdate)(void);
+ at expose_on_virtual_machine_proxy([list, list], int)
+def becomewith(w_array1, w_array2):
+    # XXX: stub, until used
+    print "InterpreterProxy >> becomewith(list, list)"
+    return 0
+
+ at expose_on_virtual_machine_proxy([int], int)
+def byteSwapped(w):
+    from rpython.rlib.rarithmetic import intmask
+    return (w >> 24) & 0xFF + (w >> 8) & 0xFF00 + (w << 8) & 0xFF0000 + (w << 24) & -16777216
+
+ at expose_on_virtual_machine_proxy([], bool)
+def failed():
+    return not IProxy.fail_reason == 0
+
+ at expose_on_virtual_machine_proxy([], int)
+def fullDisplayUpdate():
+    w_display = IProxy.space.objtable['w_display']
+    if isinstance(w_display, model.W_DisplayBitmap):
+        w_display.flush_to_screen()
+        return 0
+    else:
+        raise ProxyFunctionFailed
+
 @expose_on_virtual_machine_proxy([], int)
 def fullGC():
     # XXX: how to invoke gc?

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:41 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:41 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added testing methods,
	some of the stubbed
Message-ID: <20130610170041.551CA1C11A6@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r439:d6e12befe317
Date: 2013-06-10 18:54 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/d6e12befe317/

Log:	added testing methods, some of the stubbed

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -23,6 +23,7 @@
 sqLong = rffi.LONG
 sqDouble = rffi.DOUBLE
 sqIntArrayPtr = Ptr(rffi.CArray(sqInt))
+sqStr = rffi.CCHARP
 
 major = minor = 0
 functions = []
@@ -33,7 +34,7 @@
     pass
 
 def expose_on_virtual_machine_proxy(unwrap_spec, result_type, minor=0, major=1):
-    mapping = {oop: sqInt, int: sqInt, list: sqIntArrayPtr, bool: sqInt, float: sqDouble}
+    mapping = {oop: sqInt, int: sqInt, list: sqIntArrayPtr, bool: sqInt, float: sqDouble, str: sqStr}
     f_ptr = Ptr(FuncType([mapping[spec] for spec in unwrap_spec], mapping[result_type]))
     if minor < minor:
         minor = minor
@@ -284,17 +285,58 @@
 
 #     /* InterpreterProxy methodsFor: 'testing' */
 
+ at expose_on_virtual_machine_proxy([oop, str], bool)
+def isKindOf(w_object, name):
+    # XXX: stub, until used
+    print "InterpreterProxy >> isKindOf(object, name)"
+    return False
 #     sqInt (*isKindOf)(sqInt oop, char *aString);
+
+ at expose_on_virtual_machine_proxy([oop, str], bool)
+def isMemberOf(w_object, name):
+    # XXX: stub, until used
+    print "InterpreterProxy >> isMemberOf(object, name)"
+    return False
 #     sqInt (*isMemberOf)(sqInt oop, char *aString);
-#     sqInt (*isBytes)(sqInt oop);
-#     sqInt (*isFloatObject)(sqInt oop);
-#     sqInt (*isIndexable)(sqInt oop);
-#     sqInt (*isIntegerObject)(sqInt objectPointer);
-#     sqInt (*isIntegerValue)(sqInt intValue);
-#     sqInt (*isPointers)(sqInt oop);
-#     sqInt (*isWeak)(sqInt oop);
-#     sqInt (*isWords)(sqInt oop);
-#     sqInt (*isWordsOrBytes)(sqInt oop);
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isBytes(w_object):
+    return isinstance(w_object, model.W_BytesObject)
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isFloatObject(w_object):
+    return isinstance(w_object, model.W_Float)
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isIndexable(w_object):
+    space = IProxy.space
+    return w_object.getclass(space).as_class_get_shadow(space).isvariable()
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isIntegerObject(w_object):
+    return isinstance(w_object, model.W_SmallInteger)
+
+ at expose_on_virtual_machine_proxy([int], bool)
+def isIntegerValue(n):
+    """Checking whether the two highest bits are equal,
+    which means that the value is representable as 31/63-bit value."""
+    return n ^ (n << 1) >= 0
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isPointers(w_object):
+    return isinstance(w_object, model.W_PointersObject)
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isWeak(w_object):
+    return isinstance(w_object, model.W_WeakPointersObject)
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isWords(w_object):
+    return w_object.is_array_object() and not isinstance(w_object, model.W_BytesObject)
+
+ at expose_on_virtual_machine_proxy([oop], bool)
+def isWordsOrBytes(w_object):
+    return w_object.is_array_object()
 
 #     /* InterpreterProxy methodsFor: 'converting' */
 

From noreply at buildbot.pypy.org  Mon Jun 10 19:00:42 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:00:42 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added converting methods to
	interpreter proxy
Message-ID: <20130610170042.7540A1C136D@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r440:c00e555fc976
Date: 2013-06-10 18:55 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/c00e555fc976/

Log:	added converting methods to interpreter proxy

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -340,14 +340,42 @@
 
 #     /* InterpreterProxy methodsFor: 'converting' */
 
-#     sqInt  (*booleanValueOf)(sqInt obj);
-#     sqInt  (*checkedIntegerValueOf)(sqInt intOop);
-#     sqInt  (*floatObjectOf)(double aFloat);
-#     double (*floatValueOf)(sqInt oop);
-#     sqInt  (*integerObjectOf)(sqInt value);
-#     sqInt  (*integerValueOf)(sqInt oop);
-#     sqInt  (*positive32BitIntegerFor)(sqInt integerValue);
-#     sqInt  (*positive32BitValueOf)(sqInt oop);
+ at expose_on_virtual_machine_proxy([oop], bool)
+def booleanValueOf(w_object):
+    space = IProxy.space
+    if w_object is space.w_true:
+        return True
+    if w_object is space.w_false:
+        return False
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], int)
+def checkedIntegerValueOf(w_object):
+    return IProxy.space.unwrap_int(w_object)
+
+ at expose_on_virtual_machine_proxy([float], oop)
+def floatObjectOf(f):
+    return IProxy.space.wrap_float(f)
+
+ at expose_on_virtual_machine_proxy([oop], float)
+def floatValueOf(w_object):
+    return IProxy.space.unwrap_float(w_object)
+
+ at expose_on_virtual_machine_proxy([int], oop)
+def integerObjectOf(n):
+    return IProxy.space.wrap_int(n)
+
+ at expose_on_virtual_machine_proxy([oop], int)
+def integerValueOf(w_object):
+    return IProxy.space.unwrap_int(w_object)
+
+ at expose_on_virtual_machine_proxy([int], oop)
+def positive32BitIntegerFor(n):
+    return IProxy.space.wrap_positive_32bit_int(n)
+
+ at expose_on_virtual_machine_proxy([oop], int)
+def positive32BitValueOf(n):
+    return IProxy.space.unwrap_positive_32bit_int(n)
 
 #     /* InterpreterProxy methodsFor: 'special objects' */
 

From noreply at buildbot.pypy.org  Mon Jun 10 19:21:28 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Jun 2013 19:21:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill this hack. It's a Very Bad idea to
 do it this way: it means
Message-ID: <20130610172128.B75D81C02C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64844:123cde0fe60f
Date: 2013-06-10 19:16 +0200
http://bitbucket.org/pypy/pypy/changeset/123cde0fe60f/

Log:	Kill this hack. It's a Very Bad idea to do it this way: it means
	translation will fail on any user-provided example too :-((

diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
@@ -2,7 +2,6 @@
 import weakref
 from rpython.rlib.jit import JitDriver, dont_look_inside
 from rpython.jit.backend.llsupport.test.zrpy_gc_test import run, get_entry, compile
-from rpython.jit.backend.llsupport.test.ztranslation_test import fix_annotator_for_vrawbuffer
 
 class X(object):
     def __init__(self, x=0):
@@ -32,8 +31,7 @@
     g._dont_inline_ = True
     return g
 
-def compile_boehm_test(monkeypatch):
-    fix_annotator_for_vrawbuffer(monkeypatch)
+def compile_boehm_test():
     myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
     @dont_look_inside
     def see(lst, n):
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -9,23 +9,10 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 
 
-def fix_annotator_for_vrawbuffer(monkeypatch):
-    from rpython.rlib.nonconst import NonConstant
-    from rpython.jit.metainterp.optimizeopt.virtualize import VRawBufferValue
-    from rpython.jit.metainterp import warmspot
-
-    def my_hook_for_tests(cpu):
-        # this is needed so that the annotator can see it
-        if NonConstant(False):
-            v = VRawBufferValue(cpu, None, -1, None, None)
-    monkeypatch.setattr(warmspot, 'hook_for_tests', my_hook_for_tests)
-
-
 class TranslationTest(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_stuff_translates(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_stuff_translates(self):
         # this is a basic test that tries to hit a number of features and their
         # translation:
         # - jitting of loops and bridges
@@ -102,10 +89,9 @@
 class TranslationTestCallAssembler(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_direct_assembler_call_translates(self, monkeypatch):
+    def test_direct_assembler_call_translates(self):
         """Test CALL_ASSEMBLER and the recursion limit"""
         from rpython.rlib.rstackovf import StackOverflow
-        fix_annotator_for_vrawbuffer(monkeypatch)
 
         class Thing(object):
             def __init__(self, val):
@@ -183,8 +169,7 @@
 class TranslationTestJITStats(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_jit_get_stats(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_jit_get_stats(self):
         driver = JitDriver(greens = [], reds = ['i'])
 
         def f():
@@ -207,8 +192,7 @@
 class TranslationRemoveTypePtrTest(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_external_exception_handling_translates(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_external_exception_handling_translates(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'total'])
 
         class ImDone(Exception):
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -761,8 +761,6 @@
         cpu = jd.warmstate.cpu
 
         def ll_portal_runner(*args):
-            hook_for_tests(cpu) # usually it's empty, but tests can monkeypatch
-                                # it to fix the annotator
             start = True
             while 1:
                 try:
@@ -999,10 +997,3 @@
         graphs = self.translator.graphs
         for graph, block, i in find_force_quasi_immutable(graphs):
             self.replace_force_quasiimmut_with_direct_call(block.operations[i])
-
-def hook_for_tests(cpu):
-    """
-    This function is empty and does nothing. Its only role is to be
-    monkey-patched by tests to "fix" the annotator if needed (see
-    e.g. x86/test/test_ztranslation::test_external_exception_handling_translates
-    """

From noreply at buildbot.pypy.org  Mon Jun 10 19:29:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Jun 2013 19:29:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for 123cde0fe60f
Message-ID: <20130610172926.9E32A1C1026@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64845:48357e1d63e1
Date: 2013-06-10 19:28 +0200
http://bitbucket.org/pypy/pypy/changeset/48357e1d63e1/

Log:	fix for 123cde0fe60f

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -17,6 +17,7 @@
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
+    is_about_raw = False
     _cached_vinfo = None
 
     def __init__(self, keybox, source_op=None):
@@ -395,6 +396,7 @@
 
 
 class VRawBufferValue(AbstractVArrayValue):
+    is_about_raw = True
 
     def __init__(self, cpu, logops, size, keybox, source_op):
         AbstractVirtualValue.__init__(self, keybox, source_op)
@@ -457,6 +459,7 @@
 
 
 class VRawSliceValue(AbstractVirtualValue):
+    is_about_raw = True
 
     def __init__(self, rawbuffer_value, offset, keybox, source_op):
         AbstractVirtualValue.__init__(self, keybox, source_op)
@@ -676,13 +679,17 @@
         offsetbox = self.get_constant_box(op.getarg(1))
         if value.is_virtual() and offsetbox is not None:
             offset = offsetbox.getint()
-            if isinstance(value, VRawBufferValue):
-                self.make_virtual_raw_slice(value, offset, op.result, op)
-                return
-            elif isinstance(value, VRawSliceValue):
-                offset = offset + value.offset
-                self.make_virtual_raw_slice(value.rawbuffer_value, offset, op.result, op)
-                return
+            # the following check is constant-folded to False if the
+            # translation occurs without any VRawXxxValue instance around
+            if value.is_about_raw:
+                if isinstance(value, VRawBufferValue):
+                    self.make_virtual_raw_slice(value, offset, op.result, op)
+                    return
+                elif isinstance(value, VRawSliceValue):
+                    offset = offset + value.offset
+                    self.make_virtual_raw_slice(value.rawbuffer_value, offset,
+                                                op.result, op)
+                    return
         self.emit_operation(op)
 
     def optimize_ARRAYLEN_GC(self, op):
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -451,6 +451,7 @@
 
 class AbstractVirtualInfo(object):
     kind = REF
+    is_about_raw = False
     #def allocate(self, decoder, index):
     #    raise NotImplementedError
     def equals(self, fieldnums):
@@ -461,7 +462,7 @@
 
     def debug_prints(self):
         raise NotImplementedError
-        
+
 
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
     def __init__(self, fielddescrs):
@@ -547,6 +548,7 @@
 
 class VRawBufferStateInfo(AbstractVirtualInfo):
     kind = INT
+    is_about_raw = True
     
     def __init__(self, size, offsets, descrs):
         self.size = size
@@ -772,7 +774,9 @@
         assert self.virtuals_cache is not None
         v = self.virtuals_cache.get_int(index)
         if not v:
-            v = self.rd_virtuals[index].allocate_int(self, index)
+            v = self.rd_virtuals[index]
+            assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
+            v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")
         return v
 

From noreply at buildbot.pypy.org  Mon Jun 10 19:30:58 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:30:58 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: stubbed some more functions
 of interpreterproxy:
Message-ID: <20130610173058.072551C1026@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r444:839dc50737a1
Date: 2013-06-10 19:29 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/839dc50737a1/

Log:	stubbed some more functions of interpreterproxy: compiler-
	hooks and bitblt-calls

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -528,8 +528,16 @@
 def primitiveFail():
     raise ProxyFunctionFailed
 
-#     sqInt (*showDisplayBitsLeftTopRightBottom)(sqInt aForm, sqInt l, sqInt t, sqInt r, sqInt b);
-#     sqInt (*signalSemaphoreWithIndex)(sqInt semaIndex);
+ at expose_on_virtual_machine_proxy([oop, int, int, int, int], int)
+def showDisplayBitsLeftTopRightBottom(w_form, l, t, r, b):
+    print 'Called InterpreterProxy >> showDisplayBitsLeftTopRightBottom'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], int)
+def signalSemaphoreWithIndex(n):
+    # ((Smalltalk externalObjects) at: n) signal
+    print 'Called InterpreterProxy >> signalSemaphoreWithIndex'
+    raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([bool], int)
 def success(aBoolean):
@@ -550,18 +558,36 @@
 
 #     /* InterpreterProxy methodsFor: 'compiler' */
 
-#     CompilerHook *(*compilerHookVector)(void);
-#     sqInt          (*setCompilerInitialized)(sqInt initFlag);
 
-# #if VM_PROXY_MINOR > 1
+ at expose_on_virtual_machine_proxy([], int)
+def compilerHookVector():
+    print 'Called InterpreterProxy >> compilerHookVector'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], int)
+def setCompilerInitialized(n):
+    print 'Called InterpreterProxy >> setCompilerInitialized'
+    raise ProxyFunctionFailed
 
 #     /* InterpreterProxy methodsFor: 'BitBlt support' */
 
-#     sqInt (*loadBitBltFrom)(sqInt bbOop);
-#     sqInt (*copyBits)(void);
-#     sqInt (*copyBitsFromtoat)(sqInt leftX, sqInt rightX, sqInt yValue);
+ at expose_on_virtual_machine_proxy([int], int, minor=1)
+def loadBitBltFrom(w_bitBlit):
+    # bb := bbOop
+    print 'Called InterpreterProxy >> loadBitBltFrom'
+    raise ProxyFunctionFailed
 
-# #endif
+ at expose_on_virtual_machine_proxy([], int, minor=1)
+def copyBits():
+    # bb copyBits
+    print 'Called InterpreterProxy >> copyBits'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, int, int], int, minor=1)
+def copyBitsFromtoat(x0, x1, y):
+    # bb copyBitsFrom: x0 to: x1 at: y
+    print 'Called InterpreterProxy >> copyBitsFromtoat'
+    raise ProxyFunctionFailed
 
 # #if VM_PROXY_MINOR > 2
 

From noreply at buildbot.pypy.org  Mon Jun 10 19:30:59 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Mon, 10 Jun 2013 19:30:59 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added prebuild class "Bitmap"
 needed for assignment from image,
 the interpreter proxy and (maybe eventually) bitblt
Message-ID: <20130610173059.198A01C1026@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r445:ab55fd796e76
Date: 2013-06-10 19:30 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/ab55fd796e76/

Log:	added prebuild class "Bitmap" needed for assignment from image, the
	interpreter proxy and (maybe eventually) bitblt

diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -94,6 +94,7 @@
         define_cls("w_ArrayedCollection", "w_SequenceableCollection")
         define_cls("w_Array", "w_ArrayedCollection", varsized=True)
         define_cls("w_String", "w_ArrayedCollection", format=shadow.BYTES)
+        define_cls("w_Bitmap", "w_ArrayedCollection", varsized=True, format=shadow.WORDS)
         define_cls("w_UndefinedObject", "w_Object")
         define_cls("w_Boolean", "w_Object")
         define_cls("w_True", "w_Boolean")

From noreply at buildbot.pypy.org  Tue Jun 11 12:43:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 12:43:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill unused method
Message-ID: <20130611104354.1F7D21C0328@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64846:35e9a41d7bdd
Date: 2013-06-11 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/35e9a41d7bdd/

Log:	Kill unused method

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -103,11 +103,6 @@
         'flags'                 : 'app.null_sysflags',
     }
 
-    def setbuiltinmodule(self, w_module, name):
-        w_name = self.space.wrap(name)
-        w_modules = self.get('modules')
-        self.space.setitem(w_modules, w_name, w_module)
-
     def startup(self, space):
         if space.config.translating and not we_are_translated():
             # don't get the filesystemencoding at translation time

From noreply at buildbot.pypy.org  Tue Jun 11 12:43:55 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 12:43:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #1514
Message-ID: <20130611104355.6D8C41C0328@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64847:92546f437072
Date: 2013-06-11 12:40 +0200
http://bitbucket.org/pypy/pypy/changeset/92546f437072/

Log:	Issue #1514 A test for reimporting built-in modules (as opposed to
	reloading them).

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -565,6 +565,22 @@
         assert sys.path is oldpath
         assert 'setdefaultencoding' in dir(sys)
 
+    def test_reimport_builtin(self):
+        # ...but not reload()!
+        import sys
+        oldpath = sys.path
+        sys.setdefaultencoding = "<test_reimport_builtin removed this>"
+
+        del sys.modules['sys']
+        import sys as sys1
+        assert sys.modules['sys'] is sys1 is sys
+
+        assert sys.path is oldpath
+        assert sys.setdefaultencoding == "<test_reimport_builtin removed this>"
+
+        reload(sys)   # fix it for people that want 'setdefaultencoding'
+        assert sys.setdefaultencoding != "<test_reimport_builtin removed this>"
+
     def test_reload_infinite(self):
         import infinite_reload
 

From noreply at buildbot.pypy.org  Tue Jun 11 12:43:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 12:43:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for 92546f437072.
Message-ID: <20130611104356.9BA861C0328@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64848:f26956c61773
Date: 2013-06-11 12:41 +0200
http://bitbucket.org/pypy/pypy/changeset/f26956c61773/

Log:	Fix for 92546f437072.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -381,8 +381,10 @@
         except AttributeError:
             return self.__class__.__name__
 
-    def setbuiltinmodule(self, importname):
-        """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules"""
+    def _prepare_mixedmodule(self, importname):
+        """NOT_RPYTHON.  Load the RPython code of a lazy pypy/module and put
+        it into 'self.builtin_modules', without any further initialization.
+        """
         if '.' in importname:
             fullname = importname
             importname = fullname.rsplit('.', 1)[1]
@@ -401,20 +403,22 @@
 
         return name
 
-    def getbuiltinmodule(self, name, force_init=False):
-        w_name = self.wrap(name)
-        w_modules = self.sys.get('modules')
-        try:
-            w_mod = self.getitem(w_modules, w_name)
-        except OperationError, e:
-            if not e.match(self, self.w_KeyError):
-                raise
-        else:
-            if not force_init:
-                return w_mod
+    @jit.elidable
+    def getbuiltinmodule(self, name):
+        """Return the built-in module 'name'.  The first time it is seen,
+        it is initialized and stored in 'sys.modules'.  This function is
+        elidable by the JIT because its effect is idempotent (if you call
+        it twice with the same name, you're getting the same effect as if
+        it was only called once).
+        """
+        return self.loadbuiltinmodule(name, force_in_sys_modules=False,
+                                      force_init=False)
 
-        # If the module is a builtin but not yet imported,
-        # retrieve it and initialize it
+    def loadbuiltinmodule(self, name, force_in_sys_modules, force_init):
+        """For the importing logic.  Get the built-in module, stick it
+        into 'sys.modules' if not initialized or if force_in_sys_modules,
+        and initialize it if it was not already or if force_init.
+        """
         try:
             w_mod = self.builtin_modules[name]
         except KeyError:
@@ -422,15 +426,20 @@
                 self.w_SystemError,
                 "getbuiltinmodule() called "
                 "with non-builtin module %s", name)
-        else:
-            # Add the module to sys.modules
-            self.setitem(w_modules, w_name, w_mod)
 
-            # And initialize it
-            from pypy.interpreter.module import Module
-            if isinstance(w_mod, Module):
-                w_mod.init(self)
-            return w_mod
+        from pypy.interpreter.module import Module
+        if isinstance(w_mod, Module) and not w_mod.startup_called:
+            force_in_sys_modules = True          # not initialized so far:
+            force_init = True                    # force initialization
+
+        if force_in_sys_modules:
+            w_sys_modules = self.sys.get('modules')
+            self.setitem(w_sys_modules, self.wrap(name), w_mod)
+
+        if force_init and isinstance(w_mod, Module):
+            w_mod.init(self)
+
+        return w_mod
 
     def get_builtinmodule_to_install(self):
         """NOT_RPYTHON"""
@@ -547,12 +556,12 @@
 
     def install_mixedmodule(self, mixedname, installed_builtin_modules):
         """NOT_RPYTHON"""
-        modname = self.setbuiltinmodule(mixedname)
-        if modname:
-            assert modname not in installed_builtin_modules, (
-                "duplicate interp-level module enabled for the "
-                "app-level module %r" % (modname,))
-            installed_builtin_modules.append(modname)
+        modname = self._prepare_mixedmodule(mixedname)
+        assert modname
+        assert modname not in installed_builtin_modules, (
+            "duplicate interp-level module enabled for the "
+            "app-level module %r" % (modname,))
+        installed_builtin_modules.append(modname)
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -573,14 +573,16 @@
     load_extension_module(space, filename, modulename)
 
 @jit.dont_look_inside
-def load_module(space, w_modulename, find_info, reuse=False):
+def load_module(space, w_modulename, find_info, reuse=False, force_init=False):
     if find_info is None:
         return
     if find_info.w_loader:
         return space.call_method(find_info.w_loader, "load_module", w_modulename)
 
     if find_info.modtype == C_BUILTIN:
-        return space.getbuiltinmodule(find_info.filename, force_init=True)
+        return space.loadbuiltinmodule(find_info.filename,
+                                       force_in_sys_modules=True,
+                                       force_init=force_init)
 
     if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY):
         w_mod = None
@@ -721,7 +723,8 @@
 
         try:
             try:
-                return load_module(space, w_modulename, find_info, reuse=True)
+                return load_module(space, w_modulename, find_info, reuse=True,
+                                   force_init=True)
             finally:
                 if find_info.stream:
                     find_info.stream.close()

From noreply at buildbot.pypy.org  Tue Jun 11 13:26:16 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 13:26:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix this test to also pass on CPython
Message-ID: <20130611112616.8D4151C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64849:93a61b856da8
Date: 2013-06-11 13:24 +0200
http://bitbucket.org/pypy/pypy/changeset/93a61b856da8/

Log:	Fix this test to also pass on CPython

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -566,20 +566,19 @@
         assert 'setdefaultencoding' in dir(sys)
 
     def test_reimport_builtin(self):
-        # ...but not reload()!
-        import sys
+        import sys, time
         oldpath = sys.path
-        sys.setdefaultencoding = "<test_reimport_builtin removed this>"
+        time.tzset = "<test_reimport_builtin removed this>"
 
-        del sys.modules['sys']
-        import sys as sys1
-        assert sys.modules['sys'] is sys1 is sys
+        del sys.modules['time']
+        import time as time1
+        assert sys.modules['time'] is time1
 
-        assert sys.path is oldpath
-        assert sys.setdefaultencoding == "<test_reimport_builtin removed this>"
+        assert time.tzset == "<test_reimport_builtin removed this>"
 
-        reload(sys)   # fix it for people that want 'setdefaultencoding'
-        assert sys.setdefaultencoding != "<test_reimport_builtin removed this>"
+        reload(time1)   # don't leave a broken time.tzset behind
+        import time
+        assert time.tzset != "<test_reimport_builtin removed this>"
 
     def test_reload_infinite(self):
         import infinite_reload

From noreply at buildbot.pypy.org  Tue Jun 11 13:26:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 13:26:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add two new tests,
 one of which fails on PyPy because we don't
Message-ID: <20130611112617.EFEA31C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64850:7d2924ddfa80
Date: 2013-06-11 13:24 +0200
http://bitbucket.org/pypy/pypy/changeset/7d2924ddfa80/

Log:	Add two new tests, one of which fails on PyPy because we don't
	create several module objects for the same *built-in* module.

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -200,3 +200,48 @@
                 except KeyError:
                     pass
             rmtree(dir_name, True)
+
+    def test_builtin_reimport(self):
+        # from https://bugs.pypy.org/issue1514
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        __import__('marshal')
+        sys.modules['marshal'] = saved
+
+        assert marshal.loads == 42
+        import marshal
+        assert marshal.loads == 42
+        marshal.loads = old
+
+    def test_builtin_reimport_mess(self):
+        # taken from https://bugs.pypy.org/issue1514, with extra cases
+        # that show a difference with CPython: we can get on CPython
+        # several module objects for the same built-in module :-(
+        skip("several built-in module objects: not supported by pypy")
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        marshal2 = __import__('marshal')
+        assert marshal2 is not marshal
+        assert marshal2.loads is old
+        assert marshal2 is sys.modules['marshal']
+        assert marshal is saved
+        assert marshal.loads == 42
+
+        import marshal
+        assert marshal.loads is old
+
+        sys.modules['marshal'] = saved
+        import marshal
+        assert marshal.loads == 42
+
+        marshal.loads = old

From noreply at buildbot.pypy.org  Tue Jun 11 13:39:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 13:39:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a simpler test (missing so far)
 that shows what I did to be wrong.
Message-ID: <20130611113947.D957C1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64851:4dce95a2e895
Date: 2013-06-11 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/4dce95a2e895/

Log:	Add a simpler test (missing so far) that shows what I did to be
	wrong.

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -565,6 +565,13 @@
         assert sys.path is oldpath
         assert 'setdefaultencoding' in dir(sys)
 
+    def test_reimport_builtin_simple_case(self):
+        import sys, time
+        time.foo = "bar"
+        del sys.modules['time']
+        import time
+        assert not hasattr(time, 'foo')
+
     def test_reimport_builtin(self):
         import sys, time
         oldpath = sys.path

From noreply at buildbot.pypy.org  Tue Jun 11 13:39:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 13:39:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: backout f26956c61773.
Message-ID: <20130611113949.3D71E1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64852:1265890f2384
Date: 2013-06-11 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/1265890f2384/

Log:	backout f26956c61773.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -381,10 +381,8 @@
         except AttributeError:
             return self.__class__.__name__
 
-    def _prepare_mixedmodule(self, importname):
-        """NOT_RPYTHON.  Load the RPython code of a lazy pypy/module and put
-        it into 'self.builtin_modules', without any further initialization.
-        """
+    def setbuiltinmodule(self, importname):
+        """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules"""
         if '.' in importname:
             fullname = importname
             importname = fullname.rsplit('.', 1)[1]
@@ -403,22 +401,20 @@
 
         return name
 
-    @jit.elidable
-    def getbuiltinmodule(self, name):
-        """Return the built-in module 'name'.  The first time it is seen,
-        it is initialized and stored in 'sys.modules'.  This function is
-        elidable by the JIT because its effect is idempotent (if you call
-        it twice with the same name, you're getting the same effect as if
-        it was only called once).
-        """
-        return self.loadbuiltinmodule(name, force_in_sys_modules=False,
-                                      force_init=False)
+    def getbuiltinmodule(self, name, force_init=False):
+        w_name = self.wrap(name)
+        w_modules = self.sys.get('modules')
+        try:
+            w_mod = self.getitem(w_modules, w_name)
+        except OperationError, e:
+            if not e.match(self, self.w_KeyError):
+                raise
+        else:
+            if not force_init:
+                return w_mod
 
-    def loadbuiltinmodule(self, name, force_in_sys_modules, force_init):
-        """For the importing logic.  Get the built-in module, stick it
-        into 'sys.modules' if not initialized or if force_in_sys_modules,
-        and initialize it if it was not already or if force_init.
-        """
+        # If the module is a builtin but not yet imported,
+        # retrieve it and initialize it
         try:
             w_mod = self.builtin_modules[name]
         except KeyError:
@@ -426,20 +422,15 @@
                 self.w_SystemError,
                 "getbuiltinmodule() called "
                 "with non-builtin module %s", name)
+        else:
+            # Add the module to sys.modules
+            self.setitem(w_modules, w_name, w_mod)
 
-        from pypy.interpreter.module import Module
-        if isinstance(w_mod, Module) and not w_mod.startup_called:
-            force_in_sys_modules = True          # not initialized so far:
-            force_init = True                    # force initialization
-
-        if force_in_sys_modules:
-            w_sys_modules = self.sys.get('modules')
-            self.setitem(w_sys_modules, self.wrap(name), w_mod)
-
-        if force_init and isinstance(w_mod, Module):
-            w_mod.init(self)
-
-        return w_mod
+            # And initialize it
+            from pypy.interpreter.module import Module
+            if isinstance(w_mod, Module):
+                w_mod.init(self)
+            return w_mod
 
     def get_builtinmodule_to_install(self):
         """NOT_RPYTHON"""
@@ -556,12 +547,12 @@
 
     def install_mixedmodule(self, mixedname, installed_builtin_modules):
         """NOT_RPYTHON"""
-        modname = self._prepare_mixedmodule(mixedname)
-        assert modname
-        assert modname not in installed_builtin_modules, (
-            "duplicate interp-level module enabled for the "
-            "app-level module %r" % (modname,))
-        installed_builtin_modules.append(modname)
+        modname = self.setbuiltinmodule(mixedname)
+        if modname:
+            assert modname not in installed_builtin_modules, (
+                "duplicate interp-level module enabled for the "
+                "app-level module %r" % (modname,))
+            installed_builtin_modules.append(modname)
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -573,16 +573,14 @@
     load_extension_module(space, filename, modulename)
 
 @jit.dont_look_inside
-def load_module(space, w_modulename, find_info, reuse=False, force_init=False):
+def load_module(space, w_modulename, find_info, reuse=False):
     if find_info is None:
         return
     if find_info.w_loader:
         return space.call_method(find_info.w_loader, "load_module", w_modulename)
 
     if find_info.modtype == C_BUILTIN:
-        return space.loadbuiltinmodule(find_info.filename,
-                                       force_in_sys_modules=True,
-                                       force_init=force_init)
+        return space.getbuiltinmodule(find_info.filename, force_init=True)
 
     if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY):
         w_mod = None
@@ -723,8 +721,7 @@
 
         try:
             try:
-                return load_module(space, w_modulename, find_info, reuse=True,
-                                   force_init=True)
+                return load_module(space, w_modulename, find_info, reuse=True)
             finally:
                 if find_info.stream:
                     find_info.stream.close()

From noreply at buildbot.pypy.org  Tue Jun 11 13:39:50 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 13:39:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Skip the tests: these are new tests
 that should be fixed, as they pass
Message-ID: <20130611113950.7BC5E1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64853:337bb898c894
Date: 2013-06-11 13:37 +0200
http://bitbucket.org/pypy/pypy/changeset/337bb898c894/

Log:	Skip the tests: these are new tests that should be fixed, as they
	pass on CPython.

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -203,6 +203,7 @@
 
     def test_builtin_reimport(self):
         # from https://bugs.pypy.org/issue1514
+        skip("fix me")
         import sys, marshal
 
         old = marshal.loads
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -565,7 +565,15 @@
         assert sys.path is oldpath
         assert 'setdefaultencoding' in dir(sys)
 
-    def test_reimport_builtin_simple_case(self):
+    def test_reimport_builtin_simple_case_1(self):
+        import sys, time
+        del time.tzset
+        del sys.modules['time']
+        import time
+        assert hasattr(time, 'tzset')
+
+    def test_reimport_builtin_simple_case_2(self):
+        skip("fix me")
         import sys, time
         time.foo = "bar"
         del sys.modules['time']
@@ -573,6 +581,7 @@
         assert not hasattr(time, 'foo')
 
     def test_reimport_builtin(self):
+        skip("fix me")
         import sys, time
         oldpath = sys.path
         time.tzset = "<test_reimport_builtin removed this>"

From noreply at buildbot.pypy.org  Tue Jun 11 13:39:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 13:39:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: Easy fix
Message-ID: <20130611113951.B0F881C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64854:ad3579cf28fe
Date: 2013-06-11 13:39 +0200
http://bitbucket.org/pypy/pypy/changeset/ad3579cf28fe/

Log:	Easy fix

diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -43,6 +43,7 @@
             # the module was already imported.  Refresh its content with
             # the saved dict, as done with built-in and extension modules
             # on CPython.
+            space.call_method(self.w_dict, 'clear')
             space.call_method(self.w_dict, 'update', self.w_initialdict)
 
         for w_submodule in self.submodules_w:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -573,7 +573,6 @@
         assert hasattr(time, 'tzset')
 
     def test_reimport_builtin_simple_case_2(self):
-        skip("fix me")
         import sys, time
         time.foo = "bar"
         del sys.modules['time']

From noreply at buildbot.pypy.org  Tue Jun 11 13:57:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 13:57:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill even the "easy fix",
	add a test (thanks amaury)
Message-ID: <20130611115723.6BD441C0328@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64855:ece10991099e
Date: 2013-06-11 13:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ece10991099e/

Log:	Kill even the "easy fix", add a test (thanks amaury)

diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -43,7 +43,6 @@
             # the module was already imported.  Refresh its content with
             # the saved dict, as done with built-in and extension modules
             # on CPython.
-            space.call_method(self.w_dict, 'clear')
             space.call_method(self.w_dict, 'update', self.w_initialdict)
 
         for w_submodule in self.submodules_w:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -565,6 +565,12 @@
         assert sys.path is oldpath
         assert 'setdefaultencoding' in dir(sys)
 
+    def test_reload_builtin_doesnt_clear(self):
+        import sys
+        sys.foobar = "baz"
+        reload(sys)
+        assert sys.foobar == "baz"
+
     def test_reimport_builtin_simple_case_1(self):
         import sys, time
         del time.tzset
@@ -573,6 +579,7 @@
         assert hasattr(time, 'tzset')
 
     def test_reimport_builtin_simple_case_2(self):
+        skip("fix me")
         import sys, time
         time.foo = "bar"
         del sys.modules['time']

From noreply at buildbot.pypy.org  Tue Jun 11 15:23:22 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 15:23:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: try to be on the safe side
Message-ID: <20130611132322.EA2381C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64856:9e9835f0f54b
Date: 2013-06-11 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/9e9835f0f54b/

Log:	try to be on the safe side

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -29,14 +29,13 @@
     try:
         # first try to print the exception's class name
         stderr = sys.stderr
-        stderr.write(getattr(exctype, '__name__', exctype))
+        stderr.write(str(getattr(exctype, '__name__', exctype)))
         # then attempt to get the str() of the exception
         try:
             s = str(value)
         except:
             s = '<failure of str() on the exception instance>'
-        # then print it, and don't worry too much about the extra space
-        # between the exception class and the ':'
+        # then print it
         if s:
             stderr.write(': %s\n' % (s,))
         else:

From noreply at buildbot.pypy.org  Tue Jun 11 16:06:16 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Tue, 11 Jun 2013 16:06:16 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: hg merge default
Message-ID: <20130611140616.727AD1C0F88@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: improve-docs
Changeset: r64857:0da9b23be0af
Date: 2013-06-11 16:04 +0200
http://bitbucket.org/pypy/pypy/changeset/0da9b23be0af/

Log:	hg merge default

diff too long, truncating to 2000 out of 25881 lines

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -1,30 +1,16 @@
-"""Provide access to Python's configuration information.  The specific
-configuration variables available depend heavily on the platform and
-configuration.  The values may be retrieved using
-get_config_var(name), and the list of variables is available via
-get_config_vars().keys().  Additional convenience functions are also
-available.
-
-Written by:   Fred L. Drake, Jr.
-Email:        <fdrake at acm.org>
-"""
-
-__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
-
-import sys
-
 
 # The content of this file is redirected from
 # sysconfig_cpython or sysconfig_pypy.
+# All underscore names are imported too, because
+# people like to use undocumented sysconfig._xxx
+# directly.
 
+import sys
 if '__pypy__' in sys.builtin_module_names:
-    from distutils.sysconfig_pypy import *
-    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
-    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+    from distutils import sysconfig_pypy as _sysconfig_module
 else:
-    from distutils.sysconfig_cpython import *
-    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
-    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
+    from distutils import sysconfig_cpython as _sysconfig_module
+globals().update(_sysconfig_module.__dict__)
 
 _USE_CLANG = None
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -9,7 +9,7 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
 import os
 import re
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -1,9 +1,17 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -49,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -71,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,21 +134,25 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
 }
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
+}
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
@@ -164,7 +168,7 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+    return _levelToName.get(level, ("Level %s" % level))
 
 def addLevelName(level, levelName):
     """
@@ -174,8 +178,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -183,9 +187,9 @@
     if isinstance(level, int):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
@@ -277,7 +281,7 @@
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - long(ct)) * 1000
+        self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads and thread:
             self.thread = thread.get_ident()
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -156,7 +156,7 @@
         h = klass(*args)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -187,7 +187,7 @@
     opts = cp.options(sectname)
     if "level" in opts:
         level = cp.get(sectname, "level")
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = cp.get(sectname, "handlers")
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,11 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    self._sock._decref_socketios()
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_codecs.py b/lib-python/2.7/test/test_codecs.py
--- a/lib-python/2.7/test/test_codecs.py
+++ b/lib-python/2.7/test/test_codecs.py
@@ -2,7 +2,11 @@
 import unittest
 import codecs
 import locale
-import sys, StringIO, _testcapi
+import sys, StringIO
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 class Queue(object):
     """
@@ -1387,7 +1391,7 @@
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
-            if encoding not in broken_incremental_coders:
+            if encoding not in broken_incremental_coders and _testcapi:
                 # check incremental decoder/encoder (fetched via the Python
                 # and C API) and iterencode()/iterdecode()
                 try:
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -65,7 +65,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
         finally:
             logging._releaseLock()
 
@@ -97,8 +98,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -7,7 +7,8 @@
 import subprocess
 from copy import copy, deepcopy
 
-from test.test_support import run_unittest, TESTFN, unlink, get_attribute
+from test.test_support import (run_unittest, TESTFN, unlink, get_attribute,
+                               import_module)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -236,7 +237,10 @@
 
     def test_get_config_h_filename(self):
         config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
+        # import_module skips the test when the CPython C Extension API
+        # appears to not be supported
+        self.assertTrue(os.path.isfile(config_h) or
+                        not import_module('_testcapi'), config_h)
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -1,6 +1,9 @@
 """Test cases for traceback module"""
 
-from _testcapi import traceback_print
+try:
+    from _testcapi import traceback_print
+except ImportError:
+    traceback_print = None
 from StringIO import StringIO
 import sys
 import unittest
@@ -176,6 +179,8 @@
 class TracebackFormatTests(unittest.TestCase):
 
     def test_traceback_format(self):
+        if traceback_print is None:
+            raise unittest.SkipTest('Requires _testcapi')
         try:
             raise KeyError('blah')
         except KeyError:
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -1609,7 +1609,10 @@
         self.assertEqual("{}".format(u), '__unicode__ overridden')
 
     def test_encode_decimal(self):
-        from _testcapi import unicode_encodedecimal
+        try:
+            from _testcapi import unicode_encodedecimal
+        except ImportError:
+            raise unittest.SkipTest('Requires _testcapi')
         self.assertEqual(unicode_encodedecimal(u'123'),
                          b'123')
         self.assertEqual(unicode_encodedecimal(u'\u0663.\u0661\u0664'),
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -130,7 +130,7 @@
     RegrTest('test_bz2.py', usemodules='bz2'),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
-    RegrTest('test_capi.py'),
+    RegrTest('test_capi.py', usemodules='cpyext'),
     RegrTest('test_cd.py'),
     RegrTest('test_cfgparser.py'),
     RegrTest('test_cgi.py'),
@@ -177,7 +177,7 @@
     RegrTest('test_cprofile.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
     RegrTest('test_csv.py', usemodules='_csv'),
-    RegrTest('test_ctypes.py', usemodules="_rawffi thread"),
+    RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
     RegrTest('test_curses.py'),
     RegrTest('test_datetime.py', usemodules='binascii struct'),
     RegrTest('test_dbm.py'),
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -166,8 +166,7 @@
         if self is StructOrUnion:
             return
         if '_fields_' not in self.__dict__:
-            self._fields_ = []
-            _set_shape(self, [], self._is_union)
+            self._fields_ = []  # As a side-effet, this also sets the ffishape.
 
     __setattr__ = struct_setattr
 
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_pypy_testcapi.py
copy from lib_pypy/_testcapi.py
copy to lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,14 +1,20 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
     """
     thisdir = os.path.dirname(__file__)
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -19,13 +25,13 @@
         ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
+    res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
                            extra_preargs=ccflags)
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
+    output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
@@ -37,7 +43,7 @@
             library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
+                        '/EXPORT:init' + modulename]
     else:
         libraries = []
         extra_ldargs = []
@@ -49,9 +55,7 @@
         libraries=libraries,
         extra_preargs=extra_ldargs)
 
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,57 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testcapi'")
+else:
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.6"
-__version_info__ = (0, 6)
+__version__ = "0.7"
+__version_info__ = (0, 7)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -73,15 +73,15 @@
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        BVoidP = self._get_cached_btype(model.voidp_type)
+        self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(BVoidP, 0)
+                FFI.NULL = self.cast(self.BVoidP, 0)
                 FFI.CData, FFI.CType = backend._get_types()
         else:
             # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(BVoidP, 0)
+            self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False):
@@ -346,6 +346,12 @@
         self._cdefsources.extend(ffi_to_include._cdefsources)
         self._cdefsources.append(']')
 
+    def new_handle(self, x):
+        return self._backend.newp_handle(self.BVoidP, x)
+
+    def from_handle(self, x):
+        return self._backend.from_handle(x)
+
 
 def _make_ffi_library(ffi, libname, flags):
     import os
@@ -355,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
@@ -372,8 +378,8 @@
             BType = ffi._get_cached_btype(tp)
             try:
                 value = backendlib.load_function(BType, name)
-            except KeyError:
-                raise AttributeError(name)
+            except KeyError as e:
+                raise AttributeError('%s: %s' % (name, e))
             library.__dict__[name] = value
             return
         #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -16,6 +16,7 @@
 class CTypesData(object):
     __metaclass__ = CTypesType
     __slots__ = ['__weakref__']
+    __name__ = '<cdata>'
 
     def __init__(self, *args):
         raise TypeError("cannot instantiate %r" % (self.__class__,))
@@ -491,6 +492,8 @@
         elif BItem in (getbtype(model.PrimitiveType('signed char')),
                        getbtype(model.PrimitiveType('unsigned char'))):
             kind = 'bytep'
+        elif BItem is getbtype(model.void_type):
+            kind = 'voidp'
         else:
             kind = 'generic'
         #
@@ -546,13 +549,13 @@
             def __setitem__(self, index, value):
                 self._as_ctype_ptr[index] = BItem._to_ctypes(value)
 
-            if kind == 'charp':
+            if kind == 'charp' or kind == 'voidp':
                 @classmethod
-                def _arg_to_ctypes(cls, value):
-                    if isinstance(value, bytes):
-                        return ctypes.c_char_p(value)
+                def _arg_to_ctypes(cls, *value):
+                    if value and isinstance(value[0], bytes):
+                        return ctypes.c_char_p(value[0])
                     else:
-                        return super(CTypesPtr, cls)._arg_to_ctypes(value)
+                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
 
             if kind == 'charp' or kind == 'bytep':
                 def _to_string(self, maxlen):
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -15,6 +15,20 @@
     def patch_extension_kwds(self, kwds):
         pass
 
+    def find_module(self, module_name, path, so_suffix):
+        try:
+            f, filename, descr = imp.find_module(module_name, path)
+        except ImportError:
+            return None
+        if f is not None:
+            f.close()
+        # Note that after a setuptools installation, there are both .py
+        # and .so files with the same basename.  The code here relies on
+        # imp.find_module() locating the .so in priority.
+        if descr[0] != so_suffix:
+            return None
+        return filename
+
     def collect_types(self):
         self._typesdict = {}
         self._generate("collecttype")
@@ -142,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -427,9 +444,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -687,7 +704,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
     # ----------
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -1,4 +1,4 @@
-import sys
+import sys, os
 import types
 
 from . import model, ffiplatform
@@ -20,6 +20,16 @@
         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
+    def find_module(self, module_name, path, so_suffix):
+        basename = module_name + so_suffix
+        if path is None:
+            path = sys.path
+        for dirname in path:
+            filename = os.path.join(dirname, basename)
+            if os.path.isfile(filename):
+                return filename
+        return None
+
     def collect_types(self):
         pass      # not needed in the generic engine
 
@@ -64,6 +74,9 @@
         class FFILibrary(types.ModuleType):
             _cffi_generic_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir
         library = FFILibrary("")
         #
         # finally, call the loaded_gen_xxx() functions.  This will set
@@ -158,21 +171,22 @@
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
-            if any(isinstance(type, model.StructOrUnion) for type in tp.args):
+            if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
-                for i, type in enumerate(tp.args):
-                    if isinstance(type, model.StructOrUnion):
-                        type = model.PointerType(type)
-                        indirections.append((i, type))
-                    indirect_args.append(type)
+                for i, typ in enumerate(tp.args):
+                    if isinstance(typ, model.StructOrUnion):
+                        typ = model.PointerType(typ)
+                        indirections.append((i, typ))
+                    indirect_args.append(typ)
                 tp = model.FunctionPtrType(tuple(indirect_args),
                                            tp.result, tp.ellipsis)
             BFunc = self.ffi._get_cached_btype(tp)
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
-            for i, type in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, type)
+            for i, typ in indirections:
+                newfunction = self._make_struct_wrapper(newfunction, i, typ)
         setattr(library, name, newfunction)
+        type(library)._cffi_dir.append(name)
 
     def _make_struct_wrapper(self, oldfunc, i, tp):
         backend = self.ffi._backend
@@ -216,9 +230,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -380,6 +394,7 @@
         is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
         value = self._load_constant(is_int, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # enums
@@ -427,6 +442,7 @@
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
             setattr(library, enumerator, enumvalue)
+            type(library)._cffi_dir.append(enumerator)
 
     # ----------
     # macros: for now only for integers
@@ -440,6 +456,7 @@
     def _loaded_gen_macro(self, tp, name, module, library):
         value = self._load_constant(True, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # global variables
@@ -465,6 +482,7 @@
                 BArray = self.ffi._get_cached_btype(tp)
                 value = self.ffi.cast(BArray, value)
             setattr(library, name, value)
+            type(library)._cffi_dir.append(name)
             return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
@@ -476,7 +494,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
 cffimod_header = r'''
 #include <stdio.h>
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -102,21 +102,10 @@
                 path = pkg.__path__
             else:
                 path = None
-            try:
-                f, filename, descr = imp.find_module(self.get_module_name(),
-                                                     path)
-            except ImportError:
+            filename = self._vengine.find_module(self.get_module_name(), path,
+                                                 _get_so_suffix())
+            if filename is None:
                 return
-            if f is not None:
-                f.close()
-            if filename.lower().endswith('.py'):
-                # on PyPy, if there are both .py and .pypy-19.so files in
-                # the same directory, the .py file is returned.  That's the
-                # case after a setuptools installation.  We never want to
-                # load the .py file here...
-                filename = filename[:-3] + _get_so_suffix()
-                if not os.path.isfile(filename):
-                    return
             self.modulefilename = filename
         self._vengine.collect_types()
         self._has_module = True
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
new file mode 100644
--- /dev/null
+++ b/lib_pypy/greenlet.egg-info
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: greenlet
+Version: 0.4.0
+Summary: Lightweight in-process concurrent programming
+Home-page: https://github.com/python-greenlet/greenlet
+Author: Ralf Schmitt (for CPython), PyPy team
+Author-email: pypy-dev at python.org
+License: MIT License
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -64,7 +64,8 @@
     del working_modules["termios"]
     del working_modules["_minimal_curses"]
 
-    del working_modules["cppyy"]  # not tested on win32
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # not tested on win32
 
     # The _locale module is needed by site.py on Windows
     default_modules["_locale"] = None
@@ -77,7 +78,8 @@
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
     del working_modules["_multiprocessing"]   # depends on rctime
-    del working_modules["cppyy"]  # depends on ctypes
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # depends on ctypes
 
 
 module_dependencies = {
@@ -120,12 +122,10 @@
                     __import__(name)
             except (ImportError, CompilationError, py.test.skip.Exception), e:
                 errcls = e.__class__.__name__
-                config.add_warning(
+                raise Exception(
                     "The module %r is disabled\n" % (modname,) +
                     "because importing %s raised %s\n" % (name, errcls) +
                     str(e))
-                raise ConflictConfigError("--withmod-%s: %s" % (modname,
-                                                                errcls))
         return validator
     else:
         return None
@@ -216,10 +216,6 @@
                    "(the empty string and potentially single-char strings)",
                    default=False),
 
-        BoolOption("withsmalltuple",
-                   "use small tuples",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
@@ -364,6 +360,7 @@
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
+
     config.objspace.usemodules.suggest(**dict.fromkeys(modules, True))
 
 def enable_translationmodules(config):
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,6 @@
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 The ``__pypy__`` module
 =======================
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.0'
+release = '2.0.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -157,6 +157,9 @@
     $ genreflex MyClass.h
     $ g++ -fPIC -rdynamic -O2 -shared -I$REFLEXHOME/include MyClass_rflx.cpp -o libMyClassDict.so -L$REFLEXHOME/lib -lReflex
 
+Next, make sure that the library can be found through the dynamic lookup path
+(the ``LD_LIBRARY_PATH`` environment variable on Linux, ``PATH`` on Windows),
+for example by adding ".".
 Now you're ready to use the bindings.
 Since the bindings are designed to look pythonistic, it should be
 straightforward::
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -33,7 +33,8 @@
 Layers
 ------
 
-PyPy has layers. Those layers help us keep the respective parts separated enough
+PyPy has layers. Just like Ogres or onions.
+Those layers help us keep the respective parts separated enough
 to be worked on independently and make the complexity manageable. This is,
 again, just a sanity requirement for such a complex project. For example writing
 a new optimization for the JIT usually does **not** involve touching a Python
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -22,7 +22,8 @@
   will capture the revision number of this change for the release;
   some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
-  necessary
+  necessary; also update the version number in pypy/doc/conf.py,
+  and in pypy/doc/index.rst
 * update pypy/doc/contributor.rst (and possibly LICENSE)
 * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
   and create a fresh whatsnew_head.rst after the release
diff --git a/pypy/doc/index-old.rst b/pypy/doc/index-old.rst
--- a/pypy/doc/index-old.rst
+++ b/pypy/doc/index-old.rst
@@ -43,7 +43,7 @@
 
 * :doc:`FAQ <faq>`: some frequently asked questions.
 
-* `Release 2.0`_: the latest official release
+* `Release 2.0.2`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy
 
@@ -57,7 +57,7 @@
   particularly organized
 
 .. _PyPy blog: http://morepypy.blogspot.com/
-.. _Release 2.0: http://pypy.org/download.html
+.. _Release 2.0.2: http://pypy.org/download.html
 .. _speed.pypy.org: http://speed.pypy.org
 
 .. toctree::
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,7 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/doc/release-2.0.1.rst b/pypy/doc/release-2.0.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.1.rst
@@ -0,0 +1,46 @@
+==============================
+PyPy 2.0.1 - Bohr Sm&#248;rrebr&#248;d
+==============================
+
+We're pleased to announce PyPy 2.0.1.  This is a stable bugfix release
+over `2.0`_.  You can download it here:
+  
+    http://pypy.org/download.html
+
+The fixes are mainly about fatal errors or crashes in our stdlib.  See
+below for more details.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Support for ARM is progressing but not bug-free yet.
+
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
+
+Highlights
+==========
+
+- fix an occasional crash in the JIT that ends in `RPython Fatal error:
+  NotImplementedError`__.
+
+- `id(x)` is now always a positive number (except on int/float/long/complex).
+  This fixes an issue in ``_sqlite.py`` (mostly for 32-bit Linux).
+
+- fix crashes of callback-from-C-functions (with cffi) when used together
+  with Stackless features, on asmgcc (i.e. Linux only).  Now `gevent should
+  work better`__.
+
+- work around an eventlet issue with `socket._decref_socketios()`__.
+
+.. __: https://bugs.pypy.org/issue1482
+.. __: http://mail.python.org/pipermail/pypy-dev/2013-May/011362.html
+.. __: https://bugs.pypy.org/issue1468
+.. _2.0: release-2.0.0.html
+
+Cheers,
+arigo et. al. for the PyPy team
diff --git a/pypy/doc/release-2.0.2.rst b/pypy/doc/release-2.0.2.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.2.rst
@@ -0,0 +1,46 @@
+=========================
+PyPy 2.0.2 - Fermi Panini
+=========================
+
+We're pleased to announce PyPy 2.0.2.  This is a stable bugfix release
+over `2.0`_ and `2.0.1`_.  You can download it here:
+
+    http://pypy.org/download.html
+
+It fixes a crash in the JIT when calling external C functions (with
+ctypes/cffi) in a multithreaded context.
+
+.. _2.0: release-2.0.0.html
+.. _2.0.1: release-2.0.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Support for ARM is progressing but not bug-free yet.
+
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
+
+Highlights
+==========
+
+This release contains only the fix described above.  A crash (or wrong
+results) used to occur if all these conditions were true:
+
+- your program is multithreaded;
+
+- it runs on a single-core machine or a heavily-loaded multi-core one;
+
+- it uses ctypes or cffi to issue external calls to C functions.
+
+This was fixed in the branch `emit-call-x86`__ (see the example file
+``bug1.py``).
+
+.. __: https://bitbucket.org/pypy/pypy/commits/7c80121abbf4
+
+Cheers,
+arigo et. al. for the PyPy team
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -19,23 +19,28 @@
     branches.discard('default')
     return startrev, branches
 
-def get_merged_branches(path, startrev, endrev):
-    if getstatusoutput('hg root')[0]:
+def get_merged_branches(path, startrev, endrev, current_branch=None):
+    errcode, wc_branch = getstatusoutput('hg branch')
+    if errcode != 0:
         py.test.skip('no Mercurial repo')
+    if current_branch is None:
+        current_branch = wc_branch
 
     # X = take all the merges which are descendants of startrev and are on default
     # revset = all the parents of X which are not on default
     # ===>
     # revset contains all the branches which have been merged to default since
     # startrev
-    revset = 'parents(%s::%s and \
+    revset = "parents(%s::%s and \
                       merge() and \
-                      branch(default)) and \
-              not branch(default)' % (startrev, endrev)
+                      branch('%s')) and \
+              not branch('%s')" % (startrev, endrev,
+                                   current_branch, current_branch)
     cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset)
     out = getoutput(cmd)
     branches = set(map(str.strip, out.splitlines()))
-    return branches
+    branches.discard("default")
+    return branches, current_branch
 
 
 def test_parse_doc():
@@ -65,7 +70,8 @@
     assert branches == set(['foobar', 'hello'])
 
 def test_get_merged_branches():
-    branches = get_merged_branches(ROOT, 'f34f0c11299f', '79770e0c2f93')
+    branches, _ = get_merged_branches(ROOT, 'f34f0c11299f', '79770e0c2f93',
+                                      'default')
     assert branches == set(['numpy-indexing-by-arrays-bool',
                             'better-jit-hooks-2',
                             'numpypy-ufuncs'])
@@ -76,7 +82,9 @@
     whatsnew_list.sort()
     last_whatsnew = whatsnew_list[-1].read()
     startrev, documented = parse_doc(last_whatsnew)
-    merged = get_merged_branches(ROOT, startrev, '')
+    merged, branch = get_merged_branches(ROOT, startrev, '')
+    merged.discard('default')
+    merged.discard('')
     not_documented = merged.difference(documented)
     not_merged = documented.difference(merged)
     print 'Branches merged but not documented:'
@@ -85,4 +93,6 @@
     print 'Branches documented but not merged:'
     print '\n'.join(not_merged)
     print
-    assert not not_documented and not not_merged
+    assert not not_documented
+    if branch == 'default':
+        assert not not_merged
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,48 @@
 
 .. branch: numpy-pickle
 Pickling of numpy arrays and dtypes (including record dtypes)
+
+.. branch: remove-array-smm
+Remove multimethods in the arraymodule
+
+.. branch: callback-stacklet
+Fixed bug when switching stacklets from a C callback
+
+.. branch: remove-set-smm
+Remove multi-methods on sets
+
+.. branch: numpy-subarrays
+Implement subarrays for numpy
+
+.. branch: remove-dict-smm
+Remove multi-methods on dict
+
+.. branch: remove-list-smm-2
+Remove remaining multi-methods on list
+
+.. branch: arm-stacklet
+Stacklet support for ARM, enables _continuation support
+
+.. branch: remove-tuple-smm
+Remove multi-methods on tuple
+
+.. branch: remove-iter-smm
+Remove multi-methods on iterators
+
+.. branch: emit-call-x86
+.. branch: emit-call-arm
+
+.. branch: on-abort-resops
+Added list of resops to the pypyjit on_abort hook.
+
+.. branch: logging-perf
+Speeds up the stdlib logging module
+
+.. branch: operrfmt-NT
+Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+
+.. branch: ctypes-byref
+Add the '_obj' attribute on ctypes pointer() and byref() objects
diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py
--- a/pypy/goal/getnightly.py
+++ b/pypy/goal/getnightly.py
@@ -8,7 +8,9 @@
     arch = 'linux'
     cmd = 'wget "%s"'
     tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'"
-if sys.platform.startswith('darwin'):
+    if os.uname()[-1].startswith('arm'):
+        arch += '-armhf-raspbian'
+elif sys.platform.startswith('darwin'):
     arch = 'osx'
     cmd = 'curl -O "%s"'
     tar = "tar -x -v --strip-components=2 -f %s '*/bin/pypy'"
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -2,6 +2,7 @@
 
 import os, sys
 
+import pypy
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError
 from pypy.tool.ann_override import PyPyAnnotatorPolicy
@@ -9,6 +10,8 @@
 from rpython.config.config import ConflictConfigError
 from pypy.tool.option import make_objspace
 from pypy.conftest import pypydir
+from rpython.rlib import rthread
+from pypy.module.thread import os_thread
 
 thisdir = py.path.local(__file__).dirpath()
 
@@ -78,13 +81,65 @@
     # should be used as sparsely as possible, just to register callbacks
 
     from rpython.rlib.entrypoint import entrypoint
-    from rpython.rtyper.lltypesystem import rffi
+    from rpython.rtyper.lltypesystem import rffi, lltype
+
+    w_pathsetter = space.appexec([], """():
+    def f(path):
+        import sys
+        sys.path[:] = path
+    return f
+    """)
+
+    @entrypoint('main', [rffi.CCHARP, lltype.Signed], c_name='pypy_setup_home')
+    def pypy_setup_home(ll_home, verbose):
+        from pypy.module.sys.initpath import pypy_find_stdlib
+        if ll_home:
+            home = rffi.charp2str(ll_home)
+        else:
+            home = pypydir
+        w_path = pypy_find_stdlib(space, home)
+        if space.is_none(w_path):
+            if verbose:
+                debug("Failed to find library based on pypy_find_stdlib")
+            return 1
+        space.startup()
+        space.call_function(w_pathsetter, w_path)
+        # import site
+        try:
+            import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
+                                    space.wrap('__import__'))
+            space.call_function(import_, space.wrap('site'))
+            return 0
+        except OperationError, e:
+            if verbose:
+                debug("OperationError:")
+                debug(" operror-type: " + e.w_type.getname(space))
+                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            return 1
 
     @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
         source = rffi.charp2str(ll_source)
         return _pypy_execute_source(source)
 
+    @entrypoint('main', [], c_name='pypy_init_threads')
+    def pypy_init_threads():
+        if not space.config.objspace.usemodules.thread:
+            return
+        os_thread.setup_threads(space)
+        rffi.aroundstate.before()
+
+    @entrypoint('main', [], c_name='pypy_thread_attach')
+    def pypy_thread_attach():
+        if not space.config.objspace.usemodules.thread:
+            return
+        os_thread.setup_threads(space)
+        os_thread.bootstrapper.acquire(space, None, None)
+        rthread.gc_thread_start()
+        os_thread.bootstrapper.nbthreads += 1
+        os_thread.bootstrapper.release()
+        rffi.aroundstate.before()
+
     w_globals = space.newdict()
     space.setitem(w_globals, space.wrap('__builtins__'),
                   space.builtin_modules['__builtin__'])
@@ -101,7 +156,10 @@
             return 1
         return 0
 
-    return entry_point, _pypy_execute_source # for tests
+    return entry_point, {'pypy_execute_source': pypy_execute_source,
+                         'pypy_init_threads': pypy_init_threads,
+                         'pypy_thread_attach': pypy_thread_attach,
+                         'pypy_setup_home': pypy_setup_home}
 
 def call_finish(space):
     space.finish()
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,8 +2,8 @@
 # App-level version of py.py.
 # See test/test_app_main.
 
-# Missing vs CPython: -d, -OO, -t, -v, -x, -3
-"""\
+# Missing vs CPython: -d, -t, -v, -x, -3
+USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 -c cmd : program passed in as string (terminates option list)
@@ -12,7 +12,8 @@
 -i     : inspect interactively after running script; forces a prompt even
          if stdin does not appear to be a terminal; also PYTHONINSPECT=x
 -m mod : run library module as a script (terminates option list)
--O     : dummy optimization flag for compatibility with CPython
+-O     : skip assert statements
+-OO    : remove docstrings when importing modules in addition to -O
 -R     : ignored (see http://bugs.python.org/issue14621)
 -Q arg : division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
@@ -27,7 +28,6 @@
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
 """
-USAGE1 = __doc__
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
 Other environment variables:
@@ -470,6 +470,10 @@
         sys.py3kwarning = bool(sys.flags.py3k_warning)
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
+        if sys.flags.optimize >= 1:
+            import __pypy__
+            __pypy__.set_debug(False)
+
         if sys.py3kwarning:
             print >> sys.stderr, (
                 "Warning: pypy does not implement py3k warnings")
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -86,12 +86,9 @@
             args_w = space.fixedview(w_stararg)
         except OperationError, e:
             if e.match(space, space.w_TypeError):
-                w_type = space.type(w_stararg)
-                typename = w_type.getname(space)
-                raise OperationError(
+                raise operationerrfmt(
                     space.w_TypeError,
-                    space.wrap("argument after * must be "
-                               "a sequence, not %s" % (typename,)))
+                    "argument after * must be a sequence, not %T", w_stararg)
             raise
         self.arguments_w = self.arguments_w + args_w
 
@@ -116,12 +113,10 @@
                 w_keys = space.call_method(w_starstararg, "keys")
             except OperationError, e:
                 if e.match(space, space.w_AttributeError):
-                    w_type = space.type(w_starstararg)
-                    typename = w_type.getname(space)
-                    raise OperationError(
+                    raise operationerrfmt(
                         space.w_TypeError,
-                        space.wrap("argument after ** must be "
-                                   "a mapping, not %s" % (typename,)))
+                        "argument after ** must be a mapping, not %T",
+                        w_starstararg)
                 raise
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -245,6 +245,8 @@
         if w_len is None:
             w_len = space.len(self.w_consts)
             space.setitem(self.w_consts, w_key, w_len)
+        if space.int_w(w_len) == 0:
+            self.scope.doc_removable = False
         return space.int_w(w_len)
 
     def _make_key(self, obj):
@@ -632,6 +634,7 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+    ops.JUMP_IF_NOT_DEBUG : 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2793,8 +2793,7 @@
 
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2835,8 +2834,7 @@
 
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2881,8 +2879,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -2925,8 +2922,7 @@
 
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2971,8 +2967,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -2993,8 +2988,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3024,8 +3018,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3046,8 +3039,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3064,8 +3056,7 @@
 
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3081,8 +3072,7 @@
 
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3131,8 +3121,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3149,8 +3138,7 @@
 
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
             list_w = []
@@ -3166,8 +3154,7 @@
 
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3183,8 +3170,7 @@
 
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3234,8 +3220,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3278,8 +3263,7 @@
 
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3320,8 +3304,7 @@
 
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3341,8 +3324,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3391,8 +3373,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3415,8 +3396,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3439,8 +3419,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3489,8 +3468,7 @@

From noreply at buildbot.pypy.org  Tue Jun 11 16:29:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 16:29:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130611142903.007611C1026@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92:b30306fc8a86
Date: 2013-06-11 16:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/b30306fc8a86/

Log:	in-progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -35,6 +35,10 @@
   return (P->h_revision == stm_private_rev_num) ||
     (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
 }
+int _stm_is_private(gcptr P)
+{
+  return is_private(P);
+}
 
 /************************************************************/
 
@@ -208,82 +212,89 @@
     }
 }
 
-#if 0
-static gcptr _latest_gcptr(gcptr R)
+gcptr _stm_nonrecord_barrier(gcptr G)
 {
-  /* don't use, for tests only */
+  /* follows the logic in stm_DirectReadBarrier() */
+  struct tx_descriptor *d = thread_descriptor;
+  gcptr P = G;
   revision_t v;
- retry:
-  v = R->h_revision;
-  if (!(v & 1))  // "is a pointer", i.e.
-    {            //      "has a more recent revision"
-      if (v & 2)
+
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+    {
+    private_from_protected:
+      if (!(((gcptr)P->h_revision)->h_revision & 1))
         {
-          v &= ~2;
-          if (!stmgc_is_young_in(thread_descriptor, (gcptr)v))
-            return NULL;   /* can't access */
+          fprintf(stderr, "_stm_nonrecord_barrier: %p -> NULL "
+                  "private_from_protected but protected changed\n", G);
+          return NULL;
         }
-      R = (gcptr)v;
-      goto retry;
-    }
-  return R;
-}
-
-gcptr _stm_nonrecord_barrier(gcptr obj, int *result)
-{
-  /* warning, this is for tests only, and it is not thread-safe! */
-  struct tx_descriptor *d = thread_descriptor;
-  if (gcptrlist_size(&d->stolen_objects) > 0)
-    stmgc_normalize_stolen_objects();
-
-  enum protection_class_t e = stmgc_classify(obj);
-  if (e == K_PRIVATE)
-    {
-      *result = 2;   /* 'obj' a private object to start with */
-      return obj;
-    }
-  obj = _latest_gcptr(obj);
-  if (obj == NULL)
-    {
-      assert(e == K_PUBLIC);
-      *result = 3;   /* can't check anything: we'd need foreign access */
-      return NULL;
-    }
-  if (stmgc_classify(obj) == K_PRIVATE)
-    {
-      *result = 1;
-      return obj;
+      goto add_in_recent_reads_cache;
     }
 
-  wlog_t *item;
-  G2L_LOOP_FORWARD(d->public_to_private, item)
+  if (P->h_tid & GCFLAG_PUBLIC)
     {
-      gcptr R = item->addr;
-      gcptr L = item->val;
-      if (_latest_gcptr(R) == obj)
+      while (v = ACCESS_ONCE(P->h_revision), !(v & 1))
         {
-          /* 'obj' has a private version.  The way we detect this lets us
-             find it even if we already have a committed version that
-             will cause conflict. */
-          *result = 1;
-          return L;
+          if (v & 2)
+            goto follow_stub;
+
+          P = (gcptr)v;
+          assert(P->h_tid & GCFLAG_PUBLIC);
         }
-    } G2L_LOOP_END;
 
-  if (obj->h_revision > d->start_time)
-    {
-      /* 'obj' has no private version, and the public version was modified */
-      *result = -1;
-      return NULL;
+      if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
+        {
+          wlog_t *item;
+          G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
+
+          P = item->val;
+        found_in_stolen_objects:
+          assert(!(P->h_tid & GCFLAG_PUBLIC));
+          assert(is_private(P));
+          fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p "
+                  "public_to_private\n", G, P);
+          return P;
+
+        no_private_obj:;
+          gcptr L = _stm_find_stolen_objects(d, P);
+          if (L != NULL)
+            {
+              P = L;
+              goto found_in_stolen_objects;
+            }
+        }
+
+      if (UNLIKELY(v > d->start_time))
+        {
+          fprintf(stderr, "_stm_nonrecord_barrier: %p -> NULL changed\n", G);
+          return NULL;   // object too recent
+        }
+      fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p public\n", G, P);
     }
   else
     {
-      /* 'obj' has only an up-to-date public version */
-      *result = 0;
-      return obj;
+      fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p protected\n", G, P);
+    }
+
+ register_in_list_of_read_objects:
+ add_in_recent_reads_cache:
+  return P;
+
+ follow_stub:;
+  P = (gcptr)(v - 2);
+  assert(!(P->h_tid & GCFLAG_PUBLIC));
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+    {
+      fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p handle "
+              "private_from_protected\n", G, P);
+      goto private_from_protected;
+    }
+  else
+    {
+      fprintf(stderr, "read_barrier: %p -> %p handle\n", G, P);
+      goto register_in_list_of_read_objects;
     }
 }
-#endif
 
 #if 0
 void *stm_DirectReadBarrierFromR(void *G1, void *R_Container1, size_t offset)
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -164,7 +164,9 @@
 gcptr stm_DirectReadBarrier(gcptr);
 gcptr stm_RepeatReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
-gcptr _stm_nonrecord_barrier(gcptr, int *);
+gcptr _stm_nonrecord_barrier(gcptr);  /* debugging: read barrier, but
+                                         not recording anything */
+int _stm_is_private(gcptr);  /* debugging */
 gcptr stm_get_private_from_protected(long);  /* debugging */
 gcptr stm_get_read_obj(long);  /* debugging */
 gcptr stmgc_duplicate(gcptr);
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -96,3 +96,20 @@
     }
     gcptrlist_clear(&d->public_descriptor->stolen_objects);
 }
+
+gcptr _stm_find_stolen_objects(struct tx_descriptor *d, gcptr obj)
+{
+    /* read-only, for debugging */
+    long i, size = d->public_descriptor->stolen_objects.size;
+    gcptr *items = d->public_descriptor->stolen_objects.items;
+
+    for (i = 0; i < size; i += 2) {
+        gcptr B = items[i];
+        gcptr L = items[i + 1];
+
+        assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        if (B == obj)
+            return L;
+    }
+    return NULL;
+}
diff --git a/c4/steal.h b/c4/steal.h
--- a/c4/steal.h
+++ b/c4/steal.h
@@ -11,6 +11,7 @@
 void stm_steal_stub(gcptr);
 gcptr stm_get_stolen_obj(long index);   /* debugging */
 void stm_normalize_stolen_objects(struct tx_descriptor *);
+gcptr _stm_find_stolen_objects(struct tx_descriptor *, gcptr);
 
 
 #endif
diff --git a/c3/test/model.py b/c4/test/model.py
copy from c3/test/model.py
copy to c4/test/model.py
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -63,7 +63,8 @@
     //void stmgcpage_add_prebuilt_root(gcptr);
     void stm_clear_between_tests(void);
     //void stmgc_minor_collect(void);
-    //gcptr _stm_nonrecord_barrier(gcptr, int *);
+    gcptr _stm_nonrecord_barrier(gcptr);
+    int _stm_is_private(gcptr);
     int stm_dbgmem_is_active(void *p, int allow_outside);
     void stm_start_sharedlock(void);
     void stm_stop_sharedlock(void);
@@ -436,7 +437,7 @@
 def nalloc_refs(nrefs):
     "Allocate a fresh object from the nursery, with nrefs pointers"
     p = lib.stm_allocate(HDR + WORD * nrefs, 421 + nrefs)
-    assert p.h_revision == lib.get_local_revision()
+    assert p.h_revision == lib.get_private_rev_num()
     for i in range(nrefs):
         assert rawgetptr(p, i) == ffi.NULL   # must already be zero-filled
     return p
diff --git a/c3/test/test_random.py b/c4/test/test_random.py
copy from c3/test/test_random.py
copy to c4/test/test_random.py
--- a/c3/test/test_random.py
+++ b/c4/test/test_random.py
@@ -191,12 +191,10 @@
         return newextra
 
     def nonrecord_barrier(self, ptr):
-        result = ffi.new("int *")
-        ptr = lib._stm_nonrecord_barrier(ptr, result)
-        return ptr, result[0]
+        return lib._stm_nonrecord_barrier(ptr)
 
     def is_private(self, ptr):
-        return ptr.h_revision == lib.get_local_revision()
+        return lib._stm_is_private(ptr)
 
     def check_valid(self, lst):
         lst = list(lst)
@@ -207,23 +205,18 @@
                 continue
             self.check(p)
 
-            ptr, result = self.nonrecord_barrier(p.ptr)
-            if ptr == ffi.NULL and result == 3:
-                continue    # can't check anything: we'd need foreign access
+            ptr = self.nonrecord_barrier(p.ptr)
             has_private_copy = p.obj in self.current_rev.content
-            assert has_private_copy == (result >= 1)
             if has_private_copy:
+                assert ptr != ffi.NULL and self.is_private(ptr)
                 content = self.current_rev.content[p.obj]
             else:
                 try:
                     content = self.current_rev._try_read(p.obj)
-                    is_too_recent = False
                 except model.Deleted:
-                    is_too_recent = True
-                if result < 0:
-                    assert is_too_recent
-                if is_too_recent:
-                    continue     # can't really check more in this case
+                    assert ptr == ffi.NULL
+                    continue
+                assert ptr != ffi.NULL and not self.is_private(ptr)
 
             self.check_not_free(ptr)
             assert lib.rawgetptr(ptr, 2) == p.obj.identity

From noreply at buildbot.pypy.org  Tue Jun 11 23:09:50 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Jun 2013 23:09:50 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fixes. Now I have a clear idea about
 which thread can change exactly
Message-ID: <20130611210950.5BF731C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r93:2d0f4e06ce3a
Date: 2013-06-11 22:49 +0200
http://bitbucket.org/pypy/stmgc/changeset/2d0f4e06ce3a/

Log:	Fixes. Now I have a clear idea about which thread can change
	exactly which parts of which objects.

diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt
--- a/c4/doc-objects.txt
+++ b/c4/doc-objects.txt
@@ -245,3 +245,28 @@
     make a stub with h_revision = private object | 2
 
     after a CPU write barrier, make the public h_revision to point to the stub
+
+
+
+Change to the flags and h_revision
+----------------------------------
+
+The flags are in `h_tid`.  Changes to this field and `h_revision` must
+not occur uncontrolled:
+
+- private copies: the thread that owns the private copy can change
+freely the `h_tid` and `h_revision` fields.  The other threads must not
+touch them, and must read them carefully.  This concerns only stealing
+threads, on GCFLAG_PRIVATE_FROM_PROTECTED objects.  The flag
+GCFLAG_PRIVATE_FROM_PROTECTED itself is only changed when the owning
+thread has got its collection_lock, and as long as it is set, h_revision
+points to the backup copy.
+
+- protected copies (includes backup copies): any change requires the
+owning thread's collection_lock.  During stealing, other threads
+might add (with the collection_lock) the flags GCFLAG_PUBLIC or
+GCFLAG_PUBLIC_TO_PRIVATE.
+
+- public copies: must be changed carefully: `h_tid` is only modified to
+add GCFLAG_PUBLIC_TO_PRIVATE; and `h_revision` changes are done with
+bool_cas() in a thread-controlled way.
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -75,9 +75,11 @@
   gcptr P = G;
   revision_t v;
 
+ restart_all:
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
-    private_from_protected:
+      assert(!(P->h_revision & 1));   /* pointer to the backup copy */
+
       /* check P->h_revision->h_revision: if a pointer, then it means
          the backup copy has been stolen into a public object and then
          modified by some other thread.  Abort. */
@@ -86,15 +88,20 @@
 
       goto add_in_recent_reads_cache;
     }
+  /* else, for the rest of this function, we can assume that P was not
+     a private copy */
 
   if (P->h_tid & GCFLAG_PUBLIC)
     {
       /* follow the chained list of h_revision's as long as they are
-         regular pointers */
-    retry:
+         regular pointers.  We will only find more public objects
+         along this chain.
+      */
+    restart_all_public:
       v = ACCESS_ONCE(P->h_revision);
       if (!(v & 1))  // "is a pointer", i.e.
         {            //      "has a more recent revision"
+        retry:
           if (v & 2)
             goto follow_stub;
 
@@ -114,23 +121,33 @@
                  doing this write occasionally based on a counter in d */
               P_prev->h_revision = v;
               P = (gcptr)v;
-              goto retry;
+              v = ACCESS_ONCE(P->h_revision);
+              if (!(v & 1))  // "is a pointer", i.e. "has a more recent rev"
+                goto retry;
+            }
+
+          /* We reach this point if P != G only.  Check again the
+             read_barrier_cache: if P now hits the cache, just return it
+          */
+          if (FXCACHE_AT(P) == P)
+            {
+              fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P);
+              return P;
             }
         }
 
-      /* if we land on a P in read_barrier_cache: just return it */
-      if (FXCACHE_AT(P) == P)
-        {
-          fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P);
-          return P;
-        }
-
+      /* If we land on a P with GCFLAG_PUBLIC_TO_PRIVATE, it might be
+         because *we* have an entry in d->public_to_private.  (It might
+         also be someone else.)
+      */
       if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
         {
           wlog_t *item;
         retry_public_to_private:;
           G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
 
+          /* We have a key in 'public_to_private'.  The value is the
+             corresponding private object. */
           P = item->val;
           assert(!(P->h_tid & GCFLAG_PUBLIC));
           assert(is_private(P));
@@ -138,6 +155,8 @@
           return P;
 
         no_private_obj:
+          /* Key not found.  It might be because there really is none, or
+             because we still have it waiting in 'stolen_objects'. */
           if (d->public_descriptor->stolen_objects.size > 0)
             {
               spinlock_acquire(d->public_descriptor->collection_lock, 'N');
@@ -147,7 +166,8 @@
             }
         }
 
-      if (UNLIKELY(v > d->start_time))   // object too recent?
+      /* The answer is a public object.  Is it too recent? */
+      if (UNLIKELY(v > d->start_time))
         {
           if (v >= LOCKED)
             {
@@ -161,10 +181,17 @@
     }
   else
     {
+      /* Not private and not public: it's a protected object
+       */
       fprintf(stderr, "read_barrier: %p -> %p protected\n", G, P);
+
+      /* The risks are not high, but in parallel it's possible for the
+         object to be stolen by another thread and become public, after
+         which it can be outdated by another commit.  So the following
+         assert can actually fail in that case. */
+      /*assert(P->h_revision & 1);*/
     }
 
- register_in_list_of_read_objects:
   gcptrlist_insert(&d->list_of_read_objects, P);
 
  add_in_recent_reads_cache:
@@ -172,43 +199,32 @@
   return P;
 
  follow_stub:;
+  /* We know that P is a stub object, because only stubs can have
+     an h_revision that is == 2 mod 4.
+  */
   struct tx_public_descriptor *foreign_pd = STUB_THREAD(P);
   if (foreign_pd == d->public_descriptor)
     {
-      /* same thread */
+      /* Same thread: dereference the pointer directly.  It's possible
+         we reach any kind of object, even a public object, in case it
+         was stolen.  So we just repeat the whole procedure. */
       P = (gcptr)(v - 2);
-      assert(!(P->h_tid & GCFLAG_PUBLIC));
-      if (P->h_revision == stm_private_rev_num)
-        {
-          fprintf(stderr, "read_barrier: %p -> %p handle "
-                  "private\n", G, P);
-          return P;
-        }
-      else if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
-        {
-          fprintf(stderr, "read_barrier: %p -> %p handle "
-                  "private_from_protected\n", G, P);
-          goto private_from_protected;
-        }
-      else if (FXCACHE_AT(P) == P)
-        {
-          fprintf(stderr, "read_barrier: %p -> %p handle "
-                  "protected fxcache\n", G, P);
-          return P;
-        }
-      else
-        {
-          fprintf(stderr, "read_barrier: %p -> %p handle "
-                  "protected\n", G, P);
-          goto register_in_list_of_read_objects;
-        }
+      fprintf(stderr, "read_barrier: %p -> %p via stub\n  ", G, P);
+
+      if (UNLIKELY((P->h_revision != stm_private_rev_num) &&
+                   (FXCACHE_AT(P) != P)))
+        goto restart_all;
+
+      return P;
     }
   else
     {
       /* stealing */
       fprintf(stderr, "read_barrier: %p -> stealing %p...\n  ", G, P);
       stm_steal_stub(P);
-      goto retry;
+
+      assert(P->h_tid & GCFLAG_PUBLIC);
+      goto restart_all_public;
     }
 }
 
@@ -327,21 +343,12 @@
   return L;
 }
 
-static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R);
-
 static gcptr LocalizeProtected(struct tx_descriptor *d, gcptr P)
 {
   gcptr B;
-  spinlock_acquire(d->public_descriptor->collection_lock, 'L');
-
-  if (P->h_tid & GCFLAG_PUBLIC)
-    {
-      /* became PUBLIC while waiting for the collection_lock */
-      spinlock_release(d->public_descriptor->collection_lock);
-      return LocalizePublic(d, P);
-    }
 
   assert(P->h_revision != stm_private_rev_num);
+  assert(P->h_revision & 1);
   assert(!(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
   assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(P->h_tid & GCFLAG_STUB));
@@ -355,7 +362,6 @@
 
   gcptrlist_insert(&d->private_from_protected, P);
 
-  spinlock_release(d->public_descriptor->collection_lock);
   return P;
 }
 
@@ -406,11 +412,17 @@
   if (is_private(R))
     return R;
 
+  spinlock_acquire(d->public_descriptor->collection_lock, 'L');
+  if (d->public_descriptor->stolen_objects.size != 0)
+    stm_normalize_stolen_objects(d);
+
   if (R->h_tid & GCFLAG_PUBLIC)
     W = LocalizePublic(d, R);
   else
     W = LocalizeProtected(d, R);
 
+  spinlock_release(d->public_descriptor->collection_lock);
+
   fprintf(stderr, "write_barrier: %p -> %p -> %p\n", P, R, W);
 
   return W;
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -55,25 +55,60 @@
 
     gcptr L = (gcptr)(v - 2);
 
+    /* L might be a private_from_protected, or just a protected copy.
+       To know which case it is, read GCFLAG_PRIVATE_FROM_PROTECTED.
+    */
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
+
+        /* B is now a backup copy, i.e. a protected object, and we own
+           the foreign thread's collection_lock, so we can read/write the
+           flags
+        */
+        assert(B->h_tid & GCFLAG_BACKUP_COPY);
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
 
-        /* add {B: L} in 'public_to_private', but lazily, because we don't
-           want to walk over the feet of the foreign thread */
-        B->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
-        gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
-
+        if (B->h_tid & GCFLAG_PUBLIC_TO_PRIVATE) {
+            /* already stolen */
+        }
+        else {
+            B->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
+            /* add {B: L} in 'public_to_private', but lazily, because we
+               don't want to walk over the feet of the foreign thread
+            */
+            gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
+        }
         fprintf(stderr, "stolen: %p -> %p - - -> %p\n", P, B, L);
-
         L = B;
     }
 
-    /* change L from protected to public */
+    /* Here L is a protected (or backup) copy, and we own the foreign
+       thread's collection_lock, so we can read/write the flags.  Change
+       it from protected to public.
+    */
     L->h_tid |= GCFLAG_PUBLIC;
 
-    smp_wmb();      /* the following update must occur "after" the flag
-                       GCFLAG_PUBLIC was added, for other threads */
+    /* Note that all protected or backup copies have a h_revision that
+       is odd.
+    */
+    assert(L->h_revision & 1);
+
+    /* At this point, the object can only be seen by its owning foreign
+       thread and by us.  No 3rd thread can see it as long as we own
+       the foreign thread's collection_lock.  For the foreign thread,
+       it might suddenly see the GCFLAG_PUBLIC being added to L
+       (but it may not do any change to the flags itself, because
+       it cannot grab its own collection_lock).  L->h_revision is an
+       odd number that is also valid on a public up-to-date object.
+    */
+
+    /* If another thread (the foreign or a 3rd party) does a read
+       barrier from P, it must only reach L if all writes to L are
+       visible; i.e. it must not see P->h_revision => L that still
+       doesn't have the GCFLAG_PUBLIC.  So we need a CPU write
+       barrier here.
+    */
+    smp_wmb();
 
     /* update the original P->h_revision to point directly to L */
     P->h_revision = (revision_t)L;
@@ -92,6 +127,8 @@
         gcptr L = items[i + 1];
 
         assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
+
         g2l_insert(&d->public_to_private, B, L);
     }
     gcptrlist_clear(&d->public_descriptor->stolen_objects);

From noreply at buildbot.pypy.org  Wed Jun 12 01:39:05 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 12 Jun 2013 01:39:05 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: py3k status update #11
Message-ID: <20130611233905.0C2C61C02BA@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: extradoc
Changeset: r4975:bce4d23f15e1
Date: 2013-06-11 16:38 -0700
http://bitbucket.org/pypy/extradoc/changeset/bce4d23f15e1/

Log:	py3k status update #11

diff --git a/blog/draft/py3k-status-update-11.rst b/blog/draft/py3k-status-update-11.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/py3k-status-update-11.rst
@@ -0,0 +1,63 @@
+Py3k status update #11
+----------------------
+
+This is the 11th status update about our work on the `py3k branch`_, which we
+can work on thanks to all of the people who donated_ to the `py3k proposal`_.
+
+Here's some highlights of the progress made since the previous update:
+
+* PyPy py3k now matches CPython 3's hash code for
+  int/float/complex/Decimal/Fraction
+
+* Various outstanding unicode identifier related issues were
+  resolved. E.g. test_importlib/pep263/ucn/unicode all now fully pass. Various
+  usage of identifiers (in particular type and module names) have been fixed to
+  handle non-ascii names -- mostly around display of reprs and exception
+  messages.
+
+* The unicodedata database has been upgraded to 6.0.0.
+
+* Windows support has greatly improved, though it could still use some more
+  help (but so does the default branch to a certain degree).
+
+* Probably the last of the parsing related bugs/features have been taken care
+  of.
+
+* Of course various other smaller miscellaneous fixes
+
+This leaves the branch w/ only about 5 outstanding failures of the stdlib test
+suite:
+
+* test_float
+
+  1 failing test about containment of floats in collections.
+
+* test_memoryview
+
+  Various failures: requires some bytes/str changes among other things (Manuel
+  Jacob's has some progress on this on the `py3k-memoryview branch`_)
+
+* test_multiprocessing
+
+  1 or more tests deadlock on some platforms
+
+* test_sys and test_threading
+
+  2 failing tests for the New GIL's new API
+
+Probably the biggest feature left to tackle is the New GIL.
+
+We're now pretty close to pushing an initial release. We had planned for one
+around PyCon, but having missed that we've put some more effort into the branch
+to provide a more fully-fledged initial release.
+
+Thanks to the following for their contributions: Manuel Jacob, Amaury Forgeot
+d'Arc, Karl Ramm, Jason Chu and Christian Hudson.
+
+cheers,
+Phil
+
+.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html
+.. _`py3k proposal`: http://pypy.org/py3donate.html
+.. _`py3k branch`: https://bitbucket.org/pypy/pypy/commits/all/tip/branch%28%22py3k%22%29
+.. _`py3k-memoryview branch`: https://bitbucket.org/pypy/pypy/compare/py3k-memoryview..py3k

From noreply at buildbot.pypy.org  Wed Jun 12 02:01:01 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 12 Jun 2013 02:01:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130612000101.898CF1C0328@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64858:f1b58638d8bd
Date: 2013-06-11 17:00 -0700
http://bitbucket.org/pypy/pypy/changeset/f1b58638d8bd/

Log:	merge default

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -289,16 +289,6 @@
                "with verify() (see pypy/module/_cffi_backend/ctypefunc.py "
                "for details)"))
 
-        if USE_C_LIBFFI_MSVC and is_result_type:
-            # MSVC returns small structures in registers.  Pretend int32 or
-            # int64 return type.  This is needed as a workaround for what
-            # is really a bug of libffi_msvc seen as an independent library
-            # (ctypes has a similar workaround).
-            if ctype.size <= 4:
-                return clibffi.ffi_type_sint32
-            if ctype.size <= 8:
-                return clibffi.ffi_type_sint64
-
         # walk the fields, expanding arrays into repetitions; first,
         # only count how many flattened fields there are
         nflat = 0
@@ -318,6 +308,16 @@
                                "a struct with a zero-length array"))
             nflat += flat
 
+        if USE_C_LIBFFI_MSVC and is_result_type:
+            # MSVC returns small structures in registers.  Pretend int32 or
+            # int64 return type.  This is needed as a workaround for what
+            # is really a bug of libffi_msvc seen as an independent library
+            # (ctypes has a similar workaround).
+            if ctype.size <= 4:
+                return clibffi.ffi_type_sint32
+            if ctype.size <= 8:
+                return clibffi.ffi_type_sint64
+
         # allocate an array of (nflat + 1) ffi_types
         elements = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * (nflat + 1))
         elements = rffi.cast(FFI_TYPE_PP, elements)
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -217,6 +217,52 @@
                     pass
             rmtree(dir_name, True)
 
+    def test_builtin_reimport(self):
+        # from https://bugs.pypy.org/issue1514
+        skip("fix me")
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        __import__('marshal')
+        sys.modules['marshal'] = saved
+
+        assert marshal.loads == 42
+        import marshal
+        assert marshal.loads == 42
+        marshal.loads = old
+
+    def test_builtin_reimport_mess(self):
+        # taken from https://bugs.pypy.org/issue1514, with extra cases
+        # that show a difference with CPython: we can get on CPython
+        # several module objects for the same built-in module :-(
+        skip("several built-in module objects: not supported by pypy")
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        marshal2 = __import__('marshal')
+        assert marshal2 is not marshal
+        assert marshal2.loads is old
+        assert marshal2 is sys.modules['marshal']
+        assert marshal is saved
+        assert marshal.loads == 42
+
+        import marshal
+        assert marshal.loads is old
+
+        sys.modules['marshal'] = saved
+        import marshal
+        assert marshal.loads == 42
+
+        marshal.loads = old
+
     def test_get_tag(self):
         import imp
         import sys
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -625,6 +625,43 @@
         assert sys.path is oldpath
         assert 'settrace' in dir(sys)
 
+    def test_reload_builtin_doesnt_clear(self):
+        import sys
+        sys.foobar = "baz"
+        reload(sys)
+        assert sys.foobar == "baz"
+
+    def test_reimport_builtin_simple_case_1(self):
+        import sys, time
+        del time.tzset
+        del sys.modules['time']
+        import time
+        assert hasattr(time, 'tzset')
+
+    def test_reimport_builtin_simple_case_2(self):
+        skip("fix me")
+        import sys, time
+        time.foo = "bar"
+        del sys.modules['time']
+        import time
+        assert not hasattr(time, 'foo')
+
+    def test_reimport_builtin(self):
+        skip("fix me")
+        import sys, time
+        oldpath = sys.path
+        time.tzset = "<test_reimport_builtin removed this>"
+
+        del sys.modules['time']
+        import time as time1
+        assert sys.modules['time'] is time1
+
+        assert time.tzset == "<test_reimport_builtin removed this>"
+
+        reload(time1)   # don't leave a broken time.tzset behind
+        import time
+        assert time.tzset != "<test_reimport_builtin removed this>"
+
     def test_reload_infinite(self):
         import infinite_reload
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -97,11 +97,6 @@
         '_xoptions'             : 'app.null__xoptions',
     }
 
-    def setbuiltinmodule(self, w_module, name):
-        w_name = self.space.wrap(name)
-        w_modules = self.get('modules')
-        self.space.setitem(w_modules, w_name, w_module)
-
     def startup(self, space):
         if space.config.translating and not we_are_translated():
             # don't get the filesystemencoding at translation time
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -33,14 +33,13 @@
     try:
         # first try to print the exception's class name
         stderr = sys.stderr
-        stderr.write(getattr(exctype, '__name__', exctype))
+        stderr.write(str(getattr(exctype, '__name__', exctype)))
         # then attempt to get the str() of the exception
         try:
             s = str(value)
         except:
             s = '<failure of str() on the exception instance>'
-        # then print it, and don't worry too much about the extra space
-        # between the exception class and the ':'
+        # then print it
         if s:
             stderr.write(': %s\n' % (s,))
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -41,40 +41,43 @@
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
+        # NOTE: 'expected_*' is the numbers expected from GCC.
+        # The numbers expected from MSVC are not explicitly written
+        # in this file, and will just be taken from the compiler.
         ffi = FFI()
         ffi.cdef("struct s1 { %s };" % source)
         ctype = ffi.typeof("struct s1")
         # verify the information with gcc
-        if sys.platform != "win32":
-            ffi1 = FFI()
-            ffi1.cdef("""
-                static const int Gofs_y, Galign, Gsize;
-                struct s1 *try_with_value(int fieldnum, long long value);
-            """)
-            fnames = [name for name, cfield in ctype.fields
-                           if name and cfield.bitsize > 0]
-            setters = ['case %d: s.%s = value; break;' % iname
-                       for iname in enumerate(fnames)]
-            lib = ffi1.verify("""
-                struct s1 { %s };
-                struct sa { char a; struct s1 b; };
-                #define Gofs_y  offsetof(struct s1, y)
-                #define Galign  offsetof(struct sa, b)
-                #define Gsize   sizeof(struct s1)
-                struct s1 *try_with_value(int fieldnum, long long value)
-                {
-                    static struct s1 s;
-                    memset(&s, 0, sizeof(s));
-                    switch (fieldnum) { %s }
-                    return &s;
-                }
-            """ % (source, ' '.join(setters)))
-            assert lib.Gofs_y == expected_ofs_y
-            assert lib.Galign == expected_align
-            assert lib.Gsize  == expected_size
+        ffi1 = FFI()
+        ffi1.cdef("""
+            static const int Gofs_y, Galign, Gsize;
+            struct s1 *try_with_value(int fieldnum, long long value);
+        """)
+        fnames = [name for name, cfield in ctype.fields
+                       if name and cfield.bitsize > 0]
+        setters = ['case %d: s.%s = value; break;' % iname
+                   for iname in enumerate(fnames)]
+        lib = ffi1.verify("""
+            struct s1 { %s };
+            struct sa { char a; struct s1 b; };
+            #define Gofs_y  offsetof(struct s1, y)
+            #define Galign  offsetof(struct sa, b)
+            #define Gsize   sizeof(struct s1)
+            struct s1 *try_with_value(int fieldnum, long long value)
+            {
+                static struct s1 s;
+                memset(&s, 0, sizeof(s));
+                switch (fieldnum) { %s }
+                return &s;
+            }
+        """ % (source, ' '.join(setters)))
+        if sys.platform == 'win32':
+            expected_ofs_y = lib.Gofs_y
+            expected_align = lib.Galign
+            expected_size  = lib.Gsize
         else:
-            lib = None
-            fnames = None
+            assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (
+                expected_ofs_y, expected_align, expected_size)
         # the real test follows
         assert ffi.offsetof("struct s1", "y") == expected_ofs_y
         assert ffi.alignof("struct s1") == expected_align
@@ -99,10 +102,9 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
-        if lib is not None:
-            t = lib.try_with_value(fnames.index(name), value)
-            raw2 = ffi.buffer(t, len(raw1))[:]
-            assert raw1 == raw2
+        t = lib.try_with_value(fnames.index(name), value)
+        raw2 = ffi.buffer(t, len(raw1))[:]
+        assert raw1 == raw2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
@@ -136,9 +138,11 @@
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
         self.check("char x; int :0; char y;", 4, 1, 5)
+        self.check("char x; int :0; int :0; char y;", 4, 1, 5)
         self.check("char x; long long :0; char y;", L, 1, L + 1)
         self.check("short x, y; int :0; int :0;", 2, 2, 4)
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
     def test_error_cases(self):
         ffi = FFI()
diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
@@ -2,7 +2,6 @@
 import weakref
 from rpython.rlib.jit import JitDriver, dont_look_inside
 from rpython.jit.backend.llsupport.test.zrpy_gc_test import run, get_entry, compile
-from rpython.jit.backend.llsupport.test.ztranslation_test import fix_annotator_for_vrawbuffer
 
 class X(object):
     def __init__(self, x=0):
@@ -32,8 +31,7 @@
     g._dont_inline_ = True
     return g
 
-def compile_boehm_test(monkeypatch):
-    fix_annotator_for_vrawbuffer(monkeypatch)
+def compile_boehm_test():
     myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
     @dont_look_inside
     def see(lst, n):
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -9,23 +9,10 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 
 
-def fix_annotator_for_vrawbuffer(monkeypatch):
-    from rpython.rlib.nonconst import NonConstant
-    from rpython.jit.metainterp.optimizeopt.virtualize import VRawBufferValue
-    from rpython.jit.metainterp import warmspot
-
-    def my_hook_for_tests(cpu):
-        # this is needed so that the annotator can see it
-        if NonConstant(False):
-            v = VRawBufferValue(cpu, None, -1, None, None)
-    monkeypatch.setattr(warmspot, 'hook_for_tests', my_hook_for_tests)
-
-
 class TranslationTest(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_stuff_translates(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_stuff_translates(self):
         # this is a basic test that tries to hit a number of features and their
         # translation:
         # - jitting of loops and bridges
@@ -102,10 +89,9 @@
 class TranslationTestCallAssembler(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_direct_assembler_call_translates(self, monkeypatch):
+    def test_direct_assembler_call_translates(self):
         """Test CALL_ASSEMBLER and the recursion limit"""
         from rpython.rlib.rstackovf import StackOverflow
-        fix_annotator_for_vrawbuffer(monkeypatch)
 
         class Thing(object):
             def __init__(self, val):
@@ -183,8 +169,7 @@
 class TranslationTestJITStats(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_jit_get_stats(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_jit_get_stats(self):
         driver = JitDriver(greens = [], reds = ['i'])
 
         def f():
@@ -207,8 +192,7 @@
 class TranslationRemoveTypePtrTest(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_external_exception_handling_translates(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_external_exception_handling_translates(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'total'])
 
         class ImDone(Exception):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -17,6 +17,7 @@
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
+    is_about_raw = False
     _cached_vinfo = None
 
     def __init__(self, keybox, source_op=None):
@@ -395,6 +396,7 @@
 
 
 class VRawBufferValue(AbstractVArrayValue):
+    is_about_raw = True
 
     def __init__(self, cpu, logops, size, keybox, source_op):
         AbstractVirtualValue.__init__(self, keybox, source_op)
@@ -457,6 +459,7 @@
 
 
 class VRawSliceValue(AbstractVirtualValue):
+    is_about_raw = True
 
     def __init__(self, rawbuffer_value, offset, keybox, source_op):
         AbstractVirtualValue.__init__(self, keybox, source_op)
@@ -676,13 +679,17 @@
         offsetbox = self.get_constant_box(op.getarg(1))
         if value.is_virtual() and offsetbox is not None:
             offset = offsetbox.getint()
-            if isinstance(value, VRawBufferValue):
-                self.make_virtual_raw_slice(value, offset, op.result, op)
-                return
-            elif isinstance(value, VRawSliceValue):
-                offset = offset + value.offset
-                self.make_virtual_raw_slice(value.rawbuffer_value, offset, op.result, op)
-                return
+            # the following check is constant-folded to False if the
+            # translation occurs without any VRawXxxValue instance around
+            if value.is_about_raw:
+                if isinstance(value, VRawBufferValue):
+                    self.make_virtual_raw_slice(value, offset, op.result, op)
+                    return
+                elif isinstance(value, VRawSliceValue):
+                    offset = offset + value.offset
+                    self.make_virtual_raw_slice(value.rawbuffer_value, offset,
+                                                op.result, op)
+                    return
         self.emit_operation(op)
 
     def optimize_ARRAYLEN_GC(self, op):
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -451,6 +451,7 @@
 
 class AbstractVirtualInfo(object):
     kind = REF
+    is_about_raw = False
     #def allocate(self, decoder, index):
     #    raise NotImplementedError
     def equals(self, fieldnums):
@@ -461,7 +462,7 @@
 
     def debug_prints(self):
         raise NotImplementedError
-        
+
 
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
     def __init__(self, fielddescrs):
@@ -547,6 +548,7 @@
 
 class VRawBufferStateInfo(AbstractVirtualInfo):
     kind = INT
+    is_about_raw = True
     
     def __init__(self, size, offsets, descrs):
         self.size = size
@@ -772,7 +774,9 @@
         assert self.virtuals_cache is not None
         v = self.virtuals_cache.get_int(index)
         if not v:
-            v = self.rd_virtuals[index].allocate_int(self, index)
+            v = self.rd_virtuals[index]
+            assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
+            v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")
         return v
 
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -761,8 +761,6 @@
         cpu = jd.warmstate.cpu
 
         def ll_portal_runner(*args):
-            hook_for_tests(cpu) # usually it's empty, but tests can monkeypatch
-                                # it to fix the annotator
             start = True
             while 1:
                 try:
@@ -999,10 +997,3 @@
         graphs = self.translator.graphs
         for graph, block, i in find_force_quasi_immutable(graphs):
             self.replace_force_quasiimmut_with_direct_call(block.operations[i])
-
-def hook_for_tests(cpu):
-    """
-    This function is empty and does nothing. Its only role is to be
-    monkey-patched by tests to "fix" the annotator if needed (see
-    e.g. x86/test/test_ztranslation::test_external_exception_handling_translates
-    """

From noreply at buildbot.pypy.org  Wed Jun 12 02:16:20 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 12 Jun 2013 02:16:20 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: simplify
Message-ID: <20130612001620.EC0D61C0F12@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64859:391ba71b1425
Date: 2013-06-11 17:14 -0700
http://bitbucket.org/pypy/pypy/changeset/391ba71b1425/

Log:	simplify

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -379,7 +379,6 @@
     return tuple(parts), tuple(formats)
 
 def get_operrcls2(valuefmt):
-    from rpython.rlib.runicode import str_decode_utf_8
     valuefmt = valuefmt.decode('ascii')
     strings, formats = decompose_valuefmt(valuefmt)
     assert len(strings) == len(formats) + 1
@@ -407,8 +406,7 @@
                     if fmt == 'd':
                         result = str(value).decode('ascii')
                     elif fmt == '8':
-                        result, _ = str_decode_utf_8(value, len(value),
-                                                     'strict')
+                        result = value.decode('utf-8')
                     elif fmt == 'R':
                         result = space.unicode_w(space.repr(value))
                     elif fmt in 'NT':
@@ -440,7 +438,7 @@
 
     Supports the standard %s and %d formats, plus the following:
 
-    %8 - The result of arg.decode('utf-8', 'strict')
+    %8 - The result of arg.decode('utf-8')
     %N - The result of w_arg.getname(space)
     %R - The result of space.unicode_w(space.repr(w_arg))
     %T - The result of space.type(w_arg).getname(space)

From noreply at buildbot.pypy.org  Wed Jun 12 10:20:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 10:20:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Check the type of the text arguments,
 and cast unicodes to strings in
Message-ID: <20130612082021.1032B1C00B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64860:0c8571de8df4
Date: 2013-06-12 10:19 +0200
http://bitbucket.org/pypy/pypy/changeset/0c8571de8df4/

Log:	Check the type of the text arguments, and cast unicodes to strings
	in the default encoding (as CPython). Fix for issue 1513.

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 

From noreply at buildbot.pypy.org  Wed Jun 12 11:22:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 11:22:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Wrote bogus value: it needs to be
 new_revision (which is cur_time + 1).
Message-ID: <20130612092252.E292B1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94:0bdc2d4e6f29
Date: 2013-06-12 11:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/0bdc2d4e6f29/

Log:	Wrote bogus value: it needs to be new_revision (which is cur_time +
	1).

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -839,6 +839,8 @@
 {
   long i, size = d->private_from_protected.size;
   gcptr *items = d->private_from_protected.items;
+  revision_t new_revision = cur_time + 1;     // make an odd number
+  assert(new_revision & 1);
 
   for (i = 0; i < size; i++)
     {
@@ -856,7 +858,7 @@
         }
 
       gcptr B = (gcptr)P->h_revision;
-      P->h_revision = cur_time;
+      P->h_revision = new_revision;
 
       if (B->h_tid & GCFLAG_PUBLIC)
         {

From noreply at buildbot.pypy.org  Wed Jun 12 11:25:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 11:25:51 +0200 (CEST)
Subject: [pypy-commit] stmgc default: The single-thread version passes now
Message-ID: <20130612092551.9DF7B1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r95:06effc1c372c
Date: 2013-06-12 11:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/06effc1c372c/

Log:	The single-thread version passes now

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -5,6 +5,11 @@
 # a default seed that changes every day, but that can be easily recovered
 DEFAULT_SEED = int(time.strftime("%y%m%d", time.gmtime()))
 
+# XXXXXXXXXXXXXXXXXXXXXX
+# XXX reintroduce me XXX
+# XXXXXXXXXXXXXXXXXXXXXX
+DO_MAJOR_COLLECTS = False
+
 
 def setup_function(_):
     lib.stm_clear_between_tests()
@@ -375,7 +380,8 @@
             if k1 == 7:
                 self.dump('major collect')
                 self.push_roots()
-                major_collect()
+                if DO_MAJOR_COLLECTS:
+                    major_collect()
                 self.pop_roots()
                 p = emptypair
             if k1 == 82 and self.interruptible_transaction:

From noreply at buildbot.pypy.org  Wed Jun 12 11:47:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 11:47:31 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix: wrong label
Message-ID: <20130612094731.9072E1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r96:1c52e9c6610d
Date: 2013-06-12 11:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/1c52e9c6610d/

Log:	Fix: wrong label

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -98,6 +98,7 @@
          along this chain.
       */
     restart_all_public:
+      assert(P->h_tid & GCFLAG_PUBLIC);
       v = ACCESS_ONCE(P->h_revision);
       if (!(v & 1))  // "is a pointer", i.e.
         {            //      "has a more recent revision"
@@ -172,10 +173,10 @@
           if (v >= LOCKED)
             {
               SpinLoop(SPLP_LOCKED_INFLIGHT);
-              goto retry;           // spinloop until it is no longer LOCKED
+              goto restart_all_public; // spinloop until it is no longer LOCKED
             }
           ValidateNow(d);                  // try to move start_time forward
-          goto retry;                      // restart searching from P
+          goto restart_all_public;         // restart searching from P
         }
       fprintf(stderr, "read_barrier: %p -> %p public\n", G, P);
     }

From noreply at buildbot.pypy.org  Wed Jun 12 11:47:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 11:47:32 +0200 (CEST)
Subject: [pypy-commit] stmgc default: reorganize
Message-ID: <20130612094732.BB7C31C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97:37a90d1c2610
Date: 2013-06-12 11:41 +0200
http://bitbucket.org/pypy/stmgc/changeset/37a90d1c2610/

Log:	reorganize

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -238,49 +238,40 @@
 
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
-    private_from_protected:
-      if (!(((gcptr)P->h_revision)->h_revision & 1))
-        {
-          fprintf(stderr, "_stm_nonrecord_barrier: %p -> NULL "
-                  "private_from_protected but protected changed\n", G);
-          return NULL;
-        }
-      goto add_in_recent_reads_cache;
+      assert(!(P->h_revision & 1));
+      return P;
     }
 
   if (P->h_tid & GCFLAG_PUBLIC)
     {
-      while (v = ACCESS_ONCE(P->h_revision), !(v & 1))
+      while (1)
         {
+          wlog_t *item;
+          gcptr L;
+          G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
+
+          L = item->val;
+        found_in_stolen_objects:
+          assert(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
+          assert(!(L->h_tid & GCFLAG_PUBLIC));
+          assert(is_private(L));
+          fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p "
+                  "public_to_private\n", G, L);
+          return L;
+
+        no_private_obj:;
+          L = _stm_find_stolen_objects(d, P);
+          if (L != NULL)
+            goto found_in_stolen_objects;
+
+          v = ACCESS_ONCE(P->h_revision);
+          if (v & 1)
+            break;
           if (v & 2)
             goto follow_stub;
-
           P = (gcptr)v;
           assert(P->h_tid & GCFLAG_PUBLIC);
         }
-
-      if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
-        {
-          wlog_t *item;
-          G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
-
-          P = item->val;
-        found_in_stolen_objects:
-          assert(!(P->h_tid & GCFLAG_PUBLIC));
-          assert(is_private(P));
-          fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p "
-                  "public_to_private\n", G, P);
-          return P;
-
-        no_private_obj:;
-          gcptr L = _stm_find_stolen_objects(d, P);
-          if (L != NULL)
-            {
-              P = L;
-              goto found_in_stolen_objects;
-            }
-        }
-
       if (UNLIKELY(v > d->start_time))
         {
           fprintf(stderr, "_stm_nonrecord_barrier: %p -> NULL changed\n", G);
@@ -292,25 +283,12 @@
     {
       fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p protected\n", G, P);
     }
-
- register_in_list_of_read_objects:
- add_in_recent_reads_cache:
   return P;
 
  follow_stub:;
+  fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p stub\n  ", G, P);
   P = (gcptr)(v - 2);
-  assert(!(P->h_tid & GCFLAG_PUBLIC));
-  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
-    {
-      fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p handle "
-              "private_from_protected\n", G, P);
-      goto private_from_protected;
-    }
-  else
-    {
-      fprintf(stderr, "read_barrier: %p -> %p handle\n", G, P);
-      goto register_in_list_of_read_objects;
-    }
+  return _stm_nonrecord_barrier(P);
 }
 
 #if 0

From noreply at buildbot.pypy.org  Wed Jun 12 11:47:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 11:47:33 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fixes in test
Message-ID: <20130612094733.E50461C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98:30dd165d9c04
Date: 2013-06-12 11:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/30dd165d9c04/

Log:	Fixes in test

diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -465,11 +465,9 @@
 def minor_collect():
     lib.stmgc_minor_collect()
 
-STUB_TID = GCFLAG_STUB | GCFLAG_OLD | 0   # no user tid
-
 def is_stub(p):
     assert lib.stm_dbgmem_is_active(p, 1) != 0
-    return p.h_tid == STUB_TID
+    return p.h_tid & GCFLAG_STUB
 
 def check_not_free(p):
     assert lib.stm_dbgmem_is_active(p, 1) == 1
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -321,3 +321,6 @@
 
 def test_abort_stealing_while_modifying():
     test_stealing_while_modifying(aborting=True)
+
+def test_stub_for_refs_from_stolen():
+    xxx
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -83,10 +83,11 @@
 
     def check(self, p):
         assert isinstance(p, Pair)
-        if p != emptypair and not is_stub(p.ptr):
+        if p != emptypair:
             self.check_not_free(p.ptr)
-            pid = lib.rawgetptr(p.ptr, 2)
-            assert pid == p.obj.identity
+            if not is_stub(p.ptr):
+                pid = lib.rawgetptr(p.ptr, 2)
+                assert pid == p.obj.identity
 
     def expected_abort(self, manual=False):
         if manual:

From noreply at buildbot.pypy.org  Wed Jun 12 13:10:47 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 12 Jun 2013 13:10:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: missing checks
Message-ID: <20130612111047.F08AD1C00B9@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64861:83dcd564b807
Date: 2013-06-12 04:20 -0500
http://bitbucket.org/pypy/pypy/changeset/83dcd564b807/

Log:	missing checks

diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -344,6 +344,8 @@
         # note: we need to make a copy of inputargs because possibly_free_vars
         # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
+        self.fm.finish_binding()
+        self._check_invariants()
 
     def get_gcmap(self, forbidden_regs=[], noregs=False):
         frame_depth = self.fm.get_frame_depth()

From noreply at buildbot.pypy.org  Wed Jun 12 13:10:52 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 12 Jun 2013 13:10:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20130612111052.22D6D1C00B9@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64862:574a2ace6557
Date: 2013-06-12 04:21 -0500
http://bitbucket.org/pypy/pypy/changeset/574a2ace6557/

Log:	merge heads

diff too long, truncating to 2000 out of 7890 lines

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -58,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -80,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -151,6 +151,8 @@
     'DEBUG': DEBUG,
     'NOTSET': NOTSET,
 }
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,16 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    try:
-                        self._sock._decref_socketios()
-                    except AttributeError:
-                        pass  # bah, someone built a _fileobject manually
-                              # with some unexpected replacement of the
-                              # _socketobject class
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_testcapi.py
@@ -0,0 +1,61 @@
+import os, sys, imp
+import tempfile
+
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
+    """
+    thisdir = os.path.dirname(__file__)
+    output_dir = tempfile.mkdtemp()
+
+    from distutils.ccompiler import new_compiler
+
+    compiler = new_compiler()
+    compiler.output_dir = output_dir
+
+    # Compile .c file
+    include_dir = os.path.join(thisdir, '..', 'include')
+    if sys.platform == 'win32':
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
+    else:
+        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
+    res = compiler.compile([os.path.join(thisdir, csource)],
+                           include_dirs=[include_dir],
+                           extra_preargs=ccflags)
+    object_filename = res[0]
+
+    # set link options
+    output_filename = modulename + _get_c_extension_suffix()
+    if sys.platform == 'win32':
+        # XXX libpypy-c.lib is currently not installed automatically
+        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
+        libraries = [library, 'oleaut32']
+        extra_ldargs = ['/MANIFEST',  # needed for VC10
+                        '/EXPORT:init' + modulename]
+    else:
+        libraries = []
+        extra_ldargs = []
+
+    # link the dynamic library
+    compiler.link_shared_object(
+        [object_filename],
+        output_filename,
+        libraries=libraries,
+        extra_preargs=extra_ldargs)
+
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,62 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
 try:
     import cpyext
 except ImportError:
     raise ImportError("No module named '_testcapi'")
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -156,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -701,7 +704,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
     # ----------
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -74,6 +74,9 @@
         class FFILibrary(types.ModuleType):
             _cffi_generic_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir
         library = FFILibrary("")
         #
         # finally, call the loaded_gen_xxx() functions.  This will set
@@ -168,21 +171,22 @@
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
-            if any(isinstance(type, model.StructOrUnion) for type in tp.args):
+            if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
-                for i, type in enumerate(tp.args):
-                    if isinstance(type, model.StructOrUnion):
-                        type = model.PointerType(type)
-                        indirections.append((i, type))
-                    indirect_args.append(type)
+                for i, typ in enumerate(tp.args):
+                    if isinstance(typ, model.StructOrUnion):
+                        typ = model.PointerType(typ)
+                        indirections.append((i, typ))
+                    indirect_args.append(typ)
                 tp = model.FunctionPtrType(tuple(indirect_args),
                                            tp.result, tp.ellipsis)
             BFunc = self.ffi._get_cached_btype(tp)
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
-            for i, type in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, type)
+            for i, typ in indirections:
+                newfunction = self._make_struct_wrapper(newfunction, i, typ)
         setattr(library, name, newfunction)
+        type(library)._cffi_dir.append(name)
 
     def _make_struct_wrapper(self, oldfunc, i, tp):
         backend = self.ffi._backend
@@ -390,6 +394,7 @@
         is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
         value = self._load_constant(is_int, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # enums
@@ -437,6 +442,7 @@
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
             setattr(library, enumerator, enumvalue)
+            type(library)._cffi_dir.append(enumerator)
 
     # ----------
     # macros: for now only for integers
@@ -450,6 +456,7 @@
     def _loaded_gen_macro(self, tp, name, module, library):
         value = self._load_constant(True, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # global variables
@@ -475,6 +482,7 @@
                 BArray = self.ffi._get_cached_btype(tp)
                 value = self.ffi.cast(BArray, value)
             setattr(library, name, value)
+            type(library)._cffi_dir.append(name)
             return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
@@ -486,7 +494,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
 cffimod_header = r'''
 #include <stdio.h>
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
new file mode 100644
--- /dev/null
+++ b/lib_pypy/greenlet.egg-info
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: greenlet
+Version: 0.4.0
+Summary: Lightweight in-process concurrent programming
+Home-page: https://github.com/python-greenlet/greenlet
+Author: Ralf Schmitt (for CPython), PyPy team
+Author-email: pypy-dev at python.org
+License: MIT License
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -32,11 +32,10 @@
      "rctime" , "select", "zipimport", "_lsprof",
      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
-     "thread", "itertools", "pyexpat", "_ssl", "array",
+     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
      "binascii", "_multiprocessing", '_warnings',
      "_collections", "_multibytecodec", "micronumpy", "_ffi",
-     "_continuation", "_cffi_backend", "_csv"] # "cpyext", "cppyy"]
-# disabled until problems are fixed
+     "_continuation", "_cffi_backend", "_csv", "cppyy"]
 ))
 
 translation_modules = default_modules.copy()
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,7 @@
+
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:
 
 * Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
 
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,7 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/doc/rffi.rst b/pypy/doc/rffi.rst
--- a/pypy/doc/rffi.rst
+++ b/pypy/doc/rffi.rst
@@ -5,7 +5,7 @@
 Purpose
 -------
 
-This document describes an FFI for RPython language, concentrating
+This document describes an FFI for the RPython language, concentrating
 on low-level backends like C. It describes
 how to declare and call low-level (C) functions from RPython level.
 
@@ -50,7 +50,7 @@
 ------
 
 In rffi_ there are various declared types for C-structures, like CCHARP
-(char*), SIZE_T (size_t) and others. refer to file for details. 
+(char*), SIZE_T (size_t) and others. Refer to file for details. 
 Instances of non-primitive types must be alloced by hand, with call 
 to lltype.malloc, and freed by lltype.free both with keyword argument 
 flavor='raw'. There are several helpers like string -> char*
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -40,3 +40,15 @@
 
 .. branch: on-abort-resops
 Added list of resops to the pypyjit on_abort hook.
+
+.. branch: logging-perf
+Speeds up the stdlib logging module
+
+.. branch: operrfmt-NT
+Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+
+.. branch: ctypes-byref
+Add the '_obj' attribute on ctypes pointer() and byref() objects
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -2,7 +2,7 @@
 PyPy on Windows
 ===============
 
-Pypy is supported on Windows platforms, starting with Windows 2000.
+PyPy is supported on Windows platforms, starting with Windows 2000.
 The following text gives some hints about how to translate the PyPy
 interpreter.
 
@@ -199,9 +199,9 @@
 
 or such, depending on your mingw64 download.
 
-hacking on Pypy with the mingw compiler
+hacking on PyPy with the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since hacking on Pypy means running tests, you will need a way to specify
+Since hacking on PyPy means running tests, you will need a way to specify
 the mingw compiler when hacking (as opposed to translating). As of
 March 2012, --cc is not a valid option for pytest.py. However if you set an
 environment variable CC to the compliter exe, testing will use it.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,8 +2,8 @@
 # App-level version of py.py.
 # See test/test_app_main.
 
-# Missing vs CPython: -d, -OO, -t, -v, -x, -3
-"""\
+# Missing vs CPython: -d, -t, -v, -x, -3
+USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 -c cmd : program passed in as string (terminates option list)
@@ -12,7 +12,8 @@
 -i     : inspect interactively after running script; forces a prompt even
          if stdin does not appear to be a terminal; also PYTHONINSPECT=x
 -m mod : run library module as a script (terminates option list)
--O     : dummy optimization flag for compatibility with CPython
+-O     : skip assert statements
+-OO    : remove docstrings when importing modules in addition to -O
 -R     : ignored (see http://bugs.python.org/issue14621)
 -Q arg : division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
@@ -27,7 +28,6 @@
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
 """
-USAGE1 = __doc__
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
 Other environment variables:
@@ -470,6 +470,10 @@
         sys.py3kwarning = bool(sys.flags.py3k_warning)
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
+        if sys.flags.optimize >= 1:
+            import __pypy__
+            __pypy__.set_debug(False)
+
         if sys.py3kwarning:
             print >> sys.stderr, (
                 "Warning: pypy does not implement py3k warnings")
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -86,12 +86,9 @@
             args_w = space.fixedview(w_stararg)
         except OperationError, e:
             if e.match(space, space.w_TypeError):
-                w_type = space.type(w_stararg)
-                typename = w_type.getname(space)
-                raise OperationError(
+                raise operationerrfmt(
                     space.w_TypeError,
-                    space.wrap("argument after * must be "
-                               "a sequence, not %s" % (typename,)))
+                    "argument after * must be a sequence, not %T", w_stararg)
             raise
         self.arguments_w = self.arguments_w + args_w
 
@@ -116,12 +113,10 @@
                 w_keys = space.call_method(w_starstararg, "keys")
             except OperationError, e:
                 if e.match(space, space.w_AttributeError):
-                    w_type = space.type(w_starstararg)
-                    typename = w_type.getname(space)
-                    raise OperationError(
+                    raise operationerrfmt(
                         space.w_TypeError,
-                        space.wrap("argument after ** must be "
-                                   "a mapping, not %s" % (typename,)))
+                        "argument after ** must be a mapping, not %T",
+                        w_starstararg)
                 raise
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -245,6 +245,8 @@
         if w_len is None:
             w_len = space.len(self.w_consts)
             space.setitem(self.w_consts, w_key, w_len)
+        if space.int_w(w_len) == 0:
+            self.scope.doc_removable = False
         return space.int_w(w_len)
 
     def _make_key(self, obj):
@@ -632,6 +634,7 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+    ops.JUMP_IF_NOT_DEBUG : 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2793,8 +2793,7 @@
 
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2835,8 +2834,7 @@
 
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2881,8 +2879,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -2925,8 +2922,7 @@
 
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2971,8 +2967,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -2993,8 +2988,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3024,8 +3018,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3046,8 +3039,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3064,8 +3056,7 @@
 
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3081,8 +3072,7 @@
 
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3131,8 +3121,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3149,8 +3138,7 @@
 
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
             list_w = []
@@ -3166,8 +3154,7 @@
 
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3183,8 +3170,7 @@
 
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3234,8 +3220,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3278,8 +3263,7 @@
 
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3320,8 +3304,7 @@
 
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
             list_w = []
@@ -3341,8 +3324,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3391,8 +3373,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3415,8 +3396,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3439,8 +3419,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3489,8 +3468,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3509,8 +3487,7 @@
 
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -3530,8 +3507,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3579,8 +3555,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3603,8 +3578,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3623,8 +3597,7 @@
 
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3640,8 +3613,7 @@
 
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3690,8 +3662,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3710,8 +3681,7 @@
 
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3727,8 +3697,7 @@
 
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3776,8 +3745,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3796,8 +3764,7 @@
 
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3813,8 +3780,7 @@
 
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -3862,8 +3828,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -3886,8 +3851,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -3906,8 +3870,7 @@
 
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3954,8 +3917,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -3978,8 +3940,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4002,8 +3963,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4048,8 +4008,7 @@
 
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -4065,8 +4024,7 @@
 
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
             list_w = []
@@ -4082,8 +4040,7 @@
 
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
             list_w = []
@@ -4128,8 +4085,7 @@
 
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -4145,8 +4101,7 @@
 
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
             list_w = []
@@ -4193,8 +4148,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4217,8 +4171,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4262,8 +4215,7 @@
 
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4308,8 +4260,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4329,8 +4280,7 @@
 
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4350,8 +4300,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4399,8 +4348,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4423,8 +4371,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4447,8 +4394,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4493,8 +4439,7 @@
 
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
             list_w = []
@@ -4539,8 +4484,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4638,8 +4582,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4660,8 +4603,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4691,8 +4633,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4711,8 +4652,7 @@
 
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -4758,8 +4698,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4782,8 +4721,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4806,8 +4744,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4856,8 +4793,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4880,8 +4816,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -4929,8 +4864,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -4951,8 +4885,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5000,8 +4933,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5024,8 +4956,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5048,8 +4979,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5094,8 +5024,7 @@
 
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
             list_w = []
@@ -5111,8 +5040,7 @@
 
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
             list_w = []
@@ -5155,8 +5083,7 @@
 
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
             list_w = []
@@ -5201,8 +5128,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5221,8 +5147,7 @@
 
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5268,8 +5193,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5288,8 +5212,7 @@
 
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5335,8 +5258,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5359,8 +5281,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5379,8 +5300,7 @@
 
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5427,8 +5347,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5447,8 +5366,7 @@
 
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
             list_w = []
@@ -5494,8 +5412,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5542,8 +5459,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5562,8 +5478,7 @@
 
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
             list_w = []
@@ -5579,8 +5494,7 @@
 
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
             list_w = []
@@ -5628,8 +5542,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5648,8 +5561,7 @@
 
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
             list_w = []
@@ -5665,8 +5577,7 @@
 
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
             list_w = []
@@ -5686,8 +5597,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5710,8 +5620,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 64:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5764,8 +5673,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5812,8 +5720,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5858,8 +5765,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5904,8 +5810,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -5928,8 +5833,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -5950,8 +5854,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6000,8 +5903,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6024,8 +5926,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6048,8 +5949,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6098,8 +5998,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6120,8 +6019,7 @@
         if w_obj is not None:

From noreply at buildbot.pypy.org  Wed Jun 12 13:10:53 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 12 Jun 2013 13:10:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: rename is_reg method to is_core_reg on
	location objects
Message-ID: <20130612111053.796BE1C00B9@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64863:d417faaac238
Date: 2013-06-12 05:46 -0500
http://bitbucket.org/pypy/pypy/changeset/d417faaac238/

Log:	rename is_reg method to is_core_reg on location objects

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -113,7 +113,7 @@
         tmpreg = r.lr
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None: # store
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.load_reg(mc, excvalloc, r.ip)
         if on_frame:
             # store exc_value in JITFRAME
@@ -125,7 +125,7 @@
             self.store_reg(mc, r.ip, r.fp, ofs, helper=tmpreg)
         if exctploc is not None:
             # store pos_exception in exctploc
-            assert exctploc.is_reg()
+            assert exctploc.is_core_reg()
             mc.gen_load_int(r.ip.value, self.cpu.pos_exception())
             self.load_reg(mc, exctploc, r.ip, helper=tmpreg)
 
@@ -146,7 +146,7 @@
         tmpreg = r.lr # use lr as a second temporary reg
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None:
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.store_reg(mc, excvalloc, r.ip)
         else:
             assert exctploc is not r.fp
@@ -947,7 +947,7 @@
     # regalloc support
     def load(self, loc, value):
         """load an immediate value into a register"""
-        assert (loc.is_reg() and value.is_imm()
+        assert (loc.is_core_reg() and value.is_imm()
                     or loc.is_vfp_reg() and value.is_imm_float())
         if value.is_imm():
             self.mc.gen_load_int(loc.value, value.getint())
@@ -958,7 +958,7 @@
     def load_reg(self, mc, target, base, ofs=0, cond=c.AL, helper=r.ip):
         if target.is_vfp_reg():
             return self._load_vfp_reg(mc, target, base, ofs, cond, helper)
-        elif target.is_reg():
+        elif target.is_core_reg():
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
@@ -1012,7 +1012,7 @@
     def _mov_imm_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.type == FLOAT:
             raise AssertionError("invalid target for move from imm value")
-        if loc.is_reg():
+        if loc.is_core_reg():
             new_loc = loc
         elif loc.is_stack() or loc.is_raw_sp():
             new_loc = r.lr
@@ -1027,7 +1027,7 @@
     def _mov_reg_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.is_imm():
             raise AssertionError("mov reg to imm doesn't make sense")
-        if loc.is_reg():
+        if loc.is_core_reg():
             self.mc.MOV_rr(loc.value, prev_loc.value, cond=cond)
         elif loc.is_stack() and loc.type != FLOAT:
             # spill a core register
@@ -1050,7 +1050,7 @@
         helper = None
         offset = prev_loc.value
         tmp = None
-        if loc.is_reg():
+        if loc.is_core_reg():
             assert prev_loc.type != FLOAT, 'trying to load from an \
                 incompatible location into a core register'
             # unspill a core register
@@ -1126,7 +1126,7 @@
         """Moves a value from a previous location to some other location"""
         if prev_loc.is_imm():
             return self._mov_imm_to_loc(prev_loc, loc, cond)
-        elif prev_loc.is_reg():
+        elif prev_loc.is_core_reg():
             self._mov_reg_to_loc(prev_loc, loc, cond)
         elif prev_loc.is_stack():
             self._mov_stack_to_loc(prev_loc, loc, cond)
@@ -1215,7 +1215,7 @@
                 scratch_reg = r.vfp_ip
             self.regalloc_mov(loc, scratch_reg, cond)
             self.regalloc_push(scratch_reg, cond)
-        elif loc.is_reg():
+        elif loc.is_core_reg():
             self.mc.PUSH([loc.value], cond=cond)
         elif loc.is_vfp_reg():
             self.mc.VPUSH([loc.value], cond=cond)
@@ -1238,7 +1238,7 @@
                 scratch_reg = r.vfp_ip
             self.regalloc_pop(scratch_reg)
             self.regalloc_mov(scratch_reg, loc)
-        elif loc.is_reg():
+        elif loc.is_core_reg():
             self.mc.POP([loc.value], cond=cond)
         elif loc.is_vfp_reg():
             self.mc.VPOP([loc.value], cond=cond)
@@ -1306,7 +1306,7 @@
 
         # lengthloc is the length of the array, which we must not modify!
         assert lengthloc is not r.r0 and lengthloc is not r.r1
-        if lengthloc.is_reg():
+        if lengthloc.is_core_reg():
             varsizeloc = lengthloc
         else:
             assert lengthloc.is_stack()
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -40,7 +40,7 @@
         if self.fnloc.is_stack():
             self.asm.mov_loc_loc(self.fnloc, r.ip)
             self.fnloc = r.ip
-        assert self.fnloc.is_reg()
+        assert self.fnloc.is_core_reg()
         self.mc.BLX(self.fnloc.value)
 
     def restore_stack_pointer(self):
@@ -135,7 +135,7 @@
             return [], []
         if self.resloc.is_vfp_reg():
             return [r.r0, r.r1], []
-        assert self.resloc.is_reg()
+        assert self.resloc.is_core_reg()
         return [r.r0], []
 
     def load_result(self):
@@ -146,7 +146,7 @@
         if resloc.is_vfp_reg():
             # move result to the allocated register
             self.asm.mov_to_vfp_loc(r.r0, r.r1, resloc)
-        elif resloc.is_reg():
+        elif resloc.is_core_reg():
             # move result to the allocated register
             if resloc is not r.r0:
                 self.asm.mov_loc_loc(r.r0, resloc)
@@ -283,7 +283,7 @@
     def load_result(self):
         resloc = self.resloc
         # ensure the result is wellformed and stored in the correct location
-        if resloc is not None and resloc.is_reg():
+        if resloc is not None and resloc.is_core_reg():
             self._ensure_result_bit_extension(resloc,
                                                   self.ressize, self.ressign)
 
@@ -292,7 +292,7 @@
             return [], []
         if self.resloc.is_vfp_reg():
             return [], [r.d0]
-        assert self.resloc.is_reg()
+        assert self.resloc.is_core_reg()
         return [r.r0], []
 
 
diff --git a/rpython/jit/backend/arm/helper/assembler.py b/rpython/jit/backend/arm/helper/assembler.py
--- a/rpython/jit/backend/arm/helper/assembler.py
+++ b/rpython/jit/backend/arm/helper/assembler.py
@@ -82,7 +82,7 @@
         assert guard is not None
         l0 = arglocs[0]
         l1 = arglocs[1]
-        assert l0.is_reg()
+        assert l0.is_core_reg()
 
         if l1.is_imm():
             self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond)
diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py
--- a/rpython/jit/backend/arm/locations.py
+++ b/rpython/jit/backend/arm/locations.py
@@ -15,7 +15,7 @@
     def is_raw_sp(self):
         return False
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def is_vfp_reg(self):
@@ -43,7 +43,7 @@
     def __repr__(self):
         return 'r%d' % self.value
 
-    def is_reg(self):
+    def is_core_reg(self):
         return True
 
     def as_key(self):
@@ -62,7 +62,7 @@
     def __repr__(self):
         return 'vfp%d' % self.value
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def is_vfp_reg(self):
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -248,7 +248,7 @@
         l1 = arglocs[1]
         failargs = arglocs[2:]
 
-        if l0.is_reg():
+        if l0.is_core_reg():
             if l1.is_imm():
                 self.mc.CMP_ri(l0.value, l1.getint())
             else:
@@ -488,7 +488,7 @@
             # case GCFLAG_CARDS_SET: emit a few instructions to do
             # directly the card flag setting
             loc_index = arglocs[1]
-            assert loc_index.is_reg()
+            assert loc_index.is_core_reg()
             # must save the register loc_index before it is mutated
             mc.PUSH([loc_index.value])
             tmp1 = loc_index
@@ -588,7 +588,7 @@
 
     def emit_op_setarrayitem_gc(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         if scale.value > 0:
             self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale.value)
             ofs_loc = r.ip
@@ -606,7 +606,7 @@
             # vstr only supports imm offsets
             # so if the ofset is too large we add it to the base and use an
             # offset of 0
-            if ofs_loc.is_reg():
+            if ofs_loc.is_core_reg():
                 tmploc, save = self.get_tmp_reg([value_loc, base_loc, ofs_loc])
                 assert not save
                 self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
@@ -644,13 +644,13 @@
 
     def emit_op_raw_store(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         self._write_to_mem(value_loc, base_loc, ofs_loc, scale, fcond)
         return fcond
 
     def emit_op_getarrayitem_gc(self, op, arglocs, regalloc, fcond):
         res_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         signed = op.getdescr().is_item_signed()
 
         # scale the offset as required
@@ -672,7 +672,7 @@
             # vldr only supports imm offsets
             # if the offset is in a register we add it to the base and use a
             # tmp reg
-            if ofs_loc.is_reg():
+            if ofs_loc.is_core_reg():
                 tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
                 assert not save
                 self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
@@ -727,7 +727,7 @@
 
     def emit_op_raw_load(self, op, arglocs, regalloc, fcond):
         res_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         # no base offset
         assert ofs.value == 0
         signed = op.getdescr().is_item_signed()
@@ -805,10 +805,10 @@
             bytes_box = TempBox()
             bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, forbidden_vars)
             scale = self._get_unicode_item_scale()
-            if not length_loc.is_reg():
+            if not length_loc.is_core_reg():
                 self.regalloc_mov(length_loc, bytes_loc)
                 length_loc = bytes_loc
-            assert length_loc.is_reg()
+            assert length_loc.is_core_reg()
             self.mc.MOV_ri(r.ip.value, 1 << scale)
             self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value)
             length_box = bytes_box
@@ -835,8 +835,8 @@
 
    # result = base_loc  + (scaled_loc << scale) + static_offset
     def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0):
-        assert scaled_loc.is_reg()
-        assert base_loc.is_reg()
+        assert scaled_loc.is_core_reg()
+        assert base_loc.is_core_reg()
         assert check_imm_arg(scale)
         assert check_imm_arg(static_offset)
         if scale > 0:
@@ -1063,7 +1063,7 @@
     def emit_op_cast_float_to_int(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert arg.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VCVT_float_to_int(r.vfp_ip.value, arg.value)
         self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
@@ -1071,7 +1071,7 @@
     def emit_op_cast_int_to_float(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert res.is_vfp_reg()
-        assert arg.is_reg()
+        assert arg.is_core_reg()
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
         self.mc.VCVT_int_to_float(res.value, res.value)
@@ -1087,7 +1087,7 @@
         loc = arglocs[0]
         res = arglocs[1]
         assert loc.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VMOV_rc(res.value, r.ip.value, loc.value)
         return fcond
 
@@ -1108,7 +1108,7 @@
     def emit_op_cast_float_to_singlefloat(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert arg.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VCVT_f64_f32(r.vfp_ip.value, arg.value)
         self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
@@ -1116,7 +1116,7 @@
     def emit_op_cast_singlefloat_to_float(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert res.is_vfp_reg()
-        assert arg.is_reg()
+        assert arg.is_core_reg()
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
         self.mc.VCVT_f32_f64(res.value, res.value)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -324,7 +324,7 @@
                 loc = r.fp
             arg = inputargs[i]
             i += 1
-            if loc.is_reg():
+            if loc.is_core_reg():
                 self.rm.reg_bindings[arg] = loc
                 used[loc] = None
             elif loc.is_vfp_reg():
@@ -358,7 +358,7 @@
                 continue
             if box.type == REF and self.rm.is_still_alive(box):
                 assert not noregs
-                assert loc.is_reg()
+                assert loc.is_core_reg()
                 val = loc.value
                 gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8))
         for box, loc in self.fm.bindings.iteritems():
@@ -1154,7 +1154,7 @@
             assert isinstance(arg, Box)
             loc = self.loc(arg)
             arglocs[i] = loc
-            if loc.is_reg():
+            if loc.is_core_reg():
                 self.frame_manager.mark_as_free(arg)
         #
         descr._arm_arglocs = arglocs
diff --git a/rpython/jit/backend/arm/test/test_jump.py b/rpython/jit/backend/arm/test/test_jump.py
--- a/rpython/jit/backend/arm/test/test_jump.py
+++ b/rpython/jit/backend/arm/test/test_jump.py
@@ -255,7 +255,7 @@
                 else:
                     newvalue = 'value-vfp-%d' % i
                 regs2[loc.value] = newvalue
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 regs1[loc.value] = 'value-int-%d' % i
             elif loc.is_stack():
                 stack[loc.position] = 'value-width%d-%d' % (loc.width, i)
@@ -284,7 +284,7 @@
                 assert loc.width == expected_width*WORD
             if loc.is_vfp_reg():
                 return regs2[loc.value]
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 return regs1[loc.value]
             elif loc.is_stack():
                 got = stack[loc.position]
@@ -298,7 +298,7 @@
         def write(loc, newvalue):
             if loc.is_vfp_reg():
                 regs2[loc.value] = newvalue
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 regs1[loc.value] = newvalue
             elif loc.is_stack():
                 if loc.width > WORD:
@@ -317,17 +317,17 @@
         for op in assembler.ops:
             if op[0] == 'mov':
                 src, dst = op[1:]
-                assert src.is_reg() or src.is_vfp_reg() or src.is_stack() or src.is_imm_float() or src.is_imm()
-                assert dst.is_reg() or dst.is_vfp_reg() or dst.is_stack()
+                assert src.is_core_reg() or src.is_vfp_reg() or src.is_stack() or src.is_imm_float() or src.is_imm()
+                assert dst.is_core_reg() or dst.is_vfp_reg() or dst.is_stack()
                 assert not (src.is_stack() and dst.is_stack())
                 write(dst, read(src))
             elif op[0] == 'push':
                 src, = op[1:]
-                assert src.is_reg() or src.is_vfp_reg() or src.is_stack()
+                assert src.is_core_reg() or src.is_vfp_reg() or src.is_stack()
                 extrapushes.append(read(src))
             elif op[0] == 'pop':
                 dst, = op[1:]
-                assert dst.is_reg() or dst.is_vfp_reg() or dst.is_stack()
+                assert dst.is_core_reg() or dst.is_vfp_reg() or dst.is_stack()
                 write(dst, extrapushes.pop())
             else:
                 assert 0, "unknown op: %r" % (op,)
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -47,7 +47,7 @@
             input_i += 1
             if arg.type == REF:
                 loc = fail_locs[i]
-                if loc.is_reg():
+                if loc.is_core_reg():
                     val = self.cpu.all_reg_indexes[loc.value]
                 else:
                     val = loc.get_position() + self.cpu.JITFRAME_FIXED_SIZE
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1551,14 +1551,14 @@
         frame in jf_guard_exc
         """
         if excvalloc is not None:
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             mc.MOV(excvalloc, heap(self.cpu.pos_exc_value()))
         elif tmploc is not None: # if both are None, just ignore
             ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc')
             mc.MOV(tmploc, heap(self.cpu.pos_exc_value()))
             mc.MOV(RawEbpLoc(ofs), tmploc)
         if exctploc is not None:
-            assert exctploc.is_reg()
+            assert exctploc.is_core_reg()
             mc.MOV(exctploc, heap(self.cpu.pos_exception()))
 
         mc.MOV(heap(self.cpu.pos_exception()), imm0)
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -45,7 +45,7 @@
     def is_stack(self):
         return False
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def get_position(self):
@@ -169,7 +169,7 @@
     def is_float(self):
         return self.is_xmm
 
-    def is_reg(self):
+    def is_core_reg(self):
         return True
 
 class ImmediateAssemblerLocation(AssemblerLocation):

From noreply at buildbot.pypy.org  Wed Jun 12 13:10:54 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 12 Jun 2013 13:10:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a testcase for the failing case of
 having a jump to a label having a float that was spilled currently loaded
 in a register
Message-ID: <20130612111054.B87981C00B9@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64864:59fdc0758a45
Date: 2013-06-12 06:00 -0500
http://bitbucket.org/pypy/pypy/changeset/59fdc0758a45/

Log:	add a testcase for the failing case of having a jump to a label
	having a float that was spilled currently loaded in a register

diff --git a/rpython/jit/backend/arm/test/test_runner.py b/rpython/jit/backend/arm/test/test_runner.py
--- a/rpython/jit/backend/arm/test/test_runner.py
+++ b/rpython/jit/backend/arm/test/test_runner.py
@@ -11,6 +11,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import JitCellToken, TargetToken
 from rpython.jit.backend.arm.detect import detect_arch_version
+from rpython.jit.codewriter import longlong
 
 
 CPU = getcpuclass()
@@ -261,3 +262,43 @@
         l1 = ('debug_print', preambletoken.repr_of_descr() + ':1')
         l2 = ('debug_print', targettoken.repr_of_descr() + ':9')
         assert ('jit-backend-counts', [l0, l1, l2]) in dlog
+
+
+    def test_label_float_in_reg_and_on_stack(self):
+        targettoken = TargetToken()
+        ops = """
+        [i0, f3]
+        i2 = same_as(i0)    # but forced to be in a register
+        force_spill(i2)
+        force_spill(f3)
+        f4 = float_add(f3, 5.0)
+        label(f3, f4, descr=targettoken)
+        force_spill(f3)
+        f5 = same_as(f3)    # but forced to be in a register
+        finish(f5)
+        """
+        faildescr = BasicFailDescr(2)
+        loop = parse(ops, self.cpu, namespace=locals())
+        looptoken = JitCellToken()
+        info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        ops2 = """
+        [i0, f1]
+        i1 = same_as(i0)
+        f2 = same_as(f1)
+        f3 = float_add(f1, 10.0)
+        force_spill(f3)
+        force_spill(i1)
+        f4 = float_add(f3, f1)
+        jump(f3, f4, descr=targettoken)
+        """
+        loop2 = parse(ops2, self.cpu, namespace=locals())
+        looptoken2 = JitCellToken()
+        info = self.cpu.compile_loop(loop2.inputargs, loop2.operations, looptoken2)
+
+        deadframe = self.cpu.execute_token(looptoken, -9, longlong.getfloatstorage(-13.5))
+        res = longlong.getrealfloat(self.cpu.get_float_value(deadframe, 0))
+        assert res == -13.5
+        #
+        deadframe = self.cpu.execute_token(looptoken2, -9, longlong.getfloatstorage(-13.5))
+        res = longlong.getrealfloat(self.cpu.get_float_value(deadframe, 0))
+        assert res == -3.5

From noreply at buildbot.pypy.org  Wed Jun 12 13:10:55 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 12 Jun 2013 13:10:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigo,
 bivab): simple fix for a hard to track issue. When compiling a
 label we
Message-ID: <20130612111055.E8B3A1C00B9@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64865:4283f7772bc0
Date: 2013-06-12 06:05 -0500
http://bitbucket.org/pypy/pypy/changeset/4283f7772bc0/

Log:	(arigo, bivab): simple fix for a hard to track issue. When compiling
	a label we were keeping the stacklocation of prevously spilled
	floats that were loaded in a reg when we hit the label. When jumping
	from somewehere else to the label the float value in the register is
	forgotten, because it is assumed to be on the stack too.

diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1154,7 +1154,7 @@
             assert isinstance(arg, Box)
             loc = self.loc(arg)
             arglocs[i] = loc
-            if loc.is_core_reg():
+            if loc.is_core_reg() or loc.is_vfp_reg():
                 self.frame_manager.mark_as_free(arg)
         #
         descr._arm_arglocs = arglocs

From noreply at buildbot.pypy.org  Wed Jun 12 13:15:56 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 12 Jun 2013 13:15:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: dead code
Message-ID: <20130612111556.64FD11C00B9@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64866:ab0dd631c220
Date: 2013-06-12 13:12 +0200
http://bitbucket.org/pypy/pypy/changeset/ab0dd631c220/

Log:	dead code

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -227,18 +227,6 @@
         else:
             return self.xrm.make_sure_var_in_reg(var, forbidden_vars)
 
-    def _frame_bindings(self, locs, inputargs):
-        bindings = {}
-        i = 0
-        for loc in locs:
-            if loc is None:
-                continue
-            arg = inputargs[i]
-            i += 1
-            if not isinstance(loc, RegLoc):
-                bindings[arg] = loc
-        return bindings
-
     def _update_bindings(self, locs, inputargs):
         # XXX this should probably go to llsupport/regalloc.py
         used = {}

From noreply at buildbot.pypy.org  Wed Jun 12 17:56:45 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 12 Jun 2013 17:56:45 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Merge default
Message-ID: <20130612155645.EFF591C00B9@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64867:0466c79b5c38
Date: 2013-06-12 17:55 +0200
http://bitbucket.org/pypy/pypy/changeset/0466c79b5c38/

Log:	Merge default

diff too long, truncating to 2000 out of 24532 lines

diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -1,30 +1,16 @@
-"""Provide access to Python's configuration information.  The specific
-configuration variables available depend heavily on the platform and
-configuration.  The values may be retrieved using
-get_config_var(name), and the list of variables is available via
-get_config_vars().keys().  Additional convenience functions are also
-available.
-
-Written by:   Fred L. Drake, Jr.
-Email:        <fdrake at acm.org>
-"""
-
-__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
-
-import sys
-
 
 # The content of this file is redirected from
 # sysconfig_cpython or sysconfig_pypy.
+# All underscore names are imported too, because
+# people like to use undocumented sysconfig._xxx
+# directly.
 
+import sys
 if '__pypy__' in sys.builtin_module_names:
-    from distutils.sysconfig_pypy import *
-    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
-    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+    from distutils import sysconfig_pypy as _sysconfig_module
 else:
-    from distutils.sysconfig_cpython import *
-    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
-    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
+    from distutils import sysconfig_cpython as _sysconfig_module
+globals().update(_sysconfig_module.__dict__)
 
 _USE_CLANG = None
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -9,7 +9,7 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
 import os
 import re
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -1,9 +1,17 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -49,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -71,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,21 +134,25 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
 }
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
+}
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
@@ -164,7 +168,7 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+    return _levelToName.get(level, ("Level %s" % level))
 
 def addLevelName(level, levelName):
     """
@@ -174,8 +178,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -183,9 +187,9 @@
     if isinstance(level, int):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
@@ -277,7 +281,7 @@
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - long(ct)) * 1000
+        self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads and thread:
             self.thread = thread.get_ident()
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -156,7 +156,7 @@
         h = klass(*args)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -187,7 +187,7 @@
     opts = cp.options(sectname)
     if "level" in opts:
         level = cp.get(sectname, "level")
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = cp.get(sectname, "handlers")
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,11 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    self._sock._decref_socketios()
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_codecs.py b/lib-python/2.7/test/test_codecs.py
--- a/lib-python/2.7/test/test_codecs.py
+++ b/lib-python/2.7/test/test_codecs.py
@@ -2,7 +2,11 @@
 import unittest
 import codecs
 import locale
-import sys, StringIO, _testcapi
+import sys, StringIO
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 class Queue(object):
     """
@@ -1387,7 +1391,7 @@
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
-            if encoding not in broken_incremental_coders:
+            if encoding not in broken_incremental_coders and _testcapi:
                 # check incremental decoder/encoder (fetched via the Python
                 # and C API) and iterencode()/iterdecode()
                 try:
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -65,7 +65,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
         finally:
             logging._releaseLock()
 
@@ -97,8 +98,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -7,7 +7,8 @@
 import subprocess
 from copy import copy, deepcopy
 
-from test.test_support import run_unittest, TESTFN, unlink, get_attribute
+from test.test_support import (run_unittest, TESTFN, unlink, get_attribute,
+                               import_module)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -236,7 +237,10 @@
 
     def test_get_config_h_filename(self):
         config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
+        # import_module skips the test when the CPython C Extension API
+        # appears to not be supported
+        self.assertTrue(os.path.isfile(config_h) or
+                        not import_module('_testcapi'), config_h)
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -1,6 +1,9 @@
 """Test cases for traceback module"""
 
-from _testcapi import traceback_print
+try:
+    from _testcapi import traceback_print
+except ImportError:
+    traceback_print = None
 from StringIO import StringIO
 import sys
 import unittest
@@ -176,6 +179,8 @@
 class TracebackFormatTests(unittest.TestCase):
 
     def test_traceback_format(self):
+        if traceback_print is None:
+            raise unittest.SkipTest('Requires _testcapi')
         try:
             raise KeyError('blah')
         except KeyError:
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -1609,7 +1609,10 @@
         self.assertEqual("{}".format(u), '__unicode__ overridden')
 
     def test_encode_decimal(self):
-        from _testcapi import unicode_encodedecimal
+        try:
+            from _testcapi import unicode_encodedecimal
+        except ImportError:
+            raise unittest.SkipTest('Requires _testcapi')
         self.assertEqual(unicode_encodedecimal(u'123'),
                          b'123')
         self.assertEqual(unicode_encodedecimal(u'\u0663.\u0661\u0664'),
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -130,7 +130,7 @@
     RegrTest('test_bz2.py', usemodules='bz2'),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
-    RegrTest('test_capi.py'),
+    RegrTest('test_capi.py', usemodules='cpyext'),
     RegrTest('test_cd.py'),
     RegrTest('test_cfgparser.py'),
     RegrTest('test_cgi.py'),
@@ -177,7 +177,7 @@
     RegrTest('test_cprofile.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
     RegrTest('test_csv.py', usemodules='_csv'),
-    RegrTest('test_ctypes.py', usemodules="_rawffi thread"),
+    RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
     RegrTest('test_curses.py'),
     RegrTest('test_datetime.py', usemodules='binascii struct'),
     RegrTest('test_dbm.py'),
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -166,8 +166,7 @@
         if self is StructOrUnion:
             return
         if '_fields_' not in self.__dict__:
-            self._fields_ = []
-            _set_shape(self, [], self._is_union)
+            self._fields_ = []  # As a side-effet, this also sets the ffishape.
 
     __setattr__ = struct_setattr
 
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_testcapi.py
@@ -0,0 +1,61 @@
+import os, sys, imp
+import tempfile
+
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
+    """
+    thisdir = os.path.dirname(__file__)
+    output_dir = tempfile.mkdtemp()
+
+    from distutils.ccompiler import new_compiler
+
+    compiler = new_compiler()
+    compiler.output_dir = output_dir
+
+    # Compile .c file
+    include_dir = os.path.join(thisdir, '..', 'include')
+    if sys.platform == 'win32':
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
+    else:
+        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
+    res = compiler.compile([os.path.join(thisdir, csource)],
+                           include_dirs=[include_dir],
+                           extra_preargs=ccflags)
+    object_filename = res[0]
+
+    # set link options
+    output_filename = modulename + _get_c_extension_suffix()
+    if sys.platform == 'win32':
+        # XXX libpypy-c.lib is currently not installed automatically
+        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
+        libraries = [library, 'oleaut32']
+        extra_ldargs = ['/MANIFEST',  # needed for VC10
+                        '/EXPORT:init' + modulename]
+    else:
+        libraries = []
+        extra_ldargs = []
+
+    # link the dynamic library
+    compiler.link_shared_object(
+        [object_filename],
+        output_filename,
+        libraries=libraries,
+        extra_preargs=extra_ldargs)
+
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,57 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testcapi'")
+else:
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.6"
-__version_info__ = (0, 6)
+__version__ = "0.7"
+__version_info__ = (0, 7)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -73,15 +73,15 @@
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        BVoidP = self._get_cached_btype(model.voidp_type)
+        self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(BVoidP, 0)
+                FFI.NULL = self.cast(self.BVoidP, 0)
                 FFI.CData, FFI.CType = backend._get_types()
         else:
             # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(BVoidP, 0)
+            self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False):
@@ -346,6 +346,12 @@
         self._cdefsources.extend(ffi_to_include._cdefsources)
         self._cdefsources.append(']')
 
+    def new_handle(self, x):
+        return self._backend.newp_handle(self.BVoidP, x)
+
+    def from_handle(self, x):
+        return self._backend.from_handle(x)
+
 
 def _make_ffi_library(ffi, libname, flags):
     import os
@@ -355,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
@@ -372,8 +378,8 @@
             BType = ffi._get_cached_btype(tp)
             try:
                 value = backendlib.load_function(BType, name)
-            except KeyError:
-                raise AttributeError(name)
+            except KeyError as e:
+                raise AttributeError('%s: %s' % (name, e))
             library.__dict__[name] = value
             return
         #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -16,6 +16,7 @@
 class CTypesData(object):
     __metaclass__ = CTypesType
     __slots__ = ['__weakref__']
+    __name__ = '<cdata>'
 
     def __init__(self, *args):
         raise TypeError("cannot instantiate %r" % (self.__class__,))
@@ -491,6 +492,8 @@
         elif BItem in (getbtype(model.PrimitiveType('signed char')),
                        getbtype(model.PrimitiveType('unsigned char'))):
             kind = 'bytep'
+        elif BItem is getbtype(model.void_type):
+            kind = 'voidp'
         else:
             kind = 'generic'
         #
@@ -546,13 +549,13 @@
             def __setitem__(self, index, value):
                 self._as_ctype_ptr[index] = BItem._to_ctypes(value)
 
-            if kind == 'charp':
+            if kind == 'charp' or kind == 'voidp':
                 @classmethod
-                def _arg_to_ctypes(cls, value):
-                    if isinstance(value, bytes):
-                        return ctypes.c_char_p(value)
+                def _arg_to_ctypes(cls, *value):
+                    if value and isinstance(value[0], bytes):
+                        return ctypes.c_char_p(value[0])
                     else:
-                        return super(CTypesPtr, cls)._arg_to_ctypes(value)
+                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
 
             if kind == 'charp' or kind == 'bytep':
                 def _to_string(self, maxlen):
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -15,6 +15,20 @@
     def patch_extension_kwds(self, kwds):
         pass
 
+    def find_module(self, module_name, path, so_suffix):
+        try:
+            f, filename, descr = imp.find_module(module_name, path)
+        except ImportError:
+            return None
+        if f is not None:
+            f.close()
+        # Note that after a setuptools installation, there are both .py
+        # and .so files with the same basename.  The code here relies on
+        # imp.find_module() locating the .so in priority.
+        if descr[0] != so_suffix:
+            return None
+        return filename
+
     def collect_types(self):
         self._typesdict = {}
         self._generate("collecttype")
@@ -142,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -427,9 +444,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -687,7 +704,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
     # ----------
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -1,4 +1,4 @@
-import sys
+import sys, os
 import types
 
 from . import model, ffiplatform
@@ -20,6 +20,16 @@
         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
+    def find_module(self, module_name, path, so_suffix):
+        basename = module_name + so_suffix
+        if path is None:
+            path = sys.path
+        for dirname in path:
+            filename = os.path.join(dirname, basename)
+            if os.path.isfile(filename):
+                return filename
+        return None
+
     def collect_types(self):
         pass      # not needed in the generic engine
 
@@ -64,6 +74,9 @@
         class FFILibrary(types.ModuleType):
             _cffi_generic_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir
         library = FFILibrary("")
         #
         # finally, call the loaded_gen_xxx() functions.  This will set
@@ -158,21 +171,22 @@
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
-            if any(isinstance(type, model.StructOrUnion) for type in tp.args):
+            if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
-                for i, type in enumerate(tp.args):
-                    if isinstance(type, model.StructOrUnion):
-                        type = model.PointerType(type)
-                        indirections.append((i, type))
-                    indirect_args.append(type)
+                for i, typ in enumerate(tp.args):
+                    if isinstance(typ, model.StructOrUnion):
+                        typ = model.PointerType(typ)
+                        indirections.append((i, typ))
+                    indirect_args.append(typ)
                 tp = model.FunctionPtrType(tuple(indirect_args),
                                            tp.result, tp.ellipsis)
             BFunc = self.ffi._get_cached_btype(tp)
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
-            for i, type in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, type)
+            for i, typ in indirections:
+                newfunction = self._make_struct_wrapper(newfunction, i, typ)
         setattr(library, name, newfunction)
+        type(library)._cffi_dir.append(name)
 
     def _make_struct_wrapper(self, oldfunc, i, tp):
         backend = self.ffi._backend
@@ -216,9 +230,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -380,6 +394,7 @@
         is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
         value = self._load_constant(is_int, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # enums
@@ -427,6 +442,7 @@
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
             setattr(library, enumerator, enumvalue)
+            type(library)._cffi_dir.append(enumerator)
 
     # ----------
     # macros: for now only for integers
@@ -440,6 +456,7 @@
     def _loaded_gen_macro(self, tp, name, module, library):
         value = self._load_constant(True, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # global variables
@@ -465,6 +482,7 @@
                 BArray = self.ffi._get_cached_btype(tp)
                 value = self.ffi.cast(BArray, value)
             setattr(library, name, value)
+            type(library)._cffi_dir.append(name)
             return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
@@ -476,7 +494,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
 cffimod_header = r'''
 #include <stdio.h>
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -102,21 +102,10 @@
                 path = pkg.__path__
             else:
                 path = None
-            try:
-                f, filename, descr = imp.find_module(self.get_module_name(),
-                                                     path)
-            except ImportError:
+            filename = self._vengine.find_module(self.get_module_name(), path,
+                                                 _get_so_suffix())
+            if filename is None:
                 return
-            if f is not None:
-                f.close()
-            if filename.lower().endswith('.py'):
-                # on PyPy, if there are both .py and .pypy-19.so files in
-                # the same directory, the .py file is returned.  That's the
-                # case after a setuptools installation.  We never want to
-                # load the .py file here...
-                filename = filename[:-3] + _get_so_suffix()
-                if not os.path.isfile(filename):
-                    return
             self.modulefilename = filename
         self._vengine.collect_types()
         self._has_module = True
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
new file mode 100644
--- /dev/null
+++ b/lib_pypy/greenlet.egg-info
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: greenlet
+Version: 0.4.0
+Summary: Lightweight in-process concurrent programming
+Home-page: https://github.com/python-greenlet/greenlet
+Author: Ralf Schmitt (for CPython), PyPy team
+Author-email: pypy-dev at python.org
+License: MIT License
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -64,7 +64,8 @@
     del working_modules["termios"]
     del working_modules["_minimal_curses"]
 
-    del working_modules["cppyy"]  # not tested on win32
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # not tested on win32
 
     # The _locale module is needed by site.py on Windows
     default_modules["_locale"] = None
@@ -77,7 +78,8 @@
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
     del working_modules["_multiprocessing"]   # depends on rctime
-    del working_modules["cppyy"]  # depends on ctypes
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # depends on ctypes
 
 
 module_dependencies = {
@@ -120,12 +122,10 @@
                     __import__(name)
             except (ImportError, CompilationError, py.test.skip.Exception), e:
                 errcls = e.__class__.__name__
-                config.add_warning(
+                raise Exception(
                     "The module %r is disabled\n" % (modname,) +
                     "because importing %s raised %s\n" % (name, errcls) +
                     str(e))
-                raise ConflictConfigError("--withmod-%s: %s" % (modname,
-                                                                errcls))
         return validator
     else:
         return None
@@ -216,10 +216,6 @@
                    "(the empty string and potentially single-char strings)",
                    default=False),
 
-        BoolOption("withsmalltuple",
-                   "use small tuples",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
@@ -364,6 +360,7 @@
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
+
     config.objspace.usemodules.suggest(**dict.fromkeys(modules, True))
 
 def enable_translationmodules(config):
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,7 @@
+
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -339,8 +339,9 @@
 
 + methods and other class attributes do not change after startup
 + single inheritance is fully supported
-+ simple mixins work too, but the mixed in class needs a ``_mixin_ = True``
-  class attribute
++ simple mixins somewhat work too, but the mixed in class needs a
+  ``_mixin_ = True`` class attribute. isinstance checks against the
+  mixin type will fail when translated.
 
 + classes are first-class objects too
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.0'
+release = '2.0.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -163,6 +163,9 @@
     $ genreflex MyClass.h
     $ g++ -fPIC -rdynamic -O2 -shared -I$REFLEXHOME/include MyClass_rflx.cpp -o libMyClassDict.so -L$REFLEXHOME/lib -lReflex
 
+Next, make sure that the library can be found through the dynamic lookup path
+(the ``LD_LIBRARY_PATH`` environment variable on Linux, ``PATH`` on Windows),
+for example by adding ".".
 Now you're ready to use the bindings.
 Since the bindings are designed to look pythonistic, it should be
 straightforward::
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:
 
 * Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
 
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -28,7 +28,8 @@
 Layers
 ------
 
-PyPy has layers. Those layers help us keep the respective parts separated enough
+PyPy has layers. Just like Ogres or onions.
+Those layers help us keep the respective parts separated enough
 to be worked on independently and make the complexity manageable. This is,
 again, just a sanity requirement for such a complex project. For example writing
 a new optimization for the JIT usually does **not** involve touching a Python
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -22,7 +22,8 @@
   will capture the revision number of this change for the release;
   some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
-  necessary
+  necessary; also update the version number in pypy/doc/conf.py,
+  and in pypy/doc/index.rst
 * update pypy/doc/contributor.rst (and possibly LICENSE)
 * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
   and create a fresh whatsnew_head.rst after the release
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -40,7 +40,7 @@
 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.0`_: the latest official release
+* `Release 2.0.2`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
@@ -110,7 +110,7 @@
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.0`: http://pypy.org/download.html
+.. _`Release 2.0.2`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -16,7 +16,10 @@
     Inspect interactively after running script.
 
 -O
-    Dummy optimization flag for compatibility with C Python.
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to -O.
 
 -c *cmd*
     Program passed in as CMD (terminates option list).
diff --git a/pypy/doc/release-2.0.1.rst b/pypy/doc/release-2.0.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.1.rst
@@ -0,0 +1,46 @@
+==============================
+PyPy 2.0.1 - Bohr Sm&#248;rrebr&#248;d
+==============================
+
+We're pleased to announce PyPy 2.0.1.  This is a stable bugfix release
+over `2.0`_.  You can download it here:
+  
+    http://pypy.org/download.html
+
+The fixes are mainly about fatal errors or crashes in our stdlib.  See
+below for more details.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Support for ARM is progressing but not bug-free yet.
+
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
+
+Highlights
+==========
+
+- fix an occasional crash in the JIT that ends in `RPython Fatal error:
+  NotImplementedError`__.
+
+- `id(x)` is now always a positive number (except on int/float/long/complex).
+  This fixes an issue in ``_sqlite.py`` (mostly for 32-bit Linux).
+
+- fix crashes of callback-from-C-functions (with cffi) when used together
+  with Stackless features, on asmgcc (i.e. Linux only).  Now `gevent should
+  work better`__.
+
+- work around an eventlet issue with `socket._decref_socketios()`__.
+
+.. __: https://bugs.pypy.org/issue1482
+.. __: http://mail.python.org/pipermail/pypy-dev/2013-May/011362.html
+.. __: https://bugs.pypy.org/issue1468
+.. _2.0: release-2.0.0.html
+
+Cheers,
+arigo et. al. for the PyPy team
diff --git a/pypy/doc/release-2.0.2.rst b/pypy/doc/release-2.0.2.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.0.2.rst
@@ -0,0 +1,46 @@
+=========================
+PyPy 2.0.2 - Fermi Panini
+=========================
+
+We're pleased to announce PyPy 2.0.2.  This is a stable bugfix release
+over `2.0`_ and `2.0.1`_.  You can download it here:
+
+    http://pypy.org/download.html
+
+It fixes a crash in the JIT when calling external C functions (with
+ctypes/cffi) in a multithreaded context.
+
+.. _2.0: release-2.0.0.html
+.. _2.0.1: release-2.0.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.0 and cpython 2.7.3`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Support for ARM is progressing but not bug-free yet.
+
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
+
+Highlights
+==========
+
+This release contains only the fix described above.  A crash (or wrong
+results) used to occur if all these conditions were true:
+
+- your program is multithreaded;
+
+- it runs on a single-core machine or a heavily-loaded multi-core one;
+
+- it uses ctypes or cffi to issue external calls to C functions.
+
+This was fixed in the branch `emit-call-x86`__ (see the example file
+``bug1.py``).
+
+.. __: https://bitbucket.org/pypy/pypy/commits/7c80121abbf4
+
+Cheers,
+arigo et. al. for the PyPy team
diff --git a/pypy/doc/rffi.rst b/pypy/doc/rffi.rst
--- a/pypy/doc/rffi.rst
+++ b/pypy/doc/rffi.rst
@@ -5,7 +5,7 @@
 Purpose
 -------
 
-This document describes an FFI for RPython language, concentrating
+This document describes an FFI for the RPython language, concentrating
 on low-level backends like C. It describes
 how to declare and call low-level (C) functions from RPython level.
 
@@ -50,7 +50,7 @@
 ------
 
 In rffi_ there are various declared types for C-structures, like CCHARP
-(char*), SIZE_T (size_t) and others. refer to file for details. 
+(char*), SIZE_T (size_t) and others. Refer to file for details. 
 Instances of non-primitive types must be alloced by hand, with call 
 to lltype.malloc, and freed by lltype.free both with keyword argument 
 flavor='raw'. There are several helpers like string -> char*
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -19,23 +19,28 @@
     branches.discard('default')
     return startrev, branches
 
-def get_merged_branches(path, startrev, endrev):
-    if getstatusoutput('hg root')[0]:
+def get_merged_branches(path, startrev, endrev, current_branch=None):
+    errcode, wc_branch = getstatusoutput('hg branch')
+    if errcode != 0:
         py.test.skip('no Mercurial repo')
+    if current_branch is None:
+        current_branch = wc_branch
 
     # X = take all the merges which are descendants of startrev and are on default
     # revset = all the parents of X which are not on default
     # ===>
     # revset contains all the branches which have been merged to default since
     # startrev
-    revset = 'parents(%s::%s and \
+    revset = "parents(%s::%s and \
                       merge() and \
-                      branch(default)) and \
-              not branch(default)' % (startrev, endrev)
+                      branch('%s')) and \
+              not branch('%s')" % (startrev, endrev,
+                                   current_branch, current_branch)
     cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset)
     out = getoutput(cmd)
     branches = set(map(str.strip, out.splitlines()))
-    return branches
+    branches.discard("default")
+    return branches, current_branch
 
 
 def test_parse_doc():
@@ -65,7 +70,8 @@
     assert branches == set(['foobar', 'hello'])
 
 def test_get_merged_branches():
-    branches = get_merged_branches(ROOT, 'f34f0c11299f', '79770e0c2f93')
+    branches, _ = get_merged_branches(ROOT, 'f34f0c11299f', '79770e0c2f93',
+                                      'default')
     assert branches == set(['numpy-indexing-by-arrays-bool',
                             'better-jit-hooks-2',
                             'numpypy-ufuncs'])
@@ -76,7 +82,9 @@
     whatsnew_list.sort()
     last_whatsnew = whatsnew_list[-1].read()
     startrev, documented = parse_doc(last_whatsnew)
-    merged = get_merged_branches(ROOT, startrev, '')
+    merged, branch = get_merged_branches(ROOT, startrev, '')
+    merged.discard('default')
+    merged.discard('')
     not_documented = merged.difference(documented)
     not_merged = documented.difference(merged)
     print 'Branches merged but not documented:'
@@ -85,4 +93,6 @@
     print 'Branches documented but not merged:'
     print '\n'.join(not_merged)
     print
-    assert not not_documented and not not_merged
+    assert not not_documented
+    if branch == 'default':
+        assert not not_merged
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,48 @@
 
 .. branch: numpy-pickle
 Pickling of numpy arrays and dtypes (including record dtypes)
+
+.. branch: remove-array-smm
+Remove multimethods in the arraymodule
+
+.. branch: callback-stacklet
+Fixed bug when switching stacklets from a C callback
+
+.. branch: remove-set-smm
+Remove multi-methods on sets
+
+.. branch: numpy-subarrays
+Implement subarrays for numpy
+
+.. branch: remove-dict-smm
+Remove multi-methods on dict
+
+.. branch: remove-list-smm-2
+Remove remaining multi-methods on list
+
+.. branch: arm-stacklet
+Stacklet support for ARM, enables _continuation support
+
+.. branch: remove-tuple-smm
+Remove multi-methods on tuple
+
+.. branch: remove-iter-smm
+Remove multi-methods on iterators
+
+.. branch: emit-call-x86
+.. branch: emit-call-arm
+
+.. branch: on-abort-resops
+Added list of resops to the pypyjit on_abort hook.
+
+.. branch: logging-perf
+Speeds up the stdlib logging module
+
+.. branch: operrfmt-NT
+Adds a couple convenient format specifiers to operationerrfmt
+
+.. branch: win32-fixes3
+Skip and fix some non-translated (own) tests for win32 builds
+
+.. branch: ctypes-byref
+Add the '_obj' attribute on ctypes pointer() and byref() objects
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -2,7 +2,7 @@
 PyPy on Windows
 ===============
 
-Pypy is supported on Windows platforms, starting with Windows 2000.
+PyPy is supported on Windows platforms, starting with Windows 2000.
 The following text gives some hints about how to translate the PyPy
 interpreter.
 
@@ -199,9 +199,9 @@
 
 or such, depending on your mingw64 download.
 
-hacking on Pypy with the mingw compiler
+hacking on PyPy with the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since hacking on Pypy means running tests, you will need a way to specify
+Since hacking on PyPy means running tests, you will need a way to specify
 the mingw compiler when hacking (as opposed to translating). As of
 March 2012, --cc is not a valid option for pytest.py. However if you set an
 environment variable CC to the compliter exe, testing will use it.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -10,6 +10,8 @@
 from rpython.config.config import ConflictConfigError
 from pypy.tool.option import make_objspace
 from pypy.conftest import pypydir
+from rpython.rlib import rthread
+from pypy.module.thread import os_thread
 
 thisdir = py.path.local(__file__).dirpath()
 
@@ -120,6 +122,24 @@
         source = rffi.charp2str(ll_source)
         return _pypy_execute_source(source)
 
+    @entrypoint('main', [], c_name='pypy_init_threads')
+    def pypy_init_threads():
+        if not space.config.objspace.usemodules.thread:
+            return
+        os_thread.setup_threads(space)
+        rffi.aroundstate.before()
+
+    @entrypoint('main', [], c_name='pypy_thread_attach')
+    def pypy_thread_attach():
+        if not space.config.objspace.usemodules.thread:
+            return
+        os_thread.setup_threads(space)
+        os_thread.bootstrapper.acquire(space, None, None)
+        rthread.gc_thread_start()
+        os_thread.bootstrapper.nbthreads += 1
+        os_thread.bootstrapper.release()
+        rffi.aroundstate.before()
+
     w_globals = space.newdict()
     space.setitem(w_globals, space.wrap('__builtins__'),
                   space.builtin_modules['__builtin__'])
@@ -137,6 +157,8 @@
         return 0
 
     return entry_point, {'pypy_execute_source': pypy_execute_source,
+                         'pypy_init_threads': pypy_init_threads,
+                         'pypy_thread_attach': pypy_thread_attach,
                          'pypy_setup_home': pypy_setup_home}
 
 def call_finish(space):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,8 +2,8 @@
 # App-level version of py.py.
 # See test/test_app_main.
 
-# Missing vs CPython: -d, -OO, -t, -v, -x, -3
-"""\
+# Missing vs CPython: -d, -t, -v, -x, -3
+USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 -c cmd : program passed in as string (terminates option list)
@@ -12,7 +12,8 @@
 -i     : inspect interactively after running script; forces a prompt even
          if stdin does not appear to be a terminal; also PYTHONINSPECT=x
 -m mod : run library module as a script (terminates option list)
--O     : dummy optimization flag for compatibility with CPython
+-O     : skip assert statements
+-OO    : remove docstrings when importing modules in addition to -O
 -R     : ignored (see http://bugs.python.org/issue14621)
 -Q arg : division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
@@ -27,7 +28,6 @@
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
 """
-USAGE1 = __doc__
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
 Other environment variables:
@@ -470,6 +470,10 @@
         sys.py3kwarning = bool(sys.flags.py3k_warning)
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
+        if sys.flags.optimize >= 1:
+            import __pypy__
+            __pypy__.set_debug(False)
+
         if sys.py3kwarning:
             print >> sys.stderr, (
                 "Warning: pypy does not implement py3k warnings")
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -86,12 +86,9 @@
             args_w = space.fixedview(w_stararg)
         except OperationError, e:
             if e.match(space, space.w_TypeError):
-                w_type = space.type(w_stararg)
-                typename = w_type.getname(space)
-                raise OperationError(
+                raise operationerrfmt(
                     space.w_TypeError,
-                    space.wrap("argument after * must be "
-                               "a sequence, not %s" % (typename,)))
+                    "argument after * must be a sequence, not %T", w_stararg)
             raise
         self.arguments_w = self.arguments_w + args_w
 
@@ -116,12 +113,10 @@
                 w_keys = space.call_method(w_starstararg, "keys")
             except OperationError, e:
                 if e.match(space, space.w_AttributeError):
-                    w_type = space.type(w_starstararg)
-                    typename = w_type.getname(space)
-                    raise OperationError(
+                    raise operationerrfmt(
                         space.w_TypeError,
-                        space.wrap("argument after ** must be "
-                                   "a mapping, not %s" % (typename,)))
+                        "argument after ** must be a mapping, not %T",
+                        w_starstararg)
                 raise
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -245,6 +245,8 @@
         if w_len is None:
             w_len = space.len(self.w_consts)
             space.setitem(self.w_consts, w_key, w_len)
+        if space.int_w(w_len) == 0:
+            self.scope.doc_removable = False
         return space.int_w(w_len)
 
     def _make_key(self, obj):
@@ -632,6 +634,7 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+    ops.JUMP_IF_NOT_DEBUG : 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2793,8 +2793,7 @@
 
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2835,8 +2834,7 @@
 
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2881,8 +2879,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -2925,8 +2922,7 @@
 
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -2971,8 +2967,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -2993,8 +2988,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3024,8 +3018,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3046,8 +3039,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3064,8 +3056,7 @@
 
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3081,8 +3072,7 @@
 
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
             list_w = []
@@ -3131,8 +3121,7 @@
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3149,8 +3138,7 @@
 
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 8:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
             list_w = []
@@ -3166,8 +3154,7 @@
 
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 16:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
             list_w = []
@@ -3183,8 +3170,7 @@
 
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 32:
-        typename = space.type(w_self).getname(space)
-        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
+        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:

From noreply at buildbot.pypy.org  Wed Jun 12 19:12:18 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 19:12:18 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix the reference in stolen objects:
 from protected to public stubs.
Message-ID: <20130612171218.822B01C00B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r99:2632ae98315f
Date: 2013-06-12 15:21 +0200
http://bitbucket.org/pypy/stmgc/changeset/2632ae98315f/

Log:	Fix the reference in stolen objects: from protected to public stubs.

diff --git a/c4/lists.c b/c4/lists.c
--- a/c4/lists.c
+++ b/c4/lists.c
@@ -22,19 +22,6 @@
   memset(g2l, 0, sizeof(struct G2L));
 }
 
-struct G2L *g2l_malloc(void)
-{
-  struct G2L *g2l = malloc(sizeof(struct G2L));
-  memset(g2l, 0, sizeof(struct G2L));
-  return g2l;
-}
-
-void g2l_free(struct G2L *g2l)
-{
-  free(g2l->raw_start);
-  free(g2l);
-}
-
 wlog_t *_g2l_find(char *entry, gcptr addr)
 {
   revision_t key = (revision_t)addr;
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -35,8 +35,9 @@
 
 void g2l_clear(struct G2L *g2l);
 void g2l_delete(struct G2L *g2l);
-struct G2L *g2l_malloc(void);
-void g2l_free(struct G2L *g2l);
+static inline void g2l_delete_not_used_any_more(struct G2L *g2l) {
+    free(g2l->raw_start);
+}
 
 static inline int g2l_any_entry(struct G2L *g2l) {
     return g2l->raw_current != g2l->raw_start;
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -13,6 +13,8 @@
 
 gcptr stm_stub_malloc(struct tx_public_descriptor *pd)
 {
+    assert(pd->collection_lock != 0);
+
     gcptr p = pd->stub_free_list;
     if (p == NULL) {
         assert(sizeof(struct stub_block_s) == STUB_BLOCK_SIZE);
@@ -43,6 +45,43 @@
     return p;
 }
 
+
+struct tx_steal_data {
+    struct tx_public_descriptor *foreign_pd;
+    struct G2L all_stubs;   /* { protected: public_stub } */
+};
+static __thread struct tx_steal_data *steal_data;
+
+static void replace_ptr_to_protected_with_stub(gcptr *pobj)
+{
+    gcptr stub, obj = *pobj;
+    if (obj == NULL || (obj->h_tid & GCFLAG_PUBLIC) != 0)
+        return;
+
+    /* we use 'all_stubs', a dictionary, in order to try to avoid
+       duplicate stubs for the same object.  XXX maybe it would be
+       better to use a fast approximative cache that stays around for
+       several stealings. */
+    struct tx_steal_data *sd = steal_data;
+    wlog_t *item;
+    G2L_FIND(sd->all_stubs, obj, item, goto not_found);
+
+    /* found already */
+    stub = item->val;
+    assert(stub->h_revision == (((revision_t)obj) | 2));
+    goto done;
+
+ not_found:
+    stub = stm_stub_malloc(sd->foreign_pd);
+    stub->h_tid = obj->h_tid | GCFLAG_PUBLIC | GCFLAG_STUB;
+    stub->h_revision = ((revision_t)obj) | 2;
+    g2l_insert(&sd->all_stubs, obj, stub);
+
+ done:
+    *pobj = stub;
+    fprintf(stderr, "  stolen: fixing *%p: %p -> %p\n", pobj, obj, stub);
+}
+
 void stm_steal_stub(gcptr P)
 {
     struct tx_public_descriptor *foreign_pd = STUB_THREAD(P);
@@ -81,6 +120,9 @@
         fprintf(stderr, "stolen: %p -> %p - - -> %p\n", P, B, L);
         L = B;
     }
+    else {
+        fprintf(stderr, "stolen: %p -> %p\n", P, L);
+    }
 
     /* Here L is a protected (or backup) copy, and we own the foreign
        thread's collection_lock, so we can read/write the flags.  Change
@@ -102,6 +144,17 @@
        odd number that is also valid on a public up-to-date object.
     */
 
+    /* Fix the content of the object: we need to change all pointers
+       that reference protected copies into pointers that reference
+       stub copies.
+    */
+    struct tx_steal_data sd;
+    sd.foreign_pd = foreign_pd;
+    memset(&sd.all_stubs, 0, sizeof(sd.all_stubs));
+    steal_data = &sd;
+    stmcb_trace(L, &replace_ptr_to_protected_with_stub);
+    g2l_delete_not_used_any_more(&sd.all_stubs);
+
     /* If another thread (the foreign or a 3rd party) does a read
        barrier from P, it must only reach L if all writes to L are
        visible; i.e. it must not see P->h_revision => L that still
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -219,6 +219,7 @@
     def f2(r):
         r.wait(2)
         p2 = lib.stm_read_barrier(p)    # steals
+        assert classify(p2) == "public"
         assert lib.rawgetlong(p2, 0) == 2782172
         assert p2 == lib.stm_read_barrier(p)    # short-circuit h_revision
         assert p.h_revision == int(ffi.cast("revision_t", p2))
@@ -323,4 +324,40 @@
     test_stealing_while_modifying(aborting=True)
 
 def test_stub_for_refs_from_stolen():
-    xxx
+    p = palloc_refs(1)
+    qlist = []
+    def f1(r):
+        assert (p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE) == 0
+        p1 = lib.stm_write_barrier(p)   # private copy
+        assert p1 != p
+        assert classify(p) == "public"
+        assert classify(p1) == "private"
+        assert p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
+        q1 = nalloc(HDR + WORD)
+        qlist.append(q1)
+        lib.setlong(q1, 0, -29187)
+        lib.setptr(p1, 0, q1)
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        assert classify(p) == "public"
+        assert classify(p1) == "protected"
+        assert classify(follow_revision(p)) == "stub"
+        assert p1.h_revision & 1
+        r.set(2)
+        r.wait(3)   # wait until the other thread really starts
+    def f2(r):
+        r.wait(2)
+        r.set(3)
+        p2 = lib.stm_read_barrier(p)    # steals
+        assert classify(p2) == "public"
+        q2 = lib.getptr(p2, 0)
+        assert q2 != ffi.NULL
+        assert q2 != qlist[0]
+        assert classify(q2) == "stub"
+        assert q2.h_revision % 4 == 2
+        q3 = lib.stm_read_barrier(q2)
+        assert q3 != q2
+        assert q3 == qlist[0]
+        assert classify(q3) == "public"   # has been stolen
+        assert lib.getlong(q3, 0) == -29187
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Wed Jun 12 19:12:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 19:12:19 +0200 (CEST)
Subject: [pypy-commit] stmgc default: progress
Message-ID: <20130612171219.B2CA21C0EB9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r100:3f28eb08b77f
Date: 2013-06-12 19:12 +0200
http://bitbucket.org/pypy/stmgc/changeset/3f28eb08b77f/

Log:	progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -120,7 +120,7 @@
               /* we update P_prev->h_revision as a shortcut */
               /* XXX check if this really gives a worse performance than only
                  doing this write occasionally based on a counter in d */
-              P_prev->h_revision = v;
+              //P_prev->h_revision = v;  XXX re-enable!
               P = (gcptr)v;
               v = ACCESS_ONCE(P->h_revision);
               if (!(v & 1))  // "is a pointer", i.e. "has a more recent rev"
@@ -229,16 +229,26 @@
     }
 }
 
-gcptr _stm_nonrecord_barrier(gcptr G)
+gcptr _stm_nonrecord_barrier(gcptr P)
 {
   /* follows the logic in stm_DirectReadBarrier() */
   struct tx_descriptor *d = thread_descriptor;
-  gcptr P = G;
   revision_t v;
 
+  fprintf(stderr, "_stm_nonrecord_barrier: %p ", P);
+
+ restart_all:
+  if (P->h_revision == stm_private_rev_num)
+    {
+      /* private */
+      fprintf(stderr, "private\n");
+      return P;
+    }
+
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
       assert(!(P->h_revision & 1));
+      fprintf(stderr, "private_from_protected\n");
       return P;
     }
 
@@ -246,6 +256,9 @@
     {
       while (1)
         {
+          assert(P->h_tid & GCFLAG_PUBLIC);
+          fprintf(stderr, "public ");
+
           wlog_t *item;
           gcptr L;
           G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
@@ -255,8 +268,7 @@
           assert(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
           assert(!(L->h_tid & GCFLAG_PUBLIC));
           assert(is_private(L));
-          fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p "
-                  "public_to_private\n", G, L);
+          fprintf(stderr, "-public_to_private-> %p private\n", L);
           return L;
 
         no_private_obj:;
@@ -270,25 +282,38 @@
           if (v & 2)
             goto follow_stub;
           P = (gcptr)v;
-          assert(P->h_tid & GCFLAG_PUBLIC);
+          fprintf(stderr, "-> %p ", P);
         }
       if (UNLIKELY(v > d->start_time))
         {
-          fprintf(stderr, "_stm_nonrecord_barrier: %p -> NULL changed\n", G);
+          fprintf(stderr, "too recent!\n");
           return NULL;   // object too recent
         }
-      fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p public\n", G, P);
+      fprintf(stderr, "\n");
     }
   else
     {
-      fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p protected\n", G, P);
+      fprintf(stderr, "protected\n");
     }
   return P;
 
  follow_stub:;
-  fprintf(stderr, "_stm_nonrecord_barrier: %p -> %p stub\n  ", G, P);
-  P = (gcptr)(v - 2);
-  return _stm_nonrecord_barrier(P);
+  if (STUB_THREAD(P) == d->public_descriptor)
+    {
+      P = (gcptr)(v - 2);
+      fprintf(stderr, "stub -> %p ", P);
+    }
+  else
+    {
+      P = (gcptr)(v - 2);
+      fprintf(stderr, "stub -foreign-> %p ", P);
+      if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+        {
+          P = (gcptr)P->h_revision;     /* the backup copy */
+          fprintf(stderr, "-backup-> %p ", P);
+        }
+    }
+  goto restart_all;
 }
 
 #if 0
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -2,6 +2,9 @@
 from support import *
 
 
+SHORTCUT = False   # XXXXXXXXXXXXXXXXX
+
+
 def setup_function(f):
     lib.stm_clear_between_tests()
     lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
@@ -146,6 +149,8 @@
     p2.h_revision = ffi.cast("revision_t", p3)
     assert lib.stm_read_barrier(p1) == p3
     assert list_of_read_objects() == [p3]
+    if not SHORTCUT:
+        py.test.skip("re-enable!")
     assert p1.h_revision == int(ffi.cast("revision_t", p3))   # shortcutted
 
 def test_read_barrier_public_to_private():
@@ -222,7 +227,8 @@
         assert classify(p2) == "public"
         assert lib.rawgetlong(p2, 0) == 2782172
         assert p2 == lib.stm_read_barrier(p)    # short-circuit h_revision
-        assert p.h_revision == int(ffi.cast("revision_t", p2))
+        if SHORTCUT:
+            assert p.h_revision == int(ffi.cast("revision_t", p2))
         assert p2 == lib.stm_read_barrier(p)
         assert p2 == plist[-1]
         assert classify(p2) == "public"

From noreply at buildbot.pypy.org  Wed Jun 12 19:28:28 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 12 Jun 2013 19:28:28 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Slices returned by nditer should
 be readonly by default
Message-ID: <20130612172828.7CF841C00B9@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64868:27ffa87a4dbd
Date: 2013-06-12 19:27 +0200
http://bitbucket.org/pypy/pypy/changeset/27ffa87a4dbd/

Log:	Slices returned by nditer should be readonly by default

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -424,6 +424,13 @@
         return SliceArray(self.start, new_strides, new_backstrides, new_shape,
                           self, orig_array)
 
+    def readonly(self):
+        return NonWritableSlice(self.start, self.strides, self.backstrides, self.shape, self.parent, self.orig_arr, self.dtype)
+
+class NonWritableSlice(SliceArray):
+    def descr_setitem(self, space, orig_array, w_index, w_value):
+        raise OperationError(space.w_ValueError, space.wrap(
+            "Assignment destination is read-only"))
 
 class ArrayBuffer(RWBuffer):
     def __init__(self, impl):
diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -85,8 +85,7 @@
     return W_NDimArray(res)
 
 def get_readonly_slice(space, array, it):
-    #XXX Not readonly
-    return W_NDimArray(it.getslice())
+    return W_NDimArray(it.getslice().readonly())
 
 def get_readwrite_slice(space, array, it):
     return W_NDimArray(it.getslice())
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -59,6 +59,12 @@
             n += 1
         assert n == 12
         assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
+        e = None
+        try:
+            r[0][0] = 0
+        except ValueError, ex:
+            e = ex
+        assert e
 
     def test_interface(self):
         from numpypy import arange, nditer, zeros
@@ -207,3 +213,4 @@
         assert (it.operands[1] == [[6, 22, 38], [54, 70, 86]]).all()
         assert (it.operands[1] == a.sum(axis=2)).all()
 
+

From noreply at buildbot.pypy.org  Wed Jun 12 21:16:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 21:16:10 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Still trying to figure out what
 _stm_nonrecord_barrier() should return to make the most sense for the tests,
 but I think I
Message-ID: <20130612191610.165961C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r101:b853700f30eb
Date: 2013-06-12 21:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/b853700f30eb/

Log:	Still trying to figure out what _stm_nonrecord_barrier() should
	return to make the most sense for the tests, but I think I have it
	now.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -120,7 +120,7 @@
               /* we update P_prev->h_revision as a shortcut */
               /* XXX check if this really gives a worse performance than only
                  doing this write occasionally based on a counter in d */
-              //P_prev->h_revision = v;  XXX re-enable!
+              P_prev->h_revision = v;
               P = (gcptr)v;
               v = ACCESS_ONCE(P->h_revision);
               if (!(v & 1))  // "is a pointer", i.e. "has a more recent rev"
@@ -229,6 +229,50 @@
     }
 }
 
+static gcptr _match_public_to_private(gcptr P, gcptr pubobj, gcptr privobj)
+{
+  while ((pubobj->h_revision & 3) == 0)
+    {
+      assert(pubobj != P);
+      pubobj = (gcptr)pubobj->h_revision;
+    }
+  if (pubobj == P)
+    {
+      assert(!(privobj->h_tid & GCFLAG_PUBLIC));
+      assert(is_private(privobj));
+      fprintf(stderr, "-public_to_private-> %p private\n", privobj);
+      return privobj;
+    }
+  return NULL;
+}
+
+static gcptr _find_public_to_private(gcptr P)
+{
+  gcptr R;
+  wlog_t *item;
+  struct tx_descriptor *d = thread_descriptor;
+
+  G2L_LOOP_FORWARD(d->public_to_private, item)
+    {
+      R = _match_public_to_private(P, item->addr, item->val);
+      if (R != NULL)
+        return R;
+
+    } G2L_LOOP_END;
+
+  long i, size = d->public_descriptor->stolen_objects.size;
+  gcptr *items = d->public_descriptor->stolen_objects.items;
+
+  for (i = 0; i < size; i += 2)
+    {
+      R = _match_public_to_private(P, items[i], items[i + 1]);
+      if (R != NULL)
+        return R;
+    }
+
+  return NULL;
+}
+
 gcptr _stm_nonrecord_barrier(gcptr P)
 {
   /* follows the logic in stm_DirectReadBarrier() */
@@ -247,6 +291,7 @@
 
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
+      /* private too, with a backup copy */
       assert(!(P->h_revision & 1));
       fprintf(stderr, "private_from_protected\n");
       return P;
@@ -254,36 +299,27 @@
 
   if (P->h_tid & GCFLAG_PUBLIC)
     {
-      while (1)
+      fprintf(stderr, "public ");
+
+      while (v = P->h_revision, !(v & 1))
         {
+          if (v & 2)
+            {
+              gcptr L = _find_public_to_private(P);
+              if (L != NULL)
+                return L;
+              goto follow_stub;
+            }
+
+          P = (gcptr)v;
           assert(P->h_tid & GCFLAG_PUBLIC);
-          fprintf(stderr, "public ");
+          fprintf(stderr, "-> %p public ", P);
+        }
 
-          wlog_t *item;
-          gcptr L;
-          G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
+      gcptr L = _find_public_to_private(P);
+      if (L != NULL)
+        return L;
 
-          L = item->val;
-        found_in_stolen_objects:
-          assert(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
-          assert(!(L->h_tid & GCFLAG_PUBLIC));
-          assert(is_private(L));
-          fprintf(stderr, "-public_to_private-> %p private\n", L);
-          return L;
-
-        no_private_obj:;
-          L = _stm_find_stolen_objects(d, P);
-          if (L != NULL)
-            goto found_in_stolen_objects;
-
-          v = ACCESS_ONCE(P->h_revision);
-          if (v & 1)
-            break;
-          if (v & 2)
-            goto follow_stub;
-          P = (gcptr)v;
-          fprintf(stderr, "-> %p ", P);
-        }
       if (UNLIKELY(v > d->start_time))
         {
           fprintf(stderr, "too recent!\n");
@@ -312,6 +348,11 @@
           P = (gcptr)P->h_revision;     /* the backup copy */
           fprintf(stderr, "-backup-> %p ", P);
         }
+      if (!(P->h_tid & GCFLAG_PUBLIC))
+        {
+          fprintf(stderr, "protected by someone else!\n");
+          return (gcptr)-1;
+        }
     }
   goto restart_all;
 }
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -564,3 +564,5 @@
     r = p.h_revision
     assert (r % 4) == 0
     return ffi.cast("gcptr", r)
+
+nrb_protected = ffi.cast("gcptr", -1)
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -2,7 +2,7 @@
 from support import *
 
 
-SHORTCUT = False   # XXXXXXXXXXXXXXXXX
+SHORTCUT = True
 
 
 def setup_function(f):
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -214,14 +214,17 @@
             ptr = self.nonrecord_barrier(p.ptr)
             has_private_copy = p.obj in self.current_rev.content
             if has_private_copy:
-                assert ptr != ffi.NULL and self.is_private(ptr)
+                assert ptr != ffi.NULL and ptr != nrb_protected
+                assert self.is_private(ptr)
                 content = self.current_rev.content[p.obj]
             else:
                 try:
                     content = self.current_rev._try_read(p.obj)
                 except model.Deleted:
-                    assert ptr == ffi.NULL
+                    assert ptr == ffi.NULL or ptr == nrb_protected
                     continue
+                if ptr == nrb_protected:
+                    continue    # not much we can do
                 assert ptr != ffi.NULL and not self.is_private(ptr)
 
             self.check_not_free(ptr)

From noreply at buildbot.pypy.org  Wed Jun 12 21:22:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 21:22:44 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix the next issue
Message-ID: <20130612192244.8E3171C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r102:5eaf6b65867f
Date: 2013-06-12 21:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/5eaf6b65867f/

Log:	Fix the next issue

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -511,7 +511,18 @@
       v = ACCESS_ONCE(R->h_revision);
       if (!(v & 1))               // "is a pointer", i.e.
         {                         //   "has a more recent revision"
-          return 0;
+          if (R->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+            {
+              /* such an object R might be listed in list_of_read_objects
+                 before it was turned from protected to private */
+              continue;
+            }
+          else
+            {
+              fprintf(stderr, "validation failed: "
+                      "%p has a more recent revision\n", R);
+              return 0;
+            }
         }
       if (v >= LOCKED)            // locked
         {
@@ -524,7 +535,11 @@
           else
             {
               if (v != d->my_lock)         // not locked by me: conflict
-                return 0;
+                {
+                  fprintf(stderr, "validation failed: "
+                          "%p is locked by another thread\n", R);
+                  return 0;
+                }
             }
         }
     }

From noreply at buildbot.pypy.org  Wed Jun 12 21:56:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 21:56:36 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next fix
Message-ID: <20130612195636.53EFB1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r103:8fc7970d87ba
Date: 2013-06-12 21:56 +0200
http://bitbucket.org/pypy/stmgc/changeset/8fc7970d87ba/

Log:	Next fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -193,6 +193,7 @@
       /*assert(P->h_revision & 1);*/
     }
 
+  fprintf(stderr, "readobj: %p\n", P);
   gcptrlist_insert(&d->list_of_read_objects, P);
 
  add_in_recent_reads_cache:
@@ -231,6 +232,7 @@
 
 static gcptr _match_public_to_private(gcptr P, gcptr pubobj, gcptr privobj)
 {
+  gcptr org_pubobj = pubobj;
   while ((pubobj->h_revision & 3) == 0)
     {
       assert(pubobj != P);
@@ -238,8 +240,11 @@
     }
   if (pubobj == P)
     {
+      assert(!(org_pubobj->h_tid & GCFLAG_STUB));
       assert(!(privobj->h_tid & GCFLAG_PUBLIC));
       assert(is_private(privobj));
+      if (P != org_pubobj)
+        fprintf(stderr, "| actually %p ", org_pubobj);
       fprintf(stderr, "-public_to_private-> %p private\n", privobj);
       return privobj;
     }
@@ -305,6 +310,7 @@
         {
           if (v & 2)
             {
+              fprintf(stderr, "stub ");
               gcptr L = _find_public_to_private(P);
               if (L != NULL)
                 return L;
@@ -337,12 +343,12 @@
   if (STUB_THREAD(P) == d->public_descriptor)
     {
       P = (gcptr)(v - 2);
-      fprintf(stderr, "stub -> %p ", P);
+      fprintf(stderr, "-> %p ", P);
     }
   else
     {
       P = (gcptr)(v - 2);
-      fprintf(stderr, "stub -foreign-> %p ", P);
+      fprintf(stderr, "-foreign-> %p ", P);
       if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
         {
           P = (gcptr)P->h_revision;     /* the backup copy */
@@ -550,6 +556,16 @@
 {
   d->start_time = GetGlobalCurTime(d);   // copy from the global time
   fprintf(stderr, "et.c: ValidateNow: %ld\n", (long)d->start_time);
+
+  /* subtle: we have to normalize stolen objects, because doing so
+     might add a few extra objects in the list_of_read_objects */
+  if (d->public_descriptor->stolen_objects.size != 0)
+    {
+      spinlock_acquire(d->public_descriptor->collection_lock, 'N');
+      stm_normalize_stolen_objects(d);
+      spinlock_release(d->public_descriptor->collection_lock);
+    }
+
   if (!ValidateDuringTransaction(d, 0))
     AbortTransaction(ABRT_VALIDATE_INFLIGHT);
 }
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -183,6 +183,14 @@
         assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
 
         g2l_insert(&d->public_to_private, B, L);
+
+        /* to be on the safe side */
+        fxcache_remove(&d->recent_reads_cache, B);
+
+        /* but this is definitely needed: all keys in public_to_private
+           must appear in list_of_read_objects */
+        fprintf(stderr, "n.readobj: %p\n", B);
+        gcptrlist_insert(&d->list_of_read_objects, B);
     }
     gcptrlist_clear(&d->public_descriptor->stolen_objects);
 }
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -57,7 +57,7 @@
         text = '%d.%d$ %s\n' % (self.seed, self.counter, text)
         sys.stderr.write(text)
         self.counter += 1
-        #if text.startswith('261035.184$'):
+        #if text.startswith('261225.987$'):
         #    import pdb; pdb.set_trace()
 
     def check_not_free(self, ptr):
@@ -107,8 +107,9 @@
                 self.current_rev.write(r.obj, index, p.obj)
                 if not self.is_private(r.ptr):
                     self.current_rev.check_not_outdated(r.obj)
-            except (model.Deleted, model.Conflict):
+            except (model.Deleted, model.Conflict), e:
                 # abort! try to reproduce with C code
+                self.dump('expecting abort: %r' % (e,))
                 self.expected_abort()
                 lib.setptr(r.ptr, index, p.ptr)   # should abort
                 raise MissingAbort

From noreply at buildbot.pypy.org  Wed Jun 12 22:05:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 22:05:05 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Yes, now all of test_random passes
Message-ID: <20130612200505.2F6EB1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r104:a61854f3b10b
Date: 2013-06-12 22:04 +0200
http://bitbucket.org/pypy/stmgc/changeset/a61854f3b10b/

Log:	Yes, now all of test_random passes

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -73,7 +73,8 @@
 
  not_found:
     stub = stm_stub_malloc(sd->foreign_pd);
-    stub->h_tid = obj->h_tid | GCFLAG_PUBLIC | GCFLAG_STUB;
+    stub->h_tid = (obj->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
+                                                   | GCFLAG_STUB;
     stub->h_revision = ((revision_t)obj) | 2;
     g2l_insert(&sd->all_stubs, obj, stub);
 

From noreply at buildbot.pypy.org  Wed Jun 12 22:31:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 22:31:45 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Pom pom pom
Message-ID: <20130612203145.E68821C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r105:e7072b7314b1
Date: 2013-06-12 22:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/e7072b7314b1/

Log:	Pom pom pom

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -238,7 +238,8 @@
       assert(pubobj != P);
       pubobj = (gcptr)pubobj->h_revision;
     }
-  if (pubobj == P)
+  if (pubobj == P || ((P->h_revision & 3) == 2 &&
+                      pubobj->h_revision == P->h_revision))
     {
       assert(!(org_pubobj->h_tid & GCFLAG_STUB));
       assert(!(privobj->h_tid & GCFLAG_PUBLIC));

From noreply at buildbot.pypy.org  Wed Jun 12 23:02:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 23:02:04 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Pass another iteration
Message-ID: <20130612210204.82AA21C1241@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r108:bb408c71bf7f
Date: 2013-06-12 23:00 +0200
http://bitbucket.org/pypy/stmgc/changeset/bb408c71bf7f/

Log:	Pass another iteration

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -154,6 +154,7 @@
             except (model.Deleted, model.Conflict):
                 # abort! try to reproduce with C code
                 self.expected_abort()
+                lib.stm_clear_read_cache()
                 lib.stm_read_barrier(p.ptr)          # should abort
                 raise MissingAbort
 
@@ -480,5 +481,5 @@
 
 def test_more_multi_thread():
     #py.test.skip("more random tests")
-    for i in range(7, 1000):
+    for i in range(9, 1000):
         yield test_multi_thread, i

From noreply at buildbot.pypy.org  Wed Jun 12 23:02:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 23:02:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Clear the fxcache when we want to be
 sure to get a conflict.
Message-ID: <20130612210202.04A271C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r106:e11ad4a6f501
Date: 2013-06-12 22:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/e11ad4a6f501/

Log:	Clear the fxcache when we want to be sure to get a conflict.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -39,6 +39,10 @@
 {
   return is_private(P);
 }
+void stm_clear_read_cache(void)
+{
+  fxcache_clear(&thread_descriptor->recent_reads_cache);
+}
 
 /************************************************************/
 
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -169,6 +169,7 @@
 int _stm_is_private(gcptr);  /* debugging */
 gcptr stm_get_private_from_protected(long);  /* debugging */
 gcptr stm_get_read_obj(long);  /* debugging */
+void stm_clear_read_cache(void);  /* debugging */
 gcptr stmgc_duplicate(gcptr);
 
 int DescriptorInit(void);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -72,6 +72,7 @@
     gcptr stm_get_private_from_protected(long index);
     gcptr stm_get_read_obj(long index);
     void *STUB_THREAD(gcptr);
+    void stm_clear_read_cache(void);
 
     gcptr getptr(gcptr, long);
     void setptr(gcptr, long, gcptr);
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -126,9 +126,11 @@
             pobj = self.current_rev.read(r.obj, index)
             if not self.is_private(r.ptr):
                 self.current_rev.check_not_outdated(r.obj)
-        except (model.Deleted, model.Conflict):
+        except (model.Deleted, model.Conflict), e:
             # abort! try to reproduce with C code
+            self.dump('expecting abort: %r' % (e,))
             self.expected_abort()
+            lib.stm_clear_read_cache()
             lib.getptr(r.ptr, index)             # should abort
             raise MissingAbort
 

From noreply at buildbot.pypy.org  Wed Jun 12 23:02:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 23:02:03 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Disable testing for that,
	doesn't seem to give much
Message-ID: <20130612210203.4EF531C10DD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r107:211b51d42996
Date: 2013-06-12 22:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/211b51d42996/

Log:	Disable testing for that, doesn't seem to give much

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -41,7 +41,7 @@
         self.sync_wait = sync_wait
         self.counter = 0
         self.current_rev = None
-        self.expecting_gap_in_commit_time = 2
+        #self.expecting_gap_in_commit_time = 2
         self.dump('run')
         #
         if sync.prebuilt_object is None:
@@ -290,12 +290,12 @@
             self.commit()
         except model.Conflict, e:
             self.dump("interruptible_transaction expecting %s" % (e,))
-            if isinstance(e, model.ReadWriteConflict):
-                self.expecting_gap_in_commit_time += 2
+            #if isinstance(e, model.ReadWriteConflict):
+            #    self.expecting_gap_in_commit_time += 2
             self.expected_abort()
             return 0
         #
-        self.expecting_gap_in_commit_time = 2
+        #self.expecting_gap_in_commit_time = 2
         if restart:
             self.dump("interruptible_transaction break")
             return 1
@@ -308,16 +308,16 @@
         t = lib.get_start_time()
         self.current_rev.start_time = t
         if self.current_rev.previous is not None:
-            t_prev = t - self.expecting_gap_in_commit_time
+            t_prev = t - 2  #self.expecting_gap_in_commit_time
             if hasattr(self.current_rev.previous, 'commit_time'):
-                assert self.current_rev.previous.commit_time == t_prev
+                pass #assert self.current_rev.previous.commit_time == t_prev
             else:
                 self.current_rev.previous.commit_time = t_prev
 
     def possibly_update_time(self):
         t = lib.get_start_time()
-        t_prev = t - self.expecting_gap_in_commit_time
-        assert self.current_rev.previous.commit_time == t_prev
+        #t_prev = t - self.expecting_gap_in_commit_time
+        #assert self.current_rev.previous.commit_time == t_prev
         self.current_rev.start_time = t
 
     def commit(self):
@@ -479,6 +479,6 @@
 
 
 def test_more_multi_thread():
-    py.test.skip("more random tests")
-    for i in range(2, 1000):
+    #py.test.skip("more random tests")
+    for i in range(7, 1000):
         yield test_multi_thread, i

From noreply at buildbot.pypy.org  Wed Jun 12 23:02:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 23:02:05 +0200 (CEST)
Subject: [pypy-commit] stmgc default: A suspicious-looking fix,
	but a valid fix nevertheless
Message-ID: <20130612210205.C2F971C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r109:c9938a7b55fb
Date: 2013-06-12 23:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/c9938a7b55fb/

Log:	A suspicious-looking fix, but a valid fix nevertheless

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -105,7 +105,6 @@
            the foreign thread's collection_lock, so we can read/write the
            flags
         */
-        assert(B->h_tid & GCFLAG_BACKUP_COPY);
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
 
         if (B->h_tid & GCFLAG_PUBLIC_TO_PRIVATE) {

From noreply at buildbot.pypy.org  Wed Jun 12 23:05:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Jun 2013 23:05:23 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fixes
Message-ID: <20130612210523.F23C01C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r110:ec1fe577a446
Date: 2013-06-12 23:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/ec1fe577a446/

Log:	fixes

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -112,6 +112,9 @@
                 self.dump('expecting abort: %r' % (e,))
                 self.expected_abort()
                 lib.setptr(r.ptr, index, p.ptr)   # should abort
+                # didn't?  try again by first clearing the fxcache
+                lib.stm_clear_read_cache()
+                lib.setptr(r.ptr, index, p.ptr)
                 raise MissingAbort
 
             lib.setptr(r.ptr, index, p.ptr)   # must not abort
@@ -175,6 +178,9 @@
                 # abort! try to reproduce with C code
                 self.expected_abort()
                 lib.stm_write_barrier(p.ptr)         # should abort
+                # didn't?  try again by first clearing the fxcache
+                lib.stm_clear_read_cache()
+                lib.stm_write_barrier(p.ptr)
                 raise MissingAbort
 
             nptr = lib.stm_write_barrier(p.ptr)
@@ -481,5 +487,5 @@
 
 def test_more_multi_thread():
     #py.test.skip("more random tests")
-    for i in range(9, 1000):
+    for i in range(12, 1000):
         yield test_multi_thread, i

From noreply at buildbot.pypy.org  Thu Jun 13 10:01:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 10:01:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: test_forked_can_thread failed today on
	linux64. No clue, but try
Message-ID: <20130613080127.E8CA71C00B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64869:9b623bc48b59
Date: 2013-06-13 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/9b623bc48b59/

Log:	test_forked_can_thread failed today on linux64. No clue, but try
	randomly to increase the kill timer, in case it was due to very high
	loads.

diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py
--- a/pypy/module/thread/test/test_fork.py
+++ b/pypy/module/thread/test/test_fork.py
@@ -28,7 +28,7 @@
                 if pid == 0:
                     os._exit(0)
                 else:
-                    self.timeout_killer(pid, 5)
+                    self.timeout_killer(pid, 10)
                     exitcode = os.waitpid(pid, 0)[1]
                     assert exitcode == 0 # if 9, process was killed by timer!
             finally:
@@ -54,7 +54,7 @@
                 thread.start_new_thread(lambda: None, ())
                 os._exit(0)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 assert exitcode == 0 # if 9, process was killed by timer!
 
@@ -73,7 +73,7 @@
                 signal.signal(signal.SIGUSR1, signal.SIG_IGN)
                 os._exit(42)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 feedback.append(exitcode)
 

From noreply at buildbot.pypy.org  Thu Jun 13 10:15:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 10:15:42 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix,
 at least for the tests: don't use "-1" initially for all threads
Message-ID: <20130613081542.BA22D1C00B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r111:c3ea72ae1029
Date: 2013-06-13 10:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/c3ea72ae1029/

Log:	Fix, at least for the tests: don't use "-1" initially for all
	threads

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1274,7 +1274,7 @@
       d->my_lock = LOCKED + 2 * i;
       assert(d->my_lock & 1);
       assert(d->my_lock >= LOCKED);
-      stm_private_rev_num = -1;
+      stm_private_rev_num = -d->my_lock;
       d->private_revision_ref = &stm_private_rev_num;
       d->max_aborts = -1;
       pd->descriptor = d;

From noreply at buildbot.pypy.org  Thu Jun 13 10:36:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 10:36:35 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tweak tests
Message-ID: <20130613083635.60A851C153B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r112:573298ebb564
Date: 2013-06-13 10:33 +0200
http://bitbucket.org/pypy/stmgc/changeset/573298ebb564/

Log:	Tweak tests

diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -530,7 +530,7 @@
     backup  = (p.h_tid & GCFLAG_BACKUP_COPY) != 0
     stub    = (p.h_tid & GCFLAG_STUB) != 0
     assert private + public + backup <= 1
-    assert stub <= public
+    assert (public, stub) != (False, True)
     if private:
         return "private"
     if public:
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -210,7 +210,7 @@
         return lib._stm_nonrecord_barrier(ptr)
 
     def is_private(self, ptr):
-        return lib._stm_is_private(ptr)
+        return classify(ptr) == "private"
 
     def check_valid(self, lst):
         lst = list(lst)
@@ -349,7 +349,7 @@
         self.interruptible_transaction = False
         self.startrev()
         #
-        self.steps_remaining = 10000
+        self.steps_remaining = 1000
         #
         self.run_me(do_wait=False)
         #

From noreply at buildbot.pypy.org  Thu Jun 13 10:36:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 10:36:36 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix "stealing again the same object"
Message-ID: <20130613083636.8B7831C153B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r113:b9568f14bdd9
Date: 2013-06-13 10:36 +0200
http://bitbucket.org/pypy/stmgc/changeset/b9568f14bdd9/

Log:	Fix "stealing again the same object"

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -109,6 +109,10 @@
 
         if (B->h_tid & GCFLAG_PUBLIC_TO_PRIVATE) {
             /* already stolen */
+            assert(B->h_tid & GCFLAG_PUBLIC);
+            fprintf(stderr, "already stolen: %p -> %p <-> %p\n", P, L, B);
+            L = B;
+            goto already_stolen;
         }
         else {
             B->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
@@ -116,18 +120,26 @@
                don't want to walk over the feet of the foreign thread
             */
             gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
+            fprintf(stderr, "stolen: %p -> %p <-> %p\n", P, L, B);
+            L = B;
         }
-        fprintf(stderr, "stolen: %p -> %p - - -> %p\n", P, B, L);
-        L = B;
     }
     else {
-        fprintf(stderr, "stolen: %p -> %p\n", P, L);
+        if (L->h_tid & GCFLAG_PUBLIC) {
+            /* already stolen */
+            fprintf(stderr, "already stolen: %p -> %p\n", P, L);
+            goto already_stolen;
+        }
+        else {
+            fprintf(stderr, "stolen: %p -> %p\n", P, L);
+        }
     }
 
     /* Here L is a protected (or backup) copy, and we own the foreign
        thread's collection_lock, so we can read/write the flags.  Change
        it from protected to public.
     */
+    assert(!(L->h_tid & GCFLAG_PUBLIC));
     L->h_tid |= GCFLAG_PUBLIC;
 
     /* Note that all protected or backup copies have a h_revision that
@@ -163,6 +175,7 @@
     */
     smp_wmb();
 
+ already_stolen:
     /* update the original P->h_revision to point directly to L */
     P->h_revision = (revision_t)L;
 

From noreply at buildbot.pypy.org  Thu Jun 13 11:32:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 11:32:51 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add demo1.c from ../c3.  Works.
Message-ID: <20130613093251.891151C1241@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r114:f5727433a73e
Date: 2013-06-13 11:32 +0200
http://bitbucket.org/pypy/stmgc/changeset/f5727433a73e/

Log:	Add demo1.c from ../c3. Works.

diff --git a/c3/Makefile b/c4/Makefile
copy from c3/Makefile
copy to c4/Makefile
--- a/c3/Makefile
+++ b/c4/Makefile
@@ -2,33 +2,17 @@
 # Run with "make -jN" for maximum randomness.
 #
 
-debug: debug-demo1 debug-demo2
+debug: debug-demo1
 
 clean:
-	rm -f debug-demo1 debug-demo2
+	rm -f debug-demo1
 
 
-tests: iteration-1 iteration-2 iteration-3 iteration-4
+H_FILES = atomic_ops.h stmgc.h stmimpl.h \
+          dbgmem.h et.h fprintcolor.h lists.h steal.h stmsync.h
+C_FILES = dbgmem.c et.c fprintcolor.c lists.c steal.c stmsync.c
 
-all-builds: build-demo1 build-demo2 build-demo3 build-demo4 build-demo5
-
-iteration-1: all-builds
-	+make test-demo5 test-demo1 test-demo2 test-demo3 test-demo4
-
-iteration-2: all-builds
-	+make test-demo2 test-demo3 test-demo4 test-demo5 test-demo1
-
-iteration-3: all-builds
-	+make test-demo3 test-demo4 test-demo5 test-demo1 test-demo2
-
-iteration-4: all-builds
-	+make test-demo4 test-demo5 test-demo1 test-demo2 test-demo3
-
-
-H_FILES = et.h lists.h nursery.h gcpage.h stmsync.h dbgmem.h fprintcolor.h stmgc.h atomic_ops.h stmimpl.h
-C_FILES = et.c lists.c nursery.c gcpage.c stmsync.c dbgmem.c fprintcolor.c
-
-DEBUG = -g -DGC_NURSERY=0x10000 -D_GC_DEBUG=1
+DEBUG = -g -DGC_NURSERY=0x10000  #-D_GC_DEBUG=1
 
 
 build-%: %.c ${H_FILES} ${C_FILES}
diff --git a/c3/demo1.c b/c4/demo1.c
copy from c3/demo1.c
copy to c4/demo1.c
--- a/c3/demo1.c
+++ b/c4/demo1.c
@@ -106,7 +106,7 @@
 
 static sem_t done;
 
-extern void stmgcpage_possibly_major_collect(int force);  /* temp */
+//extern void stmgcpage_possibly_major_collect(int force);  /* temp */
 
 static int thr_mynum = 0;
 
@@ -128,7 +128,7 @@
         w_node->value = start + i;
         stm_push_root((gcptr)w_node);
         stm_perform_transaction((gcptr)w_node, insert1);
-        stmgcpage_possibly_major_collect(0); /* temp */
+        //stmgcpage_possibly_major_collect(0); /* temp */
         w_node = (struct node *)stm_pop_root();
     }
     stm_finalize();

From noreply at buildbot.pypy.org  Thu Jun 13 14:36:07 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Thu, 13 Jun 2013 14:36:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Added tag pypy-2.1-beta1-arm for
	changeset 9b623bc48b59
Message-ID: <20130613123607.DFEFB1C073E@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64870:02ae35f0be09
Date: 2013-06-13 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/02ae35f0be09/

Log:	Added tag pypy-2.1-beta1-arm for changeset 9b623bc48b59

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,4 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm

From noreply at buildbot.pypy.org  Thu Jun 13 14:36:09 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Thu, 13 Jun 2013 14:36:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: Added tag pypy-2.1-beta1-arm for
	changeset ab0dd631c220
Message-ID: <20130613123609.2B3391C073E@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64871:18f1a3e22254
Date: 2013-06-13 14:35 +0200
http://bitbucket.org/pypy/pypy/changeset/18f1a3e22254/

Log:	Added tag pypy-2.1-beta1-arm for changeset ab0dd631c220

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -4,3 +4,5 @@
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm

From noreply at buildbot.pypy.org  Thu Jun 13 15:09:13 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 15:09:13 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix of the test by being more
 adaptable to the logic that might sometimes
Message-ID: <20130613130913.2097C1C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r115:a3469e49931a
Date: 2013-06-13 15:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/a3469e49931a/

Log:	Fix of the test by being more adaptable to the logic that might
	sometimes detect conflicts even if we have private copies.

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -98,46 +98,65 @@
         self.aborted_rev = self.current_rev
         self.current_rev = None
 
+    def cancel_expected_abort(self):
+        lib.stm_set_max_aborts(0)
+        self.expected_conflict = False
+        self.current_rev = self.aborted_rev
+        del self.aborted_rev
+
     def set_into_root(self, p, index):
         r = self._r
         if r != emptypair:
             self.check(r)
             self.check(p)
-            try:
-                self.current_rev.write(r.obj, index, p.obj)
-                if not self.is_private(r.ptr):
-                    self.current_rev.check_not_outdated(r.obj)
-            except (model.Deleted, model.Conflict), e:
-                # abort! try to reproduce with C code
-                self.dump('expecting abort: %r' % (e,))
-                self.expected_abort()
-                lib.setptr(r.ptr, index, p.ptr)   # should abort
-                # didn't?  try again by first clearing the fxcache
-                lib.stm_clear_read_cache()
-                lib.setptr(r.ptr, index, p.ptr)
-                raise MissingAbort
-
-            lib.setptr(r.ptr, index, p.ptr)   # must not abort
+            self.do(r, (self.current_rev.write, r.obj, index, p.obj),
+                       (lib.setptr, r.ptr, index, p.ptr))
             self.possibly_update_time()
             self.dump('set_into_root(%r, %r, %r)' % (r.obj, index, p.obj))
 
+    def do(self, r, model_operation, real_operation):
+        abort = None
+        try:
+            x = model_operation[0](*model_operation[1:])
+        except (model.Deleted, model.Conflict), e:
+            # the model says that we should definitely get an abort
+            abort = e.__class__.__name__
+        else:
+            if r.obj.created_in_revision is not self.current_rev:
+                try:
+                    self.current_rev.check_not_outdated(r.obj)
+                except model.Deleted:
+                    if not self.is_private(r.ptr):
+                        # the model says that we should definitely get an abort
+                        abort = "CheckDeleted"
+                    else:
+                        # the model says that we *might* get an abort
+                        abort = "MaybeDeleted"
+        #
+        if abort:
+            self.dump('expecting abort: %r' % (abort,))
+            self.expected_abort()
+        y = real_operation[0](*real_operation[1:])
+        if abort:
+            # didn't abort? try again by first clearing the fxcache
+            lib.stm_clear_read_cache()
+            y = real_operation[0](*real_operation[1:])
+            # still didn't abort?
+            if abort != "MaybeDeleted":
+                raise MissingAbort
+            else:
+                # ok, it's fine if we don't actually get an abort
+                self.cancel_expected_abort()
+        #
+        return x, y
+
     def get_ref(self, r, index):
         self.check(r)
         if r == emptypair:
             return emptypair
-        try:
-            pobj = self.current_rev.read(r.obj, index)
-            if not self.is_private(r.ptr):
-                self.current_rev.check_not_outdated(r.obj)
-        except (model.Deleted, model.Conflict), e:
-            # abort! try to reproduce with C code
-            self.dump('expecting abort: %r' % (e,))
-            self.expected_abort()
-            lib.stm_clear_read_cache()
-            lib.getptr(r.ptr, index)             # should abort
-            raise MissingAbort
-
-        pptr = lib.getptr(r.ptr, index)
+        self.dump('get_ref(%s, %d)' % (r, index))
+        pobj, pptr = self.do(r, (self.current_rev.read, r.obj, index),
+                                (lib.getptr, r.ptr, index))
         self.possibly_update_time()
         p = pair(pobj, pptr)
         self.check(p)
@@ -150,18 +169,8 @@
     def read_barrier(self, p):
         if p != emptypair:
             self.check(p)
-            try:
-                self.current_rev.read_barrier(p.obj)
-                if not self.is_private(p.ptr):
-                    self.current_rev.check_not_outdated(p.obj)
-            except (model.Deleted, model.Conflict):
-                # abort! try to reproduce with C code
-                self.expected_abort()
-                lib.stm_clear_read_cache()
-                lib.stm_read_barrier(p.ptr)          # should abort
-                raise MissingAbort
-
-            nptr = lib.stm_read_barrier(p.ptr)
+            _, nptr = self.do(p, (self.current_rev.read_barrier, p.obj),
+                                 (lib.stm_read_barrier, p.ptr))
             self.possibly_update_time()
             p = pair(p.obj, nptr)
             self.check(p)
@@ -170,20 +179,8 @@
     def write_barrier(self, p):
         if p != emptypair:
             self.check(p)
-            try:
-                self.current_rev.write_barrier(p.obj)
-                if not self.is_private(p.ptr):
-                    self.current_rev.check_not_outdated(p.obj)
-            except (model.Deleted, model.Conflict):
-                # abort! try to reproduce with C code
-                self.expected_abort()
-                lib.stm_write_barrier(p.ptr)         # should abort
-                # didn't?  try again by first clearing the fxcache
-                lib.stm_clear_read_cache()
-                lib.stm_write_barrier(p.ptr)
-                raise MissingAbort
-
-            nptr = lib.stm_write_barrier(p.ptr)
+            _, nptr = self.do(p, (self.current_rev.write_barrier, p.obj),
+                                 (lib.stm_write_barrier, p.ptr))
             self.possibly_update_time()
             p = pair(p.obj, nptr)
             self.check(p)
@@ -487,5 +484,5 @@
 
 def test_more_multi_thread():
     #py.test.skip("more random tests")
-    for i in range(12, 1000):
+    for i in range(326//2, 1000):
         yield test_multi_thread, i

From noreply at buildbot.pypy.org  Thu Jun 13 15:11:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 15:11:40 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Re-skip this never-ending part
Message-ID: <20130613131140.DDC501C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r116:f1ccf5bbcb6f
Date: 2013-06-13 15:11 +0200
http://bitbucket.org/pypy/stmgc/changeset/f1ccf5bbcb6f/

Log:	Re-skip this never-ending part

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -483,6 +483,6 @@
 
 
 def test_more_multi_thread():
-    #py.test.skip("more random tests")
-    for i in range(326//2, 1000):
+    py.test.skip("more random tests")
+    for i in range(580//2, 1000):
         yield test_multi_thread, i

From noreply at buildbot.pypy.org  Thu Jun 13 15:48:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 15:48:33 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Added tag c4-without-gc for changeset
	f1ccf5bbcb6f
Message-ID: <20130613134833.A95E71C03B1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r117:7c75ea00f9fd
Date: 2013-06-13 15:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/7c75ea00f9fd/

Log:	Added tag c4-without-gc for changeset f1ccf5bbcb6f

diff --git a/.hgtags b/.hgtags
new file mode 100644
--- /dev/null
+++ b/.hgtags
@@ -0,0 +1,1 @@
+f1ccf5bbcb6f01fb995622f2b569e9858fbea08a c4-without-gc

From noreply at buildbot.pypy.org  Thu Jun 13 18:21:05 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Jun 2013 18:21:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: A new copyright holder. Thanks!
Message-ID: <20130613162105.89A921C0400@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r64873:d3c70e7a2523
Date: 2013-06-13 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/d3c70e7a2523/

Log:	A new copyright holder. Thanks!

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike

From noreply at buildbot.pypy.org  Thu Jun 13 18:21:04 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Jun 2013 18:21:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add implementation of _tkinter,
	using cffi bindings.
Message-ID: <20130613162104.55D1B1C03B1@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r64872:2c9002e840f0
Date: 2013-06-13 14:06 +0200
http://bitbucket.org/pypy/pypy/changeset/2c9002e840f0/

Log:	Add implementation of _tkinter, using cffi bindings.

diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,43 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+from .tklib import tklib, tkffi
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {			/* The internal representation: */
+	long longValue;		/*   - an long integer value. */
+	double doubleValue;	/*   - a double-precision floating value. */
+	struct {		/*   - internal rep as two pointers. */
+	    void *ptr1;
+	    void *ptr2;
+	} twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)

From noreply at buildbot.pypy.org  Thu Jun 13 20:02:23 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 13 Jun 2013 20:02:23 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Implement the c_index and the
 f_index flags on the nditer class
Message-ID: <20130613180223.EC27D1C029E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64874:cbc60b20e6c2
Date: 2013-06-13 20:01 +0200
http://bitbucket.org/pypy/pypy/changeset/cbc60b20e6c2/

Log:	Implement the c_index and the f_index flags on the nditer class

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -201,18 +201,52 @@
                                     shape, backward)
     return MultiDimViewIterator(imp, imp.dtype, imp.start, r[0], r[1], shape)
 
+def is_backward(imp, order):
+    if order == 'K' or (order == 'C' and imp.order == 'C'):
+        return False
+    elif order =='F' and imp.order == 'C':
+        return True
+    else:
+        raise NotImplementedError('not implemented yet')
+
 def get_external_loop_iter(space, order, arr, shape):
     imp = arr.implementation
-    if order == 'K' or (order == 'C' and imp.order == 'C'):
-        backward = False
-    elif order =='F' and imp.order == 'C':
-        backward = True
-    else:
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-                'not implemented yet'))
+
+    backward = is_backward(imp, order)
 
     return SliceIterator(arr, imp.strides, imp.backstrides, shape, order=order, backward=backward)
 
+class IndexIterator(object):
+    def __init__(self, shape, backward=False):
+        self.shape = shape
+        self.index = [0] * len(shape)
+        self.backward = backward
+        self.called = False
+
+    def next(self):
+        # TODO It's probably possible to refactor all the "next" method from each iterator
+        if not self.called:
+            self.called = True
+            return
+        for i in range(len(self.shape) - 1, -1, -1):
+            if self.index[i] < self.shape[i] - 1:
+                self.index[i] += 1
+                break
+            else:
+                self.index[i] = 0
+
+    def getvalue(self):
+        if not self.called:
+            return 0
+        if not self.backward:
+            ret = self.index[-1]
+            for i in range(len(self.shape) - 2, -1, -1):
+                ret += self.index[i] * self.shape[i - 1]
+        else:
+            ret = self.index[0]
+            for i in range(1, len(self.shape)):
+                ret += self.index[i] * self.shape[i - 1]
+        return ret
 
 class W_NDIter(W_Root):
 
@@ -229,6 +263,7 @@
         self.refs_ok = False
         self.reduce_ok = False
         self.zerosize_ok = False
+        self.index_iter = None
         if space.isinstance_w(w_seq, space.w_tuple) or \
            space.isinstance_w(w_seq, space.w_list):
             w_seq_as_list = space.listview(w_seq)
@@ -240,6 +275,10 @@
                                      len(self.seq), parse_op_flag)
         self.iters=[]
         self.shape = iter_shape = shape_agreement_multiple(space, self.seq)
+        if self.tracked_index != "":
+            if self.order == "K":
+                self.order = self.seq[0].implementation.order
+            self.index_iter = IndexIterator(iter_shape, backward=self.order != self.tracked_index)
         if self.external_loop:
             for i in range(len(self.seq)):
                 self.iters.append(ExternalLoopIterator(get_external_loop_iter(space, self.order,
@@ -271,6 +310,8 @@
         else:
             raise OperationError(space.w_StopIteration, space.w_None)
         res = []
+        if self.index_iter:
+            self.index_iter.next()
         for i in range(len(self.iters)):
             res.append(self.iters[i].getitem(space, self.seq[i]))
             self.iters[i].next()
@@ -324,12 +365,12 @@
             'not implemented yet'))
 
     def descr_get_has_index(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        return space.wrap(not self.tracked_index == "")
 
     def descr_get_index(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        if self.tracked_index == "":
+            raise OperationError(space.w_ValueError, "Iterator does not have an index")
+        return space.wrap(self.index_iter.getvalue())
 
     def descr_get_has_multi_index(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -66,6 +66,24 @@
             e = ex
         assert e
 
+    def test_index(self):
+        from numpypy import arange, nditer, zeros
+        a = arange(6).reshape(2,3)
+
+        r = []
+        it = nditer(a, flags=['c_index'])
+        assert it.has_index
+        for value in it:
+            r.append((value, it.index))
+        assert r == [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)]
+
+        r = []
+        it = nditer(a, flags=['f_index'])
+        assert it.has_index
+        for value in it:
+            r.append((value, it.index))
+        assert r == [(0, 0), (1, 2), (2, 4), (3, 1), (4, 3), (5, 5)]
+
     def test_interface(self):
         from numpypy import arange, nditer, zeros
         a = arange(6).reshape(2,3)

From noreply at buildbot.pypy.org  Thu Jun 13 20:22:53 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 13 Jun 2013 20:22:53 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Add a test for nditers with
	mixed order
Message-ID: <20130613182253.9751D1C0400@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64875:5a7505174934
Date: 2013-06-13 20:22 +0200
http://bitbucket.org/pypy/pypy/changeset/5a7505174934/

Log:	Add a test for nditers with mixed order

diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -84,6 +84,17 @@
             r.append((value, it.index))
         assert r == [(0, 0), (1, 2), (2, 4), (3, 1), (4, 3), (5, 5)]
 
+    @py.test.mark.xfail(reason="Fortran order not implemented")
+    def test_iters_with_different_order(self):
+        from numpypy import nditer, array
+
+        a = array([[1, 2], [3, 4]], order="C")
+        b = array([[1, 2], [3, 4]], order="F")
+
+        it = nditer([a, b])
+
+        assert list(it) == zip(range(1, 5), range(1, 5))
+
     def test_interface(self):
         from numpypy import arange, nditer, zeros
         a = arange(6).reshape(2,3)

From noreply at buildbot.pypy.org  Thu Jun 13 20:36:23 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 13 Jun 2013 20:36:23 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Fix translation
Message-ID: <20130613183623.5E1A61C1241@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64876:0a0ae7e99b40
Date: 2013-06-13 20:35 +0200
http://bitbucket.org/pypy/pypy/changeset/0a0ae7e99b40/

Log:	Fix translation

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -369,7 +369,7 @@
 
     def descr_get_index(self, space):
         if self.tracked_index == "":
-            raise OperationError(space.w_ValueError, "Iterator does not have an index")
+            raise OperationError(space.w_ValueError, space.wrap("Iterator does not have an index"))
         return space.wrap(self.index_iter.getvalue())
 
     def descr_get_has_multi_index(self, space):

From noreply at buildbot.pypy.org  Thu Jun 13 22:26:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 22:26:58 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Clean-ups in preparation for nursery.c
Message-ID: <20130613202658.E8EF91C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r118:afd0da81c314
Date: 2013-06-13 22:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/afd0da81c314/

Log:	Clean-ups in preparation for nursery.c

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -6,58 +6,18 @@
 #include <sys/mman.h>
 
 #define PAGE_SIZE  4096
-#define MMAP_LENGTH  67108864   /* 64MB */
-
-struct zone_s {
-    struct zone_s *next;
-    char *start;
-    uint8_t active[MMAP_LENGTH / WORD];
-};
+#define MMAP_TOTAL  671088640   /* 640MB */
 
 static pthread_mutex_t malloc_mutex = PTHREAD_MUTEX_INITIALIZER;
-static char *free_zone = NULL, *free_zone_end = NULL;
-static struct zone_s *zones = NULL;
+static char *zone_current = NULL, *zone_end = NULL;
 
-void *stm_malloc(size_t sz)
-{
-    pthread_mutex_lock(&malloc_mutex);
-
-    size_t nb_pages = (sz + PAGE_SIZE - 1) / PAGE_SIZE + 1;
-    if (free_zone_end - free_zone < nb_pages * PAGE_SIZE) {
-        free_zone = mmap(NULL, MMAP_LENGTH, PROT_NONE,
-                         MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-        if (free_zone == NULL || free_zone == MAP_FAILED) {
-            fprintf(stderr, "out of memory: mmap() failed\n");
-            abort();
-        }
-        free_zone_end = free_zone + MMAP_LENGTH;
-        assert((MMAP_LENGTH % PAGE_SIZE) == 0);
-
-        struct zone_s *z = calloc(1, sizeof(struct zone_s));
-        if (z == NULL) {
-            fprintf(stderr, "out of memory: malloc(zone_s) failed\n");
-            abort();
-        }
-        z->start = free_zone;
-        z->next = zones;
-        zones = z;
-    }
-
-    char *result = free_zone;
-    free_zone += nb_pages * PAGE_SIZE;
-    pthread_mutex_unlock(&malloc_mutex);
-
-    result += (-sz) & (PAGE_SIZE-1);
-    assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
-    stm_dbgmem_used_again(result, sz, 1);
-    return result;
-}
 
 static void _stm_dbgmem(void *p, size_t sz, int prot)
 {
-    fprintf(stderr, "_stm_dbgmem(%p, 0x%lx, %d)\n", p, (long)sz, prot);
     if (sz == 0)
         return;
+
+    assert((ssize_t)sz > 0);
     intptr_t align = ((intptr_t)p) & (PAGE_SIZE-1);
     p = ((char *)p) - align;
     sz += align;
@@ -65,64 +25,42 @@
     assert(err == 0);
 }
 
+void *stm_malloc(size_t sz)
+{
+    pthread_mutex_lock(&malloc_mutex);
+
+    if (zone_current == NULL) {
+        zone_current = mmap(NULL, MMAP_TOTAL, PROT_NONE,
+                            MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (zone_current == NULL || zone_current == MAP_FAILED) {
+            fprintf(stderr, "not enough memory: mmap() failed\n");
+            abort();
+        }
+        zone_end = zone_current + MMAP_TOTAL;
+        assert((MMAP_TOTAL % PAGE_SIZE) == 0);
+        _stm_dbgmem(zone_current, MMAP_TOTAL, PROT_NONE);
+    }
+
+    size_t nb_pages = (sz + PAGE_SIZE - 1) / PAGE_SIZE + 1;
+    char *result = zone_current;
+    zone_current += nb_pages * PAGE_SIZE;
+    if (zone_current > zone_end) {
+        fprintf(stderr, "dbgmem.c: %ld MB of memory have been exausted\n",
+                (long)(MMAP_TOTAL / (1024*1024)));
+        abort();
+    }
+    pthread_mutex_unlock(&malloc_mutex);
+
+    result += (-sz) & (PAGE_SIZE-1);
+    assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
+    _stm_dbgmem(result, sz, PROT_READ | PROT_WRITE);
+    return result;
+}
+
 void stm_free(void *p, size_t sz)
 {
-    _stm_dbgmem(p, sz, PROT_READ | PROT_WRITE);
     memset(p, 0xDD, sz);
-    stm_dbgmem_not_used(p, sz, 1);
-}
-
-static void _stm_dbg_mark(char *p, size_t sz, uint8_t marker)
-{
-    long startofs, numofs, i;
-    struct zone_s *z = zones;
-    while (!(z->start <= p && p < (z->start + MMAP_LENGTH))) {
-        z = z->next;
-        assert(z);
-    }
-    startofs = p - z->start;
-    numofs = sz;
-    assert((startofs & (WORD-1)) == 0);
-    assert((numofs & (WORD-1)) == 0);
-    assert(startofs + numofs <= MMAP_LENGTH);
-    startofs /= WORD;
-    numofs /= WORD;
-    for (i=0; i<numofs; i++)
-        z->active[startofs + i] = marker;
-}
-
-void stm_dbgmem_not_used(void *p, size_t sz, int protect)
-{
-    _stm_dbg_mark(p, sz, 0);
-    if (protect)
-        _stm_dbgmem(p, sz, PROT_NONE);
-}
-
-void stm_dbgmem_used_again(void *p, size_t sz, int protect)
-{
-    _stm_dbg_mark(p, sz, 42);
-    if (protect)
-        _stm_dbgmem(p, sz, PROT_READ | PROT_WRITE);
-}
-
-int stm_dbgmem_is_active(void *p1, int allow_outside)
-{
-    char *p = (char *)p1;
-    long startofs;
-    uint8_t result;
-    struct zone_s *z = zones;
-    while (z && !(z->start <= p && p < (z->start + MMAP_LENGTH))) {
-        z = z->next;
-    }
-    if (!z) {
-        assert(allow_outside);
-        return -1;
-    }
-    startofs = p - z->start;
-    startofs /= WORD;
-    result = z->active[startofs];
-    assert(result == 0 || result == 42);
-    return (result != 0);
+    _stm_dbgmem(p, sz, PROT_NONE);
 }
 
 /************************************************************/
diff --git a/c4/dbgmem.h b/c4/dbgmem.h
--- a/c4/dbgmem.h
+++ b/c4/dbgmem.h
@@ -7,18 +7,10 @@
 void *stm_malloc(size_t);
 void stm_free(void *, size_t);
 
-/* Debugging: for tracking which memory regions should be read or not. */
-void stm_dbgmem_not_used(void *p, size_t size, int protect);
-void stm_dbgmem_used_again(void *p, size_t size, int protect);
-int stm_dbgmem_is_active(void *p, int allow_outside);
-
 #else
 
-#define stm_malloc(sz)                 malloc(sz)
-#define stm_free(p,sz)                 free(p)
-#define stm_dbgmem_not_used(p,sz,i)    /* nothing */
-#define stm_dbgmem_used_again(p,sz,i)  /* nothing */
-#define stm_dbgmem_is_active(p,i)      1
+#define stm_malloc(sz)    malloc(sz)
+#define stm_free(p,sz)    free(p)
 
 #endif
 
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -956,7 +956,7 @@
         }
       else
         {
-          //stm_free(B);
+          stm_free(B, stmcb_size(B));
         }
     };
   gcptrlist_clear(&d->private_from_protected);
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -105,6 +105,7 @@
  * thread shuts down.  It is reused the next time a thread starts. */
 struct tx_public_descriptor {
   revision_t collection_lock;
+  NURSERY_FIELDS_DECL
   struct tx_descriptor *descriptor;
   struct stub_block_s *stub_blocks;
   gcptr stub_free_list;
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -26,7 +26,6 @@
 
 
 /* allocate an object out of the local nursery */
-gcptr stm_allocate_object_of_size(size_t size);
 gcptr stm_allocate(size_t size, unsigned long tid);
 
 /* to push/pop objects into the local shadowstack */
@@ -65,10 +64,6 @@
 /* callback: trace the content of an object */
 extern void stmcb_trace(gcptr, void visit(gcptr *));
 
-/* debugging: allocate but immediately old, not via the nursery */
-gcptr _stm_allocate_object_of_size_old(size_t size);
-gcptr _stm_allocate_old(size_t size, unsigned long tid);
-
 /* You can put one GC-tracked thread-local object here.
    (Obviously it can be a container type containing more GC objects.)
    It is set to NULL by stm_initialize(). */
diff --git a/c4/stmimpl.h b/c4/stmimpl.h
--- a/c4/stmimpl.h
+++ b/c4/stmimpl.h
@@ -30,6 +30,7 @@
 #include "fprintcolor.h"
 #include "lists.h"
 #include "dbgmem.h"
+#include "nursery.h"
 #include "et.h"
 #include "steal.h"
 #include "stmsync.h"
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -59,7 +59,7 @@
 {
     int r = DescriptorInit();
     assert(r == 1);
-    //stmgc_init_tls();
+    stm_init_nursery();
     init_shadowstack();
     //stmgcpage_init_tls();
     BeginInevitableTransaction();
@@ -93,20 +93,6 @@
     return obj;
 }
 
-gcptr stm_allocate(size_t size, unsigned long tid)
-{
-    gcptr result = stm_malloc(size);
-    assert(tid == (tid & STM_USER_TID_MASK));
-    result->h_tid = tid;
-    result->h_revision = stm_private_rev_num;
-    return result;
-}
-
-gcptr _stm_allocate_old(size_t size, unsigned long tid)
-{
-    abort();
-}
-
 /************************************************************/
 
 static revision_t sync_required = 0;
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -5,11 +5,11 @@
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 header_files = [os.path.join(parent_dir, _n) for _n in
-                "et.h lists.h steal.h "
+                "et.h lists.h steal.h nursery.h "
                 "stmsync.h dbgmem.h fprintcolor.h "
                 "stmgc.h stmimpl.h atomic_ops.h".split()]
 source_files = [os.path.join(parent_dir, _n) for _n in
-                "et.c lists.c steal.c "
+                "et.c lists.c steal.c nursery.c "
                 "stmsync.c dbgmem.c fprintcolor.c".split()]
 
 _pycache_ = os.path.join(parent_dir, 'test', '__pycache__')
@@ -40,8 +40,7 @@
     #define PREBUILT_FLAGS         ...
     #define PREBUILT_REVISION      ...
 
-    //gcptr stm_allocate_object_of_size(size_t size);
-    gcptr stm_allocate(size_t size, unsigned long tid);
+    gcptr stm_allocate(size_t size, unsigned int tid);
     void stm_push_root(gcptr);
     gcptr stm_pop_root(void);
     void stm_set_max_aborts(int max_aborts);
@@ -65,7 +64,6 @@
     //void stmgc_minor_collect(void);
     gcptr _stm_nonrecord_barrier(gcptr);
     int _stm_is_private(gcptr);
-    int stm_dbgmem_is_active(void *p, int allow_outside);
     void stm_start_sharedlock(void);
     void stm_stop_sharedlock(void);
     void AbortTransaction(int);
@@ -125,7 +123,6 @@
 
     int gettid(gcptr obj)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         int result = stm_get_tid(obj);
         assert(0 <= result && result < 521);
         return result;
@@ -133,13 +130,11 @@
 
     void settid(gcptr obj, int newtid)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         stm_set_tid(obj, newtid);
     }
 
     gcptr rawgetptr(gcptr obj, long index)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         assert(gettid(obj) > 421 + index);
         return ((gcptr *)(obj + 1))[index];
     }
@@ -147,21 +142,18 @@
     void rawsetptr(gcptr obj, long index, gcptr newvalue)
     {
         fprintf(stderr, "%p->[%ld] = %p\n", obj, index, newvalue);
-        assert(stm_dbgmem_is_active(obj, 1));
         assert(gettid(obj) > 421 + index);
         ((gcptr *)(obj + 1))[index] = newvalue;
     }
 
     gcptr getptr(gcptr obj, long index)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         obj = stm_read_barrier(obj);
         return rawgetptr(obj, index);
     }
 
     void setptr(gcptr obj, long index, gcptr newvalue)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         obj = stm_write_barrier(obj);
         fprintf(stderr, "setptr: write_barrier: %p, writing [%ld] = %p\n",
                 obj, index, newvalue);
@@ -170,28 +162,24 @@
 
     long rawgetlong(gcptr obj, long index)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         assert(stmcb_size(obj) >= sizeof(gcptr *) + (index+1)*sizeof(void *));
         return (long)((void **)(obj + 1))[index];
     }
 
     void rawsetlong(gcptr obj, long index, long newvalue)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         assert(stmcb_size(obj) >= sizeof(gcptr *) + (index+1)*sizeof(void *));
         ((void **)(obj + 1))[index] = (void *)newvalue;
     }
 
     long getlong(gcptr obj, long index)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         obj = stm_read_barrier(obj);
         return rawgetlong(obj, index);
     }
 
     void setlong(gcptr obj, long index, long newvalue)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         obj = stm_write_barrier(obj);
         rawsetlong(obj, index, newvalue);
     }
@@ -214,18 +202,6 @@
         return (void *)thread_descriptor->public_descriptor;
     }
 
-    /*gcptr *addr_of_thread_local(void)
-    {
-        return &stm_thread_local_obj;
-    }*/
-
-    /*int in_nursery(gcptr obj)
-    {
-        assert(stm_dbgmem_is_active(obj, 1));
-        struct tx_descriptor *d = thread_descriptor;
-        return (d->nursery <= (char*)obj && ((char*)obj) < d->nursery_end);
-    }*/
-
     void stm_initialize_tests(int max_aborts)
     {
         stm_initialize();
@@ -234,7 +210,6 @@
 
     size_t stmcb_size(gcptr obj)
     {
-        assert(stm_dbgmem_is_active(obj, 1));
         if (gettid(obj) < 421) {
             /* basic case: tid equals 42 plus the size of the object */
             assert(gettid(obj) >= 42 + sizeof(struct stm_object_s));
@@ -250,7 +225,6 @@
     void stmcb_trace(gcptr obj, void visit(gcptr *))
     {
         int i;
-        assert(stm_dbgmem_is_active(obj, 1));
         if (gettid(obj) < 421) {
             /* basic case: no references */
             return;
@@ -467,24 +441,15 @@
     lib.stmgc_minor_collect()
 
 def is_stub(p):
-    assert lib.stm_dbgmem_is_active(p, 1) != 0
     return p.h_tid & GCFLAG_STUB
 
 def check_not_free(p):
-    assert lib.stm_dbgmem_is_active(p, 1) == 1
     assert 42 < (p.h_tid & 0xFFFF) < 521
 
 def check_prebuilt(p):
-    assert lib.stm_dbgmem_is_active(p, 1) == -1
     assert 42 < (p.h_tid & 0xFFFF) < 521
     assert p.h_tid & GCFLAG_PREBUILT_ORIGINAL
 
-def check_free(p):
-    assert not lib.stm_dbgmem_is_active(p, 0)
-
-def check_nursery_free(p):
-    assert not lib.stm_dbgmem_is_active(p, 0) or p.h_tid == 0
-
 def make_global(p1):
     assert p1.h_revision == lib.get_local_revision()
     p1.h_revision = (lib.stm_global_cur_time() | 1) - 2

From noreply at buildbot.pypy.org  Thu Jun 13 22:49:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 22:49:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Start copying from c3 portions of code
	and tests.
Message-ID: <20130613204925.68B111C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r119:e26c4be262b8
Date: 2013-06-13 22:49 +0200
http://bitbucket.org/pypy/stmgc/changeset/e26c4be262b8/

Log:	Start copying from c3 portions of code and tests.

diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -105,7 +105,6 @@
  * thread shuts down.  It is reused the next time a thread starts. */
 struct tx_public_descriptor {
   revision_t collection_lock;
-  NURSERY_FIELDS_DECL
   struct tx_descriptor *descriptor;
   struct stub_block_s *stub_blocks;
   gcptr stub_free_list;
@@ -125,6 +124,8 @@
   gcptr *shadowstack;
   gcptr **shadowstack_end_ref;
 
+  NURSERY_FIELDS_DECL
+
   long atomic;   /* 0 = not atomic, > 0 atomic */
   unsigned long count_reads;
   unsigned long reads_size_limit;        /* see should_break_tr. */
diff --git a/c4/nursery.c b/c4/nursery.c
new file mode 100644
--- /dev/null
+++ b/c4/nursery.c
@@ -0,0 +1,161 @@
+#include "stmimpl.h"
+
+
+static int is_in_nursery(struct tx_descriptor *d, gcptr obj)
+{
+    return (d->nursery_base <= (char*)obj && ((char*)obj) < d->nursery_end);
+}
+
+/************************************************************/
+
+void stm_init_nursery(void)
+{
+    struct tx_descriptor *d = thread_descriptor;
+
+    assert(d->nursery_base == NULL);
+    d->nursery_base = stm_malloc(GC_NURSERY);
+    memset(d->nursery_base, 0, GC_NURSERY);
+    d->nursery_end = d->nursery_base + GC_NURSERY;
+    d->nursery_current = d->nursery_base;
+}
+
+static char *collect_and_allocate_size(size_t size);  /* forward */
+
+gcptr stm_allocate(size_t size, unsigned long tid)
+{
+    /* XXX inline the fast path */
+    struct tx_descriptor *d = thread_descriptor;
+    char *cur = d->nursery_current;
+    char *end = cur + size;
+    d->nursery_current = end;
+    if (end > d->nursery_end) {
+        cur = collect_and_allocate_size(size);
+    }
+    gcptr P = (gcptr)cur;
+    assert(tid == (tid & STM_USER_TID_MASK));
+    P->h_tid = tid;
+    P->h_revision = stm_private_rev_num;
+    return P;
+}
+
+/************************************************************/
+
+static void visit_if_young(gcptr *root)
+{
+    gcptr obj = *root;
+    gcptr fresh_old_copy;
+    struct tx_descriptor *d = thread_descriptor;
+
+    if (!is_in_nursery(d, obj)) {
+        /* not a nursery object */
+    }
+    else {
+        /* a nursery object */
+        fresh_old_copy = stmgc_duplicate(obj);
+        fresh_old_copy->h_tid |= GCFLAG_OLD;
+        obj->h_tid |= GCFLAG_NURSERY_MOVED;
+        obj->h_revision = (revision_t)fresh_old_copy;
+        *root = fresh_old_copy;
+    }
+}
+
+static void mark_young_roots(gcptr *root, gcptr *end)
+{
+    /* XXX use a way to avoid walking all roots again and again */
+    for (; root != end; root++) {
+        visit_if_young(root);
+    }
+}
+
+static void setup_minor_collect(struct tx_descriptor *d)
+{
+    spinlock_acquire(d->public_descriptor->collection_lock, 'M');  /*minor*/
+    assert(gcptrlist_size(&d->old_objects_to_trace) == 0);
+
+    if (d->public_descriptor->stolen_objects.size != 0)
+        stm_normalize_stolen_objects(d);
+}
+
+static void teardown_minor_collect(struct tx_descriptor *d)
+{
+    assert(gcptrlist_size(&d->old_objects_to_trace) == 0);
+    assert(gcptrlist_size(&d->public_descriptor->stolen_objects) == 0);
+
+    spinlock_release(d->public_descriptor->collection_lock);
+}
+
+static void minor_collect(struct tx_descriptor *d)
+{
+    fprintf(stderr, "minor collection [%p to %p]\n",
+            d->nursery_base, d->nursery_end);
+
+    /* acquire the "collection lock" first */
+    setup_minor_collect(d);
+
+    mark_young_roots(d->shadowstack, *d->shadowstack_end_ref);
+
+#if 0
+    mark_private_from_protected(d);
+
+    mark_public_to_young(d);
+
+    mark_private_old_pointing_to_young(d);
+
+    visit_all_outside_objects(d);
+    fix_list_of_read_objects(d);
+
+    /* now all surviving nursery objects have been moved out, and all
+       surviving young-but-outside-the-nursery objects have been flagged
+       with GCFLAG_OLD */
+    finish_public_to_young(d);
+
+    if (g2l_any_entry(&d->young_objects_outside_nursery))
+        free_unvisited_young_objects_outside_nursery(d);
+#endif
+
+    teardown_minor_collect(d);
+
+    /* clear the nursery */
+    memset(d->nursery_base, 0, GC_NURSERY);
+    d->nursery_current = d->nursery_base;
+
+    assert(!stmgc_minor_collect_anything_to_do(d));
+}
+
+void stmgc_minor_collect(void)
+{
+    struct tx_descriptor *d = thread_descriptor;
+    assert(d->active >= 1);
+    minor_collect(d);
+    AbortNowIfDelayed();
+}
+
+int stmgc_minor_collect_anything_to_do(struct tx_descriptor *d)
+{
+    if (d->nursery_current == d->nursery_base /*&&
+        !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
+        /* there is no young object */
+        //assert(gcptrlist_size(&d->private_old_pointing_to_young) == 0);
+        //assert(gcptrlist_size(&d->public_to_young) == 0);
+        return 0;
+    }
+    else {
+        /* there are young objects */
+        return 1;
+    }
+}
+
+static char *collect_and_allocate_size(size_t allocate_size)
+{
+    stmgc_minor_collect();
+    //stmgcpage_possibly_major_collect(0);
+
+    struct tx_descriptor *d = thread_descriptor;
+    assert(d->nursery_current == d->nursery_base);
+
+    //_debug_roots(d->shadowstack, *d->shadowstack_end_ref);
+
+    d->nursery_current = d->nursery_base + allocate_size;
+    assert(d->nursery_current <= d->nursery_end);  /* XXX object too big */
+    return d->nursery_base;
+}
diff --git a/c4/nursery.h b/c4/nursery.h
new file mode 100644
--- /dev/null
+++ b/c4/nursery.h
@@ -0,0 +1,21 @@
+#ifndef _SRCSTM_NURSERY_H
+#define _SRCSTM_NURSERY_H
+
+#ifndef GC_NURSERY
+#define GC_NURSERY        4194304    /* 4 MB */
+#endif
+
+
+#define NURSERY_FIELDS_DECL                             \
+    char *nursery_current;                              \
+    char *nursery_end;                                  \
+    char *nursery_base;                                 \
+    struct GcPtrList old_objects_to_trace;
+
+struct tx_descriptor;  /* from et.h */
+
+void stm_init_nursery(void);
+void stmgc_minor_collect(void);
+int stmgc_minor_collect_anything_to_do(struct tx_descriptor *);
+
+#endif
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
new file mode 100644
--- /dev/null
+++ b/c4/test/test_nursery.py
@@ -0,0 +1,33 @@
+import py
+from support import *
+
+
+def setup_function(f):
+    lib.stm_clear_between_tests()
+    lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
+
+def teardown_function(_):
+    lib.stm_finalize()
+
+
+def test_nursery_alloc():
+    for i in range(20):
+        p = nalloc(HDR)
+        check_not_free(p)
+
+def test_stm_roots():
+    p1 = nalloc(HDR)
+    p2 = nalloc(HDR)
+    p3 = nalloc(HDR)
+    seen = set()
+    for i in range(20):
+        lib.stm_push_root(p1)
+        lib.stm_push_root(p3)
+        p = nalloc(HDR)
+        check_not_free(p)
+        seen.add(p)
+        p3 = lib.stm_pop_root()
+        p1 = lib.stm_pop_root()
+        check_not_free(p1)
+        check_not_free(p3)
+    assert p2 in seen    # the pointer location was reused

From noreply at buildbot.pypy.org  Thu Jun 13 22:54:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 22:54:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: stmgc_done_nursery()
Message-ID: <20130613205401.A228A1C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r120:e1d5588970ba
Date: 2013-06-13 22:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/e1d5588970ba/

Log:	stmgc_done_nursery()

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -8,7 +8,7 @@
 
 /************************************************************/
 
-void stm_init_nursery(void)
+void stmgc_init_nursery(void)
 {
     struct tx_descriptor *d = thread_descriptor;
 
@@ -19,6 +19,15 @@
     d->nursery_current = d->nursery_base;
 }
 
+void stmgc_done_nursery(void)
+{
+    struct tx_descriptor *d = thread_descriptor;
+    assert(!stmgc_minor_collect_anything_to_do(d));
+    stm_free(d->nursery_base, GC_NURSERY);
+
+    gcptrlist_delete(&d->old_objects_to_trace);
+}
+
 static char *collect_and_allocate_size(size_t size);  /* forward */
 
 gcptr stm_allocate(size_t size, unsigned long tid)
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -14,7 +14,8 @@
 
 struct tx_descriptor;  /* from et.h */
 
-void stm_init_nursery(void);
+void stmgc_init_nursery(void);
+void stmgc_done_nursery(void);
 void stmgc_minor_collect(void);
 int stmgc_minor_collect_anything_to_do(struct tx_descriptor *);
 
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -59,7 +59,7 @@
 {
     int r = DescriptorInit();
     assert(r == 1);
-    stm_init_nursery();
+    stmgc_init_nursery();
     init_shadowstack();
     //stmgcpage_init_tls();
     BeginInevitableTransaction();
@@ -67,11 +67,11 @@
 
 void stm_finalize(void)
 {
-    //stmgc_minor_collect();   /* force everything out of the nursery */
+    stmgc_minor_collect();   /* force everything out of the nursery */
     CommitTransaction();
     //stmgcpage_done_tls();
     done_shadowstack();
-    //stmgc_done_tls();
+    stmgc_done_nursery();
     DescriptorDone();
 }
 

From noreply at buildbot.pypy.org  Thu Jun 13 23:13:13 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 13 Jun 2013 23:13:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add aliases for the float64 and string
	dtypes
Message-ID: <20130613211313.1A28A1C00B9@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r64877:c0ad61f37cbe
Date: 2013-06-13 23:11 +0200
http://bitbucket.org/pypy/pypy/changeset/c0ad61f37cbe/

Log:	Add aliases for the float64 and string dtypes

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -582,7 +582,7 @@
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
                                    ],
-            aliases=["float"],
+            aliases=["float", "double"],
         )
         self.w_complex64dtype = W_ComplexDtype(
             types.Complex64(),
@@ -663,6 +663,7 @@
             char='S',
             w_box_type = space.gettypefor(interp_boxes.W_StringBox),
             alternate_constructors=[space.w_str],
+            aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
             types.UnicodeType(1),

From noreply at buildbot.pypy.org  Thu Jun 13 23:20:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 23:20:47 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Copy more code, the next test passes
Message-ID: <20130613212047.F04EF1C00B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r121:17c239320c14
Date: 2013-06-13 23:00 +0200
http://bitbucket.org/pypy/stmgc/changeset/17c239320c14/

Log:	Copy more code, the next test passes

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -60,11 +60,20 @@
     }
     else {
         /* a nursery object */
+        assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
+        assert(!(obj->h_tid & GCFLAG_OLD));
+        assert(!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
+
+        /* make a copy of it outside */
         fresh_old_copy = stmgc_duplicate(obj);
-        fresh_old_copy->h_tid |= GCFLAG_OLD;
         obj->h_tid |= GCFLAG_NURSERY_MOVED;
         obj->h_revision = (revision_t)fresh_old_copy;
+
+        /* fix the original reference */
         *root = fresh_old_copy;
+
+        /* add 'fresh_old_copy' to the list of objects to trace */
+        gcptrlist_insert(&d->old_objects_to_trace, fresh_old_copy);
     }
 }
 
@@ -76,6 +85,19 @@
     }
 }
 
+static void visit_all_outside_objects(struct tx_descriptor *d)
+{
+    while (gcptrlist_size(&d->old_objects_to_trace) > 0) {
+        gcptr obj = gcptrlist_pop(&d->old_objects_to_trace);
+
+        assert(!(obj->h_tid & GCFLAG_OLD));
+        assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
+        obj->h_tid |= GCFLAG_OLD | GCFLAG_WRITE_BARRIER;
+
+        stmcb_trace(obj, &visit_if_young);
+    }
+}
+
 static void setup_minor_collect(struct tx_descriptor *d)
 {
     spinlock_acquire(d->public_descriptor->collection_lock, 'M');  /*minor*/
@@ -109,8 +131,10 @@
     mark_public_to_young(d);
 
     mark_private_old_pointing_to_young(d);
+#endif
 
     visit_all_outside_objects(d);
+#if 0
     fix_list_of_read_objects(d);
 
     /* now all surviving nursery objects have been moved out, and all
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -61,7 +61,7 @@
     revision_t stm_global_cur_time(void);
     //void stmgcpage_add_prebuilt_root(gcptr);
     void stm_clear_between_tests(void);
-    //void stmgc_minor_collect(void);
+    void stmgc_minor_collect(void);
     gcptr _stm_nonrecord_barrier(gcptr);
     int _stm_is_private(gcptr);
     void stm_start_sharedlock(void);
@@ -446,6 +446,9 @@
 def check_not_free(p):
     assert 42 < (p.h_tid & 0xFFFF) < 521
 
+def check_nursery_free(p):
+    assert p.h_tid == p.h_revision == 0
+
 def check_prebuilt(p):
     assert 42 < (p.h_tid & 0xFFFF) < 521
     assert p.h_tid & GCFLAG_PREBUILT_ORIGINAL
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -31,3 +31,15 @@
         check_not_free(p1)
         check_not_free(p3)
     assert p2 in seen    # the pointer location was reused
+
+def test_nursery_follows():
+    p1 = nalloc_refs(1)
+    p2 = nalloc_refs(1)
+    rawsetptr(p1, 0, p2)
+    lib.stm_push_root(p1)
+    minor_collect()
+    check_nursery_free(p1)
+    check_nursery_free(p2)
+    p1b = lib.stm_pop_root()
+    p2b = rawgetptr(p1b, 0)
+    assert rawgetptr(p2b, 0) == ffi.NULL

From noreply at buildbot.pypy.org  Thu Jun 13 23:20:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Jun 2013 23:20:49 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130613212049.218C41C00B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r122:03a8ca843df4
Date: 2013-06-13 23:20 +0200
http://bitbucket.org/pypy/stmgc/changeset/03a8ca843df4/

Log:	Next test

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -9,7 +9,8 @@
 #define MMAP_TOTAL  671088640   /* 640MB */
 
 static pthread_mutex_t malloc_mutex = PTHREAD_MUTEX_INITIALIZER;
-static char *zone_current = NULL, *zone_end = NULL;
+static char *zone_start, *zone_current = NULL, *zone_end = NULL;
+static signed char accessible_pages[MMAP_TOTAL / PAGE_SIZE] = {0};
 
 
 static void _stm_dbgmem(void *p, size_t sz, int prot)
@@ -30,15 +31,16 @@
     pthread_mutex_lock(&malloc_mutex);
 
     if (zone_current == NULL) {
-        zone_current = mmap(NULL, MMAP_TOTAL, PROT_NONE,
-                            MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-        if (zone_current == NULL || zone_current == MAP_FAILED) {
+        zone_start = mmap(NULL, MMAP_TOTAL, PROT_NONE,
+                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (zone_start == NULL || zone_start == MAP_FAILED) {
             fprintf(stderr, "not enough memory: mmap() failed\n");
             abort();
         }
-        zone_end = zone_current + MMAP_TOTAL;
+        zone_current = zone_start;
+        zone_end = zone_start + MMAP_TOTAL;
         assert((MMAP_TOTAL % PAGE_SIZE) == 0);
-        _stm_dbgmem(zone_current, MMAP_TOTAL, PROT_NONE);
+        _stm_dbgmem(zone_start, MMAP_TOTAL, PROT_NONE);
     }
 
     size_t nb_pages = (sz + PAGE_SIZE - 1) / PAGE_SIZE + 1;
@@ -54,14 +56,33 @@
     result += (-sz) & (PAGE_SIZE-1);
     assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
     _stm_dbgmem(result, sz, PROT_READ | PROT_WRITE);
+
+    long i, base = (result - zone_start) / PAGE_SIZE;
+    for (i = 0; i < nb_pages; i++)
+        accessible_pages[base + i] = 42;
+
     return result;
 }
 
 void stm_free(void *p, size_t sz)
 {
+    size_t nb_pages = (sz + PAGE_SIZE - 1) / PAGE_SIZE + 1;
+    long i, base = ((char *)p - zone_start) / PAGE_SIZE;
+    assert(0 <= base && base < (MMAP_TOTAL / PAGE_SIZE));
+    for (i = 0; i < nb_pages; i++) {
+        assert(accessible_pages[base + i] == 42);
+        accessible_pages[base + i] = -1;
+    }
     memset(p, 0xDD, sz);
     _stm_dbgmem(p, sz, PROT_NONE);
 }
 
+int _stm_can_access_memory(char *p)
+{
+    long base = ((char *)p - zone_start) / PAGE_SIZE;
+    assert(0 <= base && base < (MMAP_TOTAL / PAGE_SIZE));
+    return accessible_pages[base] == 42;
+}
+
 /************************************************************/
 #endif
diff --git a/c4/dbgmem.h b/c4/dbgmem.h
--- a/c4/dbgmem.h
+++ b/c4/dbgmem.h
@@ -6,6 +6,7 @@
 
 void *stm_malloc(size_t);
 void stm_free(void *, size_t);
+int _stm_can_access_memory(char *);
 
 #else
 
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -87,8 +87,7 @@
     revision_t get_start_time(void);
     void *my_stub_thread(void);
 
-    //gcptr *addr_of_thread_local(void);
-    //int in_nursery(gcptr);
+    int _stm_can_access_memory(char *);
     void stm_initialize_tests(int max_aborts);
 
     /* some constants normally private that are useful in the tests */
@@ -449,6 +448,9 @@
 def check_nursery_free(p):
     assert p.h_tid == p.h_revision == 0
 
+def check_inaccessible(p):
+    assert not lib._stm_can_access_memory(p)
+
 def check_prebuilt(p):
     assert 42 < (p.h_tid & 0xFFFF) < 521
     assert p.h_tid & GCFLAG_PREBUILT_ORIGINAL
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -43,3 +43,9 @@
     p1b = lib.stm_pop_root()
     p2b = rawgetptr(p1b, 0)
     assert rawgetptr(p2b, 0) == ffi.NULL
+
+def test_free_nursery_at_thread_end():
+    p1 = nalloc(HDR)
+    lib.stm_finalize()
+    check_inaccessible(p1)
+    lib.stm_initialize_tests(0)

From noreply at buildbot.pypy.org  Thu Jun 13 23:48:17 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Jun 2013 23:48:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Implement dtype.str. Try hard to do the
	same as cNumpy.
Message-ID: <20130613214817.9C3AB1C022D@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r64878:96ef4e26e420
Date: 2013-06-13 23:47 +0200
http://bitbucket.org/pypy/pypy/changeset/96ef4e26e420/

Log:	Implement dtype.str. Try hard to do the same as cNumpy.

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -115,6 +115,21 @@
             return space.wrap('=')
         return space.wrap(nonnative_byteorder_prefix)
 
+    def descr_get_str(self, space):
+        size = self.get_size()
+        basic = self.kind
+        if basic == UNICODELTR:
+            size >>= 2
+            endian = byteorder_prefix
+        elif size <= 1:
+            endian = '|'  # ignore
+        elif self.native:
+            endian = byteorder_prefix
+        else:
+            endian = nonnative_byteorder_prefix
+
+        return space.wrap("%s%s%s" % (endian, basic, size))
+
     def descr_get_alignment(self, space):
         return space.wrap(self.itemtype.alignment)
 
@@ -421,6 +436,7 @@
     char = interp_attrproperty("char", cls=W_Dtype),
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
     byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
+    str = GetSetProperty(W_Dtype.descr_get_str),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
     alignment = GetSetProperty(W_Dtype.descr_get_alignment),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
@@ -666,7 +682,7 @@
             aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
-            types.UnicodeType(1),
+            types.UnicodeType(0),
             num=19,
             kind=UNICODELTR,
             name='unicode',
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -683,6 +683,20 @@
         assert dtype('=i8').byteorder == '='
         assert dtype(byteorder + 'i8').byteorder == '='
 
+    def test_dtype_str(self):
+        from numpypy import dtype
+        byteorder = self.native_prefix
+        assert dtype('i8').str == byteorder + 'i8'
+        assert dtype('<i8').str == '<i8'
+        assert dtype('>i8').str == '>i8'
+        assert dtype('int8').str == '|i1'
+        assert dtype('float').str == byteorder + 'f8'
+        # strange
+        assert dtype('string').str == '|S0'
+        assert dtype('unicode').str == byteorder + 'U0'
+        # assert dtype(('string', 7)).str == '|S7'
+        # assert dtype(('unicode', 7)).str == '<U7'
+
     def test_intp(self):
         from numpypy import dtype
         assert dtype('p') == dtype('intp')

From noreply at buildbot.pypy.org  Fri Jun 14 00:37:53 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Jun 2013 00:37:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fill more values in
	array.__array_interface__
Message-ID: <20130613223753.1CA001C02BA@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r64879:2ebf68f7c7be
Date: 2013-06-14 00:12 +0200
http://bitbucket.org/pypy/pypy/changeset/2ebf68f7c7be/

Log:	Fill more values in array.__array_interface__

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -418,8 +418,16 @@
         addr = self.implementation.get_storage_as_int(space)
         # will explode if it can't
         w_d = space.newdict()
-        space.setitem_str(w_d, 'data', space.newtuple([space.wrap(addr),
-                                                       space.w_False]))
+        space.setitem_str(w_d, 'data',
+                          space.newtuple([space.wrap(addr), space.w_False]))
+        space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
+        space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
+        if self.implementation.order == 'C':
+            # Array is contiguous, no strides in the interface.
+            strides = space.w_None
+        else:
+            strides = self.descr_get_strides(space)
+        space.setitem_str(w_d, 'strides', strides)
         return w_d
 
     w_pypy_data = None
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2138,6 +2138,9 @@
         a = array([1, 2, 3])
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
+        assert i['shape'] == (3,)
+        assert i['strides'] == None  # Because array is in C order
+        assert i['typestr'] == a.dtype.str
         a = a[::2]
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)

From noreply at buildbot.pypy.org  Fri Jun 14 14:58:25 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Jun 2013 14:58:25 +0200 (CEST)
Subject: [pypy-commit] pypy argsort-segfault: fix malloc, cleanup
Message-ID: <20130614125825.6FB421C13B0@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: argsort-segfault
Changeset: r64880:938d68eb8a73
Date: 2013-06-14 15:57 +0300
http://bitbucket.org/pypy/pypy/changeset/938d68eb8a73/

Log:	fix malloc, cleanup

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -33,11 +33,8 @@
             self.indexes = indexes
 
         def __del__(self):
-            print 'Repr.del',self.values
 
         def getitem(self, item):
-            #print 'getting',item,'of',self.size,self.values
-            #print 'from',item*self.stride_size + self.start,'to',item*(self.stride_size+1) + self.start
             if count < 2:
                 v = raw_storage_getitem(TP, self.values, item * self.stride_size
                                     + self.start)
@@ -79,7 +76,7 @@
             indexes = dtype.itemtype.malloc(size*dtype.get_size())
             values = alloc_raw_storage(size * stride_size,
                                             track_allocation=False)
-            Repr.__init__(self, index_stride_size, stride_size,
+            Repr.__init__(self, dtype.get_size(), stride_size,
                           size, values, indexes, start, start)
 
         def __del__(self):
@@ -96,12 +93,10 @@
         return lst.size
 
     def arg_getitem_slice(lst, start, stop):
-        print 'arg_getitem_slice',lst.values
         retval = ArgArrayRepWithStorage(lst.index_stride_size, lst.stride_size,
                 stop-start)
         for i in range(stop-start):
             retval.setitem(i, lst.getitem(i+start))
-        print 'arg_getitem_slice done',lst
         return retval
 
     if count < 2:
@@ -155,16 +150,13 @@
             stride_size = arr.strides[axis]
             index_stride_size = index_impl.strides[axis]
             axis_size = arr.shape[axis]
-            print '5'
             while not iter.done():
                 for i in range(axis_size):
                     raw_storage_setitem(storage, i * index_stride_size +
                                         index_iter.offset, i)
                 r = Repr(index_stride_size, stride_size, axis_size,
                          arr.get_storage(), storage, index_iter.offset, iter.offset)
-                print '6'
                 ArgSort(r).sort()
-                print '7'
                 iter.next()
                 index_iter.next()
         return index_arr

From noreply at buildbot.pypy.org  Fri Jun 14 15:08:45 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Jun 2013 15:08:45 +0200 (CEST)
Subject: [pypy-commit] pypy argsort-segfault: whoops
Message-ID: <20130614130845.B193A1C13B0@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: argsort-segfault
Changeset: r64881:65ca8e4122fb
Date: 2013-06-14 16:07 +0300
http://bitbucket.org/pypy/pypy/changeset/65ca8e4122fb/

Log:	whoops

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -32,8 +32,6 @@
             self.values = values
             self.indexes = indexes
 
-        def __del__(self):
-
         def getitem(self, item):
             if count < 2:
                 v = raw_storage_getitem(TP, self.values, item * self.stride_size

From noreply at buildbot.pypy.org  Fri Jun 14 15:18:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 15:18:10 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130614131810.3E1B61C3354@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r123:171f8b47b009
Date: 2013-06-14 15:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/171f8b47b009/

Log:	Next test

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -391,14 +391,6 @@
 #endif
 }
 
-gcptr stmgc_duplicate(gcptr P)
-{
-  size_t size = stmcb_size(P);
-  gcptr L = stm_malloc(size);
-  memcpy(L, P, size);
-  return L;
-}
-
 static gcptr LocalizeProtected(struct tx_descriptor *d, gcptr P)
 {
   gcptr B;
@@ -447,6 +439,7 @@
                 0);
   L->h_revision = stm_private_rev_num;
   g2l_insert(&d->public_to_private, R, L);
+  gcptrlist_insert(&d->public_to_young, R);
   fprintf(stderr, "write_barrier: adding %p -> %p to public_to_private\n",
           R, L);
 
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -172,7 +172,6 @@
 gcptr stm_get_private_from_protected(long);  /* debugging */
 gcptr stm_get_read_obj(long);  /* debugging */
 void stm_clear_read_cache(void);  /* debugging */
-gcptr stmgc_duplicate(gcptr);
 
 int DescriptorInit(void);
 void DescriptorDone(void);
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -26,13 +26,13 @@
     stm_free(d->nursery_base, GC_NURSERY);
 
     gcptrlist_delete(&d->old_objects_to_trace);
+    gcptrlist_delete(&d->public_to_young);
 }
 
 static char *collect_and_allocate_size(size_t size);  /* forward */
 
-gcptr stm_allocate(size_t size, unsigned long tid)
+inline static char *allocate_nursery(size_t size)
 {
-    /* XXX inline the fast path */
     struct tx_descriptor *d = thread_descriptor;
     char *cur = d->nursery_current;
     char *end = cur + size;
@@ -40,15 +40,47 @@
     if (end > d->nursery_end) {
         cur = collect_and_allocate_size(size);
     }
-    gcptr P = (gcptr)cur;
+    return cur;
+}
+
+gcptr stm_allocate(size_t size, unsigned long tid)
+{
+    /* XXX inline the fast path */
+    gcptr P = (gcptr)allocate_nursery(size);
     assert(tid == (tid & STM_USER_TID_MASK));
     P->h_tid = tid;
     P->h_revision = stm_private_rev_num;
     return P;
 }
 
+gcptr stmgc_duplicate(gcptr P)
+{
+    size_t size = stmcb_size(P);
+    gcptr L = (gcptr)allocate_nursery(size);
+    memcpy(L, P, size);
+    L->h_tid &= ~GCFLAG_OLD;
+    return L;
+}
+
 /************************************************************/
 
+static inline gcptr create_old_object_copy(gcptr obj)
+{
+    assert(!(obj->h_tid & GCFLAG_NURSERY_MOVED));
+    assert(!(obj->h_tid & GCFLAG_VISITED));
+    assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
+    assert(!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
+    assert(!(obj->h_tid & GCFLAG_OLD));
+
+    size_t size = stmcb_size(obj);
+    gcptr fresh_old_copy = stm_malloc(size);
+    memcpy(fresh_old_copy, obj, size);
+    fresh_old_copy->h_tid |= GCFLAG_OLD;
+
+    fprintf(stderr, "minor: %p is copied to %p\n", obj, fresh_old_copy);
+    return fresh_old_copy;
+}
+
 static void visit_if_young(gcptr *root)
 {
     gcptr obj = *root;
@@ -60,12 +92,8 @@
     }
     else {
         /* a nursery object */
-        assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
-        assert(!(obj->h_tid & GCFLAG_OLD));
-        assert(!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
-
         /* make a copy of it outside */
-        fresh_old_copy = stmgc_duplicate(obj);
+        fresh_old_copy = create_old_object_copy(obj);
         obj->h_tid |= GCFLAG_NURSERY_MOVED;
         obj->h_revision = (revision_t)fresh_old_copy;
 
@@ -77,22 +105,40 @@
     }
 }
 
-static void mark_young_roots(gcptr *root, gcptr *end)
+static void mark_young_roots(struct tx_descriptor *d)
 {
+    gcptr *root = d->shadowstack;
+    gcptr *end = *d->shadowstack_end_ref;
+
     /* XXX use a way to avoid walking all roots again and again */
     for (; root != end; root++) {
         visit_if_young(root);
     }
 }
 
+static void mark_public_to_young(struct tx_descriptor *d)
+{
+    long i, size = d->public_to_young.size;
+    gcptr *items = d->public_to_young.items;
+
+    for (i = 0; i < size; i++) {
+        gcptr P = items[i];
+        wlog_t *item;
+
+        G2L_FIND(d->public_to_private, P, item, continue);
+        visit_if_young(&item->val);
+    }
+    gcptrlist_clear(&d->public_to_young);
+}
+
 static void visit_all_outside_objects(struct tx_descriptor *d)
 {
     while (gcptrlist_size(&d->old_objects_to_trace) > 0) {
         gcptr obj = gcptrlist_pop(&d->old_objects_to_trace);
 
-        assert(!(obj->h_tid & GCFLAG_OLD));
+        assert(obj->h_tid & GCFLAG_OLD);
         assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
-        obj->h_tid |= GCFLAG_OLD | GCFLAG_WRITE_BARRIER;
+        obj->h_tid |= GCFLAG_WRITE_BARRIER;
 
         stmcb_trace(obj, &visit_if_young);
     }
@@ -123,13 +169,15 @@
     /* acquire the "collection lock" first */
     setup_minor_collect(d);
 
-    mark_young_roots(d->shadowstack, *d->shadowstack_end_ref);
+    mark_young_roots(d);
 
 #if 0
     mark_private_from_protected(d);
+#endif
 
     mark_public_to_young(d);
 
+#if 0
     mark_private_old_pointing_to_young(d);
 #endif
 
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -10,7 +10,8 @@
     char *nursery_current;                              \
     char *nursery_end;                                  \
     char *nursery_base;                                 \
-    struct GcPtrList old_objects_to_trace;
+    struct GcPtrList old_objects_to_trace;              \
+    struct GcPtrList public_to_young;
 
 struct tx_descriptor;  /* from et.h */
 
@@ -18,5 +19,6 @@
 void stmgc_done_nursery(void);
 void stmgc_minor_collect(void);
 int stmgc_minor_collect_anything_to_do(struct tx_descriptor *);
+gcptr stmgc_duplicate(gcptr);
 
 #endif
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -71,6 +71,7 @@
     gcptr stm_get_read_obj(long index);
     void *STUB_THREAD(gcptr);
     void stm_clear_read_cache(void);
+    int in_nursery(gcptr);
 
     gcptr getptr(gcptr, long);
     void setptr(gcptr, long, gcptr);
@@ -183,6 +184,13 @@
         rawsetlong(obj, index, newvalue);
     }
 
+    int in_nursery(gcptr obj)
+    {
+        struct tx_descriptor *d = thread_descriptor;
+        return (d->nursery_base <= (char*)obj &&
+                ((char*)obj) < d->nursery_end);
+    }
+
     gcptr pseudoprebuilt(size_t size, int tid)
     {
         gcptr x = calloc(1, size);
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -49,3 +49,25 @@
     lib.stm_finalize()
     check_inaccessible(p1)
     lib.stm_initialize_tests(0)
+
+def test_local_copy_out_of_nursery():
+    p1 = palloc(HDR + WORD)
+    lib.rawsetlong(p1, 0, 420063)
+    assert not lib.in_nursery(p1)
+    assert p1.h_revision != lib.get_private_rev_num()
+    #
+    p2 = lib.stm_write_barrier(p1)
+    assert lib.rawgetlong(p2, 0) == 420063
+    lib.rawsetlong(p2, 0, -91467)
+    assert lib.in_nursery(p2)
+    assert p2.h_revision == lib.get_private_rev_num()
+    #
+    lib.stm_push_root(p1)
+    minor_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    assert lib.rawgetlong(p1b, 0) == 420063
+    #
+    p3 = lib.stm_read_barrier(p1)
+    assert not lib.in_nursery(p3) and p3 != p2
+    assert lib.rawgetlong(p3, 0) == -91467

From noreply at buildbot.pypy.org  Fri Jun 14 15:19:22 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Jun 2013 15:19:22 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: Add support for an opcode
 recently introduced in pypy.
Message-ID: <20130614131922.7C6E61C3354@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r214:92c5c74ce34c
Date: 2013-06-14 15:19 +0200
http://bitbucket.org/pypy/benchmarks/changeset/92c5c74ce34c/

Log:	Add support for an opcode recently introduced in pypy. Necessary
	when translating with such a recent interpreter.

diff --git a/lib/pypy/rpython/flowspace/flowcontext.py b/lib/pypy/rpython/flowspace/flowcontext.py
--- a/lib/pypy/rpython/flowspace/flowcontext.py
+++ b/lib/pypy/rpython/flowspace/flowcontext.py
@@ -800,6 +800,9 @@
         self.popvalue()
         return next_instr
 
+    def JUMP_IF_NOT_DEBUG(self, target, next_instr):
+        return next_instr
+
     def GET_ITER(self, oparg, next_instr):
         w_iterable = self.popvalue()
         w_iterator = self.space.iter(w_iterable)

From noreply at buildbot.pypy.org  Fri Jun 14 15:26:33 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Jun 2013 15:26:33 +0200 (CEST)
Subject: [pypy-commit] pypy argsort-segfault: merge default into branch
Message-ID: <20130614132633.095E21C3354@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: argsort-segfault
Changeset: r64882:2312717ee8a5
Date: 2013-06-14 16:21 +0300
http://bitbucket.org/pypy/pypy/changeset/2312717ee8a5/

Log:	merge default into branch

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,43 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+from .tklib import tklib, tkffi
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {			/* The internal representation: */
+	long longValue;		/*   - an long integer value. */
+	double doubleValue;	/*   - a double-precision floating value. */
+	struct {		/*   - internal rep as two pointers. */
+	    void *ptr1;
+	    void *ptr2;
+	} twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -289,16 +289,6 @@
                "with verify() (see pypy/module/_cffi_backend/ctypefunc.py "
                "for details)"))
 
-        if USE_C_LIBFFI_MSVC and is_result_type:
-            # MSVC returns small structures in registers.  Pretend int32 or
-            # int64 return type.  This is needed as a workaround for what
-            # is really a bug of libffi_msvc seen as an independent library
-            # (ctypes has a similar workaround).
-            if ctype.size <= 4:
-                return clibffi.ffi_type_sint32
-            if ctype.size <= 8:
-                return clibffi.ffi_type_sint64
-
         # walk the fields, expanding arrays into repetitions; first,
         # only count how many flattened fields there are
         nflat = 0
@@ -318,6 +308,16 @@
                                "a struct with a zero-length array"))
             nflat += flat
 
+        if USE_C_LIBFFI_MSVC and is_result_type:
+            # MSVC returns small structures in registers.  Pretend int32 or
+            # int64 return type.  This is needed as a workaround for what
+            # is really a bug of libffi_msvc seen as an independent library
+            # (ctypes has a similar workaround).
+            if ctype.size <= 4:
+                return clibffi.ffi_type_sint32
+            if ctype.size <= 8:
+                return clibffi.ffi_type_sint64
+
         # allocate an array of (nflat + 1) ffi_types
         elements = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * (nflat + 1))
         elements = rffi.cast(FFI_TYPE_PP, elements)
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -200,3 +200,49 @@
                 except KeyError:
                     pass
             rmtree(dir_name, True)
+
+    def test_builtin_reimport(self):
+        # from https://bugs.pypy.org/issue1514
+        skip("fix me")
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        __import__('marshal')
+        sys.modules['marshal'] = saved
+
+        assert marshal.loads == 42
+        import marshal
+        assert marshal.loads == 42
+        marshal.loads = old
+
+    def test_builtin_reimport_mess(self):
+        # taken from https://bugs.pypy.org/issue1514, with extra cases
+        # that show a difference with CPython: we can get on CPython
+        # several module objects for the same built-in module :-(
+        skip("several built-in module objects: not supported by pypy")
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        marshal2 = __import__('marshal')
+        assert marshal2 is not marshal
+        assert marshal2.loads is old
+        assert marshal2 is sys.modules['marshal']
+        assert marshal is saved
+        assert marshal.loads == 42
+
+        import marshal
+        assert marshal.loads is old
+
+        sys.modules['marshal'] = saved
+        import marshal
+        assert marshal.loads == 42
+
+        marshal.loads = old
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -565,6 +565,43 @@
         assert sys.path is oldpath
         assert 'setdefaultencoding' in dir(sys)
 
+    def test_reload_builtin_doesnt_clear(self):
+        import sys
+        sys.foobar = "baz"
+        reload(sys)
+        assert sys.foobar == "baz"
+
+    def test_reimport_builtin_simple_case_1(self):
+        import sys, time
+        del time.tzset
+        del sys.modules['time']
+        import time
+        assert hasattr(time, 'tzset')
+
+    def test_reimport_builtin_simple_case_2(self):
+        skip("fix me")
+        import sys, time
+        time.foo = "bar"
+        del sys.modules['time']
+        import time
+        assert not hasattr(time, 'foo')
+
+    def test_reimport_builtin(self):
+        skip("fix me")
+        import sys, time
+        oldpath = sys.path
+        time.tzset = "<test_reimport_builtin removed this>"
+
+        del sys.modules['time']
+        import time as time1
+        assert sys.modules['time'] is time1
+
+        assert time.tzset == "<test_reimport_builtin removed this>"
+
+        reload(time1)   # don't leave a broken time.tzset behind
+        import time
+        assert time.tzset != "<test_reimport_builtin removed this>"
+
     def test_reload_infinite(self):
         import infinite_reload
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -115,6 +115,21 @@
             return space.wrap('=')
         return space.wrap(nonnative_byteorder_prefix)
 
+    def descr_get_str(self, space):
+        size = self.get_size()
+        basic = self.kind
+        if basic == UNICODELTR:
+            size >>= 2
+            endian = byteorder_prefix
+        elif size <= 1:
+            endian = '|'  # ignore
+        elif self.native:
+            endian = byteorder_prefix
+        else:
+            endian = nonnative_byteorder_prefix
+
+        return space.wrap("%s%s%s" % (endian, basic, size))
+
     def descr_get_alignment(self, space):
         return space.wrap(self.itemtype.alignment)
 
@@ -421,6 +436,7 @@
     char = interp_attrproperty("char", cls=W_Dtype),
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
     byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
+    str = GetSetProperty(W_Dtype.descr_get_str),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
     alignment = GetSetProperty(W_Dtype.descr_get_alignment),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
@@ -582,7 +598,7 @@
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
                                    ],
-            aliases=["float"],
+            aliases=["float", "double"],
         )
         self.w_complex64dtype = W_ComplexDtype(
             types.Complex64(),
@@ -663,9 +679,10 @@
             char='S',
             w_box_type = space.gettypefor(interp_boxes.W_StringBox),
             alternate_constructors=[space.w_str],
+            aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
-            types.UnicodeType(1),
+            types.UnicodeType(0),
             num=19,
             kind=UNICODELTR,
             name='unicode',
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -418,8 +418,16 @@
         addr = self.implementation.get_storage_as_int(space)
         # will explode if it can't
         w_d = space.newdict()
-        space.setitem_str(w_d, 'data', space.newtuple([space.wrap(addr),
-                                                       space.w_False]))
+        space.setitem_str(w_d, 'data',
+                          space.newtuple([space.wrap(addr), space.w_False]))
+        space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
+        space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
+        if self.implementation.order == 'C':
+            # Array is contiguous, no strides in the interface.
+            strides = space.w_None
+        else:
+            strides = self.descr_get_strides(space)
+        space.setitem_str(w_d, 'strides', strides)
         return w_d
 
     w_pypy_data = None
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -683,6 +683,20 @@
         assert dtype('=i8').byteorder == '='
         assert dtype(byteorder + 'i8').byteorder == '='
 
+    def test_dtype_str(self):
+        from numpypy import dtype
+        byteorder = self.native_prefix
+        assert dtype('i8').str == byteorder + 'i8'
+        assert dtype('<i8').str == '<i8'
+        assert dtype('>i8').str == '>i8'
+        assert dtype('int8').str == '|i1'
+        assert dtype('float').str == byteorder + 'f8'
+        # strange
+        assert dtype('string').str == '|S0'
+        assert dtype('unicode').str == byteorder + 'U0'
+        # assert dtype(('string', 7)).str == '|S7'
+        # assert dtype(('unicode', 7)).str == '<U7'
+
     def test_intp(self):
         from numpypy import dtype
         assert dtype('p') == dtype('intp')
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2138,6 +2138,9 @@
         a = array([1, 2, 3])
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
+        assert i['shape'] == (3,)
+        assert i['strides'] == None  # Because array is in C order
+        assert i['typestr'] == a.dtype.str
         a = a[::2]
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -103,11 +103,6 @@
         'flags'                 : 'app.null_sysflags',
     }
 
-    def setbuiltinmodule(self, w_module, name):
-        w_name = self.space.wrap(name)
-        w_modules = self.get('modules')
-        self.space.setitem(w_modules, w_name, w_module)
-
     def startup(self, space):
         if space.config.translating and not we_are_translated():
             # don't get the filesystemencoding at translation time
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -29,14 +29,13 @@
     try:
         # first try to print the exception's class name
         stderr = sys.stderr
-        stderr.write(getattr(exctype, '__name__', exctype))
+        stderr.write(str(getattr(exctype, '__name__', exctype)))
         # then attempt to get the str() of the exception
         try:
             s = str(value)
         except:
             s = '<failure of str() on the exception instance>'
-        # then print it, and don't worry too much about the extra space
-        # between the exception class and the ':'
+        # then print it
         if s:
             stderr.write(': %s\n' % (s,))
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -41,40 +41,43 @@
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
+        # NOTE: 'expected_*' is the numbers expected from GCC.
+        # The numbers expected from MSVC are not explicitly written
+        # in this file, and will just be taken from the compiler.
         ffi = FFI()
         ffi.cdef("struct s1 { %s };" % source)
         ctype = ffi.typeof("struct s1")
         # verify the information with gcc
-        if sys.platform != "win32":
-            ffi1 = FFI()
-            ffi1.cdef("""
-                static const int Gofs_y, Galign, Gsize;
-                struct s1 *try_with_value(int fieldnum, long long value);
-            """)
-            fnames = [name for name, cfield in ctype.fields
-                           if name and cfield.bitsize > 0]
-            setters = ['case %d: s.%s = value; break;' % iname
-                       for iname in enumerate(fnames)]
-            lib = ffi1.verify("""
-                struct s1 { %s };
-                struct sa { char a; struct s1 b; };
-                #define Gofs_y  offsetof(struct s1, y)
-                #define Galign  offsetof(struct sa, b)
-                #define Gsize   sizeof(struct s1)
-                struct s1 *try_with_value(int fieldnum, long long value)
-                {
-                    static struct s1 s;
-                    memset(&s, 0, sizeof(s));
-                    switch (fieldnum) { %s }
-                    return &s;
-                }
-            """ % (source, ' '.join(setters)))
-            assert lib.Gofs_y == expected_ofs_y
-            assert lib.Galign == expected_align
-            assert lib.Gsize  == expected_size
+        ffi1 = FFI()
+        ffi1.cdef("""
+            static const int Gofs_y, Galign, Gsize;
+            struct s1 *try_with_value(int fieldnum, long long value);
+        """)
+        fnames = [name for name, cfield in ctype.fields
+                       if name and cfield.bitsize > 0]
+        setters = ['case %d: s.%s = value; break;' % iname
+                   for iname in enumerate(fnames)]
+        lib = ffi1.verify("""
+            struct s1 { %s };
+            struct sa { char a; struct s1 b; };
+            #define Gofs_y  offsetof(struct s1, y)
+            #define Galign  offsetof(struct sa, b)
+            #define Gsize   sizeof(struct s1)
+            struct s1 *try_with_value(int fieldnum, long long value)
+            {
+                static struct s1 s;
+                memset(&s, 0, sizeof(s));
+                switch (fieldnum) { %s }
+                return &s;
+            }
+        """ % (source, ' '.join(setters)))
+        if sys.platform == 'win32':
+            expected_ofs_y = lib.Gofs_y
+            expected_align = lib.Galign
+            expected_size  = lib.Gsize
         else:
-            lib = None
-            fnames = None
+            assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (
+                expected_ofs_y, expected_align, expected_size)
         # the real test follows
         assert ffi.offsetof("struct s1", "y") == expected_ofs_y
         assert ffi.alignof("struct s1") == expected_align
@@ -99,10 +102,9 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
-        if lib is not None:
-            t = lib.try_with_value(fnames.index(name), value)
-            raw2 = ffi.buffer(t, len(raw1))[:]
-            assert raw1 == raw2
+        t = lib.try_with_value(fnames.index(name), value)
+        raw2 = ffi.buffer(t, len(raw1))[:]
+        assert raw1 == raw2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
@@ -136,9 +138,11 @@
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
         self.check("char x; int :0; char y;", 4, 1, 5)
+        self.check("char x; int :0; int :0; char y;", 4, 1, 5)
         self.check("char x; long long :0; char y;", L, 1, L + 1)
         self.check("short x, y; int :0; int :0;", 2, 2, 4)
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
     def test_error_cases(self):
         ffi = FFI()
diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py
--- a/pypy/module/thread/test/test_fork.py
+++ b/pypy/module/thread/test/test_fork.py
@@ -28,7 +28,7 @@
                 if pid == 0:
                     os._exit(0)
                 else:
-                    self.timeout_killer(pid, 5)
+                    self.timeout_killer(pid, 10)
                     exitcode = os.waitpid(pid, 0)[1]
                     assert exitcode == 0 # if 9, process was killed by timer!
             finally:
@@ -54,7 +54,7 @@
                 thread.start_new_thread(lambda: None, ())
                 os._exit(0)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 assert exitcode == 0 # if 9, process was killed by timer!
 
@@ -73,7 +73,7 @@
                 signal.signal(signal.SIGUSR1, signal.SIG_IGN)
                 os._exit(42)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 feedback.append(exitcode)
 
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -113,7 +113,7 @@
         tmpreg = r.lr
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None: # store
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.load_reg(mc, excvalloc, r.ip)
         if on_frame:
             # store exc_value in JITFRAME
@@ -125,7 +125,7 @@
             self.store_reg(mc, r.ip, r.fp, ofs, helper=tmpreg)
         if exctploc is not None:
             # store pos_exception in exctploc
-            assert exctploc.is_reg()
+            assert exctploc.is_core_reg()
             mc.gen_load_int(r.ip.value, self.cpu.pos_exception())
             self.load_reg(mc, exctploc, r.ip, helper=tmpreg)
 
@@ -146,7 +146,7 @@
         tmpreg = r.lr # use lr as a second temporary reg
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None:
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.store_reg(mc, excvalloc, r.ip)
         else:
             assert exctploc is not r.fp
@@ -947,7 +947,7 @@
     # regalloc support
     def load(self, loc, value):
         """load an immediate value into a register"""
-        assert (loc.is_reg() and value.is_imm()
+        assert (loc.is_core_reg() and value.is_imm()
                     or loc.is_vfp_reg() and value.is_imm_float())
         if value.is_imm():
             self.mc.gen_load_int(loc.value, value.getint())
@@ -958,7 +958,7 @@
     def load_reg(self, mc, target, base, ofs=0, cond=c.AL, helper=r.ip):
         if target.is_vfp_reg():
             return self._load_vfp_reg(mc, target, base, ofs, cond, helper)
-        elif target.is_reg():
+        elif target.is_core_reg():
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
@@ -1012,7 +1012,7 @@
     def _mov_imm_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.type == FLOAT:
             raise AssertionError("invalid target for move from imm value")
-        if loc.is_reg():
+        if loc.is_core_reg():
             new_loc = loc
         elif loc.is_stack() or loc.is_raw_sp():
             new_loc = r.lr
@@ -1027,7 +1027,7 @@
     def _mov_reg_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.is_imm():
             raise AssertionError("mov reg to imm doesn't make sense")
-        if loc.is_reg():
+        if loc.is_core_reg():
             self.mc.MOV_rr(loc.value, prev_loc.value, cond=cond)
         elif loc.is_stack() and loc.type != FLOAT:
             # spill a core register
@@ -1050,7 +1050,7 @@
         helper = None
         offset = prev_loc.value
         tmp = None
-        if loc.is_reg():
+        if loc.is_core_reg():
             assert prev_loc.type != FLOAT, 'trying to load from an \
                 incompatible location into a core register'
             # unspill a core register
@@ -1126,7 +1126,7 @@
         """Moves a value from a previous location to some other location"""
         if prev_loc.is_imm():
             return self._mov_imm_to_loc(prev_loc, loc, cond)
-        elif prev_loc.is_reg():
+        elif prev_loc.is_core_reg():
             self._mov_reg_to_loc(prev_loc, loc, cond)
         elif prev_loc.is_stack():
             self._mov_stack_to_loc(prev_loc, loc, cond)
@@ -1215,7 +1215,7 @@
                 scratch_reg = r.vfp_ip
             self.regalloc_mov(loc, scratch_reg, cond)
             self.regalloc_push(scratch_reg, cond)
-        elif loc.is_reg():
+        elif loc.is_core_reg():
             self.mc.PUSH([loc.value], cond=cond)
         elif loc.is_vfp_reg():
             self.mc.VPUSH([loc.value], cond=cond)
@@ -1238,7 +1238,7 @@
                 scratch_reg = r.vfp_ip
             self.regalloc_pop(scratch_reg)
             self.regalloc_mov(scratch_reg, loc)
-        elif loc.is_reg():
+        elif loc.is_core_reg():
             self.mc.POP([loc.value], cond=cond)
         elif loc.is_vfp_reg():
             self.mc.VPOP([loc.value], cond=cond)
@@ -1306,7 +1306,7 @@
 
         # lengthloc is the length of the array, which we must not modify!
         assert lengthloc is not r.r0 and lengthloc is not r.r1
-        if lengthloc.is_reg():
+        if lengthloc.is_core_reg():
             varsizeloc = lengthloc
         else:
             assert lengthloc.is_stack()
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -40,7 +40,7 @@
         if self.fnloc.is_stack():
             self.asm.mov_loc_loc(self.fnloc, r.ip)
             self.fnloc = r.ip
-        assert self.fnloc.is_reg()
+        assert self.fnloc.is_core_reg()
         self.mc.BLX(self.fnloc.value)
 
     def restore_stack_pointer(self):
@@ -135,7 +135,7 @@
             return [], []
         if self.resloc.is_vfp_reg():
             return [r.r0, r.r1], []
-        assert self.resloc.is_reg()
+        assert self.resloc.is_core_reg()
         return [r.r0], []
 
     def load_result(self):
@@ -146,7 +146,7 @@
         if resloc.is_vfp_reg():
             # move result to the allocated register
             self.asm.mov_to_vfp_loc(r.r0, r.r1, resloc)
-        elif resloc.is_reg():
+        elif resloc.is_core_reg():
             # move result to the allocated register
             if resloc is not r.r0:
                 self.asm.mov_loc_loc(r.r0, resloc)
@@ -283,7 +283,7 @@
     def load_result(self):
         resloc = self.resloc
         # ensure the result is wellformed and stored in the correct location
-        if resloc is not None and resloc.is_reg():
+        if resloc is not None and resloc.is_core_reg():
             self._ensure_result_bit_extension(resloc,
                                                   self.ressize, self.ressign)
 
@@ -292,7 +292,7 @@
             return [], []
         if self.resloc.is_vfp_reg():
             return [], [r.d0]
-        assert self.resloc.is_reg()
+        assert self.resloc.is_core_reg()
         return [r.r0], []
 
 
diff --git a/rpython/jit/backend/arm/helper/assembler.py b/rpython/jit/backend/arm/helper/assembler.py
--- a/rpython/jit/backend/arm/helper/assembler.py
+++ b/rpython/jit/backend/arm/helper/assembler.py
@@ -82,7 +82,7 @@
         assert guard is not None
         l0 = arglocs[0]
         l1 = arglocs[1]
-        assert l0.is_reg()
+        assert l0.is_core_reg()
 
         if l1.is_imm():
             self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond)
diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py
--- a/rpython/jit/backend/arm/locations.py
+++ b/rpython/jit/backend/arm/locations.py
@@ -15,7 +15,7 @@
     def is_raw_sp(self):
         return False
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def is_vfp_reg(self):
@@ -43,7 +43,7 @@
     def __repr__(self):
         return 'r%d' % self.value
 
-    def is_reg(self):
+    def is_core_reg(self):
         return True
 
     def as_key(self):
@@ -62,7 +62,7 @@
     def __repr__(self):
         return 'vfp%d' % self.value
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def is_vfp_reg(self):
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -248,7 +248,7 @@
         l1 = arglocs[1]
         failargs = arglocs[2:]
 
-        if l0.is_reg():
+        if l0.is_core_reg():
             if l1.is_imm():
                 self.mc.CMP_ri(l0.value, l1.getint())
             else:
@@ -488,7 +488,7 @@
             # case GCFLAG_CARDS_SET: emit a few instructions to do
             # directly the card flag setting
             loc_index = arglocs[1]
-            assert loc_index.is_reg()
+            assert loc_index.is_core_reg()
             # must save the register loc_index before it is mutated
             mc.PUSH([loc_index.value])
             tmp1 = loc_index
@@ -588,7 +588,7 @@
 
     def emit_op_setarrayitem_gc(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         if scale.value > 0:
             self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale.value)
             ofs_loc = r.ip
@@ -606,7 +606,7 @@
             # vstr only supports imm offsets
             # so if the ofset is too large we add it to the base and use an
             # offset of 0
-            if ofs_loc.is_reg():
+            if ofs_loc.is_core_reg():
                 tmploc, save = self.get_tmp_reg([value_loc, base_loc, ofs_loc])
                 assert not save
                 self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
@@ -644,13 +644,13 @@
 
     def emit_op_raw_store(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         self._write_to_mem(value_loc, base_loc, ofs_loc, scale, fcond)
         return fcond
 
     def emit_op_getarrayitem_gc(self, op, arglocs, regalloc, fcond):
         res_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         signed = op.getdescr().is_item_signed()
 
         # scale the offset as required
@@ -672,7 +672,7 @@
             # vldr only supports imm offsets
             # if the offset is in a register we add it to the base and use a
             # tmp reg
-            if ofs_loc.is_reg():
+            if ofs_loc.is_core_reg():
                 tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
                 assert not save
                 self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
@@ -727,7 +727,7 @@
 
     def emit_op_raw_load(self, op, arglocs, regalloc, fcond):
         res_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         # no base offset
         assert ofs.value == 0
         signed = op.getdescr().is_item_signed()
@@ -805,10 +805,10 @@
             bytes_box = TempBox()
             bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, forbidden_vars)
             scale = self._get_unicode_item_scale()
-            if not length_loc.is_reg():
+            if not length_loc.is_core_reg():
                 self.regalloc_mov(length_loc, bytes_loc)
                 length_loc = bytes_loc
-            assert length_loc.is_reg()
+            assert length_loc.is_core_reg()
             self.mc.MOV_ri(r.ip.value, 1 << scale)
             self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value)
             length_box = bytes_box
@@ -835,8 +835,8 @@
 
    # result = base_loc  + (scaled_loc << scale) + static_offset
     def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0):
-        assert scaled_loc.is_reg()
-        assert base_loc.is_reg()
+        assert scaled_loc.is_core_reg()
+        assert base_loc.is_core_reg()
         assert check_imm_arg(scale)
         assert check_imm_arg(static_offset)
         if scale > 0:
@@ -1063,7 +1063,7 @@
     def emit_op_cast_float_to_int(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert arg.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VCVT_float_to_int(r.vfp_ip.value, arg.value)
         self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
@@ -1071,7 +1071,7 @@
     def emit_op_cast_int_to_float(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert res.is_vfp_reg()
-        assert arg.is_reg()
+        assert arg.is_core_reg()
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
         self.mc.VCVT_int_to_float(res.value, res.value)
@@ -1087,7 +1087,7 @@
         loc = arglocs[0]
         res = arglocs[1]
         assert loc.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VMOV_rc(res.value, r.ip.value, loc.value)
         return fcond
 
@@ -1108,7 +1108,7 @@
     def emit_op_cast_float_to_singlefloat(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert arg.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VCVT_f64_f32(r.vfp_ip.value, arg.value)
         self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
@@ -1116,7 +1116,7 @@
     def emit_op_cast_singlefloat_to_float(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert res.is_vfp_reg()
-        assert arg.is_reg()
+        assert arg.is_core_reg()
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
         self.mc.VCVT_f32_f64(res.value, res.value)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -324,7 +324,7 @@
                 loc = r.fp
             arg = inputargs[i]
             i += 1
-            if loc.is_reg():
+            if loc.is_core_reg():
                 self.rm.reg_bindings[arg] = loc
                 used[loc] = None
             elif loc.is_vfp_reg():
@@ -346,6 +346,8 @@
         # note: we need to make a copy of inputargs because possibly_free_vars
         # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
+        self.fm.finish_binding()
+        self._check_invariants()
 
     def get_gcmap(self, forbidden_regs=[], noregs=False):
         frame_depth = self.fm.get_frame_depth()
@@ -356,7 +358,7 @@
                 continue
             if box.type == REF and self.rm.is_still_alive(box):
                 assert not noregs
-                assert loc.is_reg()
+                assert loc.is_core_reg()
                 val = loc.value
                 gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8))
         for box, loc in self.fm.bindings.iteritems():
@@ -1152,7 +1154,7 @@
             assert isinstance(arg, Box)
             loc = self.loc(arg)
             arglocs[i] = loc
-            if loc.is_reg():
+            if loc.is_core_reg() or loc.is_vfp_reg():
                 self.frame_manager.mark_as_free(arg)
         #
         descr._arm_arglocs = arglocs
diff --git a/rpython/jit/backend/arm/test/test_jump.py b/rpython/jit/backend/arm/test/test_jump.py
--- a/rpython/jit/backend/arm/test/test_jump.py
+++ b/rpython/jit/backend/arm/test/test_jump.py
@@ -255,7 +255,7 @@
                 else:
                     newvalue = 'value-vfp-%d' % i
                 regs2[loc.value] = newvalue
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 regs1[loc.value] = 'value-int-%d' % i
             elif loc.is_stack():
                 stack[loc.position] = 'value-width%d-%d' % (loc.width, i)
@@ -284,7 +284,7 @@
                 assert loc.width == expected_width*WORD
             if loc.is_vfp_reg():
                 return regs2[loc.value]
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 return regs1[loc.value]
             elif loc.is_stack():
                 got = stack[loc.position]
@@ -298,7 +298,7 @@
         def write(loc, newvalue):
             if loc.is_vfp_reg():
                 regs2[loc.value] = newvalue
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 regs1[loc.value] = newvalue
             elif loc.is_stack():
                 if loc.width > WORD:
@@ -317,17 +317,17 @@
         for op in assembler.ops:
             if op[0] == 'mov':
                 src, dst = op[1:]
-                assert src.is_reg() or src.is_vfp_reg() or src.is_stack() or src.is_imm_float() or src.is_imm()
-                assert dst.is_reg() or dst.is_vfp_reg() or dst.is_stack()
+                assert src.is_core_reg() or src.is_vfp_reg() or src.is_stack() or src.is_imm_float() or src.is_imm()
+                assert dst.is_core_reg() or dst.is_vfp_reg() or dst.is_stack()
                 assert not (src.is_stack() and dst.is_stack())
                 write(dst, read(src))
             elif op[0] == 'push':
                 src, = op[1:]
-                assert src.is_reg() or src.is_vfp_reg() or src.is_stack()
+                assert src.is_core_reg() or src.is_vfp_reg() or src.is_stack()
                 extrapushes.append(read(src))
             elif op[0] == 'pop':
                 dst, = op[1:]
-                assert dst.is_reg() or dst.is_vfp_reg() or dst.is_stack()
+                assert dst.is_core_reg() or dst.is_vfp_reg() or dst.is_stack()
                 write(dst, extrapushes.pop())
             else:
                 assert 0, "unknown op: %r" % (op,)
diff --git a/rpython/jit/backend/arm/test/test_runner.py b/rpython/jit/backend/arm/test/test_runner.py
--- a/rpython/jit/backend/arm/test/test_runner.py
+++ b/rpython/jit/backend/arm/test/test_runner.py
@@ -11,6 +11,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import JitCellToken, TargetToken
 from rpython.jit.backend.arm.detect import detect_arch_version
+from rpython.jit.codewriter import longlong
 
 
 CPU = getcpuclass()
@@ -261,3 +262,43 @@
         l1 = ('debug_print', preambletoken.repr_of_descr() + ':1')
         l2 = ('debug_print', targettoken.repr_of_descr() + ':9')
         assert ('jit-backend-counts', [l0, l1, l2]) in dlog
+
+
+    def test_label_float_in_reg_and_on_stack(self):
+        targettoken = TargetToken()
+        ops = """
+        [i0, f3]
+        i2 = same_as(i0)    # but forced to be in a register
+        force_spill(i2)
+        force_spill(f3)
+        f4 = float_add(f3, 5.0)
+        label(f3, f4, descr=targettoken)
+        force_spill(f3)
+        f5 = same_as(f3)    # but forced to be in a register
+        finish(f5)
+        """
+        faildescr = BasicFailDescr(2)
+        loop = parse(ops, self.cpu, namespace=locals())
+        looptoken = JitCellToken()
+        info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        ops2 = """
+        [i0, f1]
+        i1 = same_as(i0)
+        f2 = same_as(f1)
+        f3 = float_add(f1, 10.0)
+        force_spill(f3)
+        force_spill(i1)
+        f4 = float_add(f3, f1)
+        jump(f3, f4, descr=targettoken)
+        """
+        loop2 = parse(ops2, self.cpu, namespace=locals())
+        looptoken2 = JitCellToken()
+        info = self.cpu.compile_loop(loop2.inputargs, loop2.operations, looptoken2)
+
+        deadframe = self.cpu.execute_token(looptoken, -9, longlong.getfloatstorage(-13.5))
+        res = longlong.getrealfloat(self.cpu.get_float_value(deadframe, 0))
+        assert res == -13.5
+        #
+        deadframe = self.cpu.execute_token(looptoken2, -9, longlong.getfloatstorage(-13.5))
+        res = longlong.getrealfloat(self.cpu.get_float_value(deadframe, 0))
+        assert res == -3.5
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -47,7 +47,7 @@
             input_i += 1
             if arg.type == REF:
                 loc = fail_locs[i]
-                if loc.is_reg():
+                if loc.is_core_reg():
                     val = self.cpu.all_reg_indexes[loc.value]
                 else:
                     val = loc.get_position() + self.cpu.JITFRAME_FIXED_SIZE
diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_boehm_test.py
@@ -2,7 +2,6 @@
 import weakref
 from rpython.rlib.jit import JitDriver, dont_look_inside
 from rpython.jit.backend.llsupport.test.zrpy_gc_test import run, get_entry, compile
-from rpython.jit.backend.llsupport.test.ztranslation_test import fix_annotator_for_vrawbuffer
 
 class X(object):
     def __init__(self, x=0):
@@ -32,8 +31,7 @@
     g._dont_inline_ = True
     return g
 
-def compile_boehm_test(monkeypatch):
-    fix_annotator_for_vrawbuffer(monkeypatch)
+def compile_boehm_test():
     myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
     @dont_look_inside
     def see(lst, n):
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -9,23 +9,10 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 
 
-def fix_annotator_for_vrawbuffer(monkeypatch):
-    from rpython.rlib.nonconst import NonConstant
-    from rpython.jit.metainterp.optimizeopt.virtualize import VRawBufferValue
-    from rpython.jit.metainterp import warmspot
-
-    def my_hook_for_tests(cpu):
-        # this is needed so that the annotator can see it
-        if NonConstant(False):
-            v = VRawBufferValue(cpu, None, -1, None, None)
-    monkeypatch.setattr(warmspot, 'hook_for_tests', my_hook_for_tests)
-
-
 class TranslationTest(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_stuff_translates(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_stuff_translates(self):
         # this is a basic test that tries to hit a number of features and their
         # translation:
         # - jitting of loops and bridges
@@ -102,10 +89,9 @@
 class TranslationTestCallAssembler(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_direct_assembler_call_translates(self, monkeypatch):
+    def test_direct_assembler_call_translates(self):
         """Test CALL_ASSEMBLER and the recursion limit"""
         from rpython.rlib.rstackovf import StackOverflow
-        fix_annotator_for_vrawbuffer(monkeypatch)
 
         class Thing(object):
             def __init__(self, val):
@@ -183,8 +169,7 @@
 class TranslationTestJITStats(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_jit_get_stats(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_jit_get_stats(self):
         driver = JitDriver(greens = [], reds = ['i'])
 
         def f():
@@ -207,8 +192,7 @@
 class TranslationRemoveTypePtrTest(CCompiledMixin):
     CPUClass = getcpuclass()
 
-    def test_external_exception_handling_translates(self, monkeypatch):
-        fix_annotator_for_vrawbuffer(monkeypatch)
+    def test_external_exception_handling_translates(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'total'])
 
         class ImDone(Exception):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1551,14 +1551,14 @@
         frame in jf_guard_exc
         """
         if excvalloc is not None:
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             mc.MOV(excvalloc, heap(self.cpu.pos_exc_value()))
         elif tmploc is not None: # if both are None, just ignore
             ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc')
             mc.MOV(tmploc, heap(self.cpu.pos_exc_value()))
             mc.MOV(RawEbpLoc(ofs), tmploc)
         if exctploc is not None:
-            assert exctploc.is_reg()
+            assert exctploc.is_core_reg()
             mc.MOV(exctploc, heap(self.cpu.pos_exception()))
 
         mc.MOV(heap(self.cpu.pos_exception()), imm0)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -227,18 +227,6 @@
         else:
             return self.xrm.make_sure_var_in_reg(var, forbidden_vars)
 
-    def _frame_bindings(self, locs, inputargs):
-        bindings = {}
-        i = 0
-        for loc in locs:
-            if loc is None:
-                continue
-            arg = inputargs[i]
-            i += 1
-            if not isinstance(loc, RegLoc):
-                bindings[arg] = loc
-        return bindings
-
     def _update_bindings(self, locs, inputargs):
         # XXX this should probably go to llsupport/regalloc.py
         used = {}
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -45,7 +45,7 @@
     def is_stack(self):
         return False
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def get_position(self):
@@ -169,7 +169,7 @@
     def is_float(self):
         return self.is_xmm
 
-    def is_reg(self):
+    def is_core_reg(self):
         return True
 
 class ImmediateAssemblerLocation(AssemblerLocation):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -17,6 +17,7 @@
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
+    is_about_raw = False
     _cached_vinfo = None
 
     def __init__(self, keybox, source_op=None):
@@ -395,6 +396,7 @@
 
 
 class VRawBufferValue(AbstractVArrayValue):
+    is_about_raw = True
 
     def __init__(self, cpu, logops, size, keybox, source_op):
         AbstractVirtualValue.__init__(self, keybox, source_op)
@@ -457,6 +459,7 @@
 
 
 class VRawSliceValue(AbstractVirtualValue):
+    is_about_raw = True
 
     def __init__(self, rawbuffer_value, offset, keybox, source_op):
         AbstractVirtualValue.__init__(self, keybox, source_op)
@@ -676,13 +679,17 @@
         offsetbox = self.get_constant_box(op.getarg(1))
         if value.is_virtual() and offsetbox is not None:
             offset = offsetbox.getint()
-            if isinstance(value, VRawBufferValue):
-                self.make_virtual_raw_slice(value, offset, op.result, op)
-                return
-            elif isinstance(value, VRawSliceValue):
-                offset = offset + value.offset
-                self.make_virtual_raw_slice(value.rawbuffer_value, offset, op.result, op)
-                return
+            # the following check is constant-folded to False if the
+            # translation occurs without any VRawXxxValue instance around
+            if value.is_about_raw:
+                if isinstance(value, VRawBufferValue):
+                    self.make_virtual_raw_slice(value, offset, op.result, op)
+                    return
+                elif isinstance(value, VRawSliceValue):
+                    offset = offset + value.offset
+                    self.make_virtual_raw_slice(value.rawbuffer_value, offset,
+                                                op.result, op)
+                    return
         self.emit_operation(op)
 
     def optimize_ARRAYLEN_GC(self, op):
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -451,6 +451,7 @@
 
 class AbstractVirtualInfo(object):
     kind = REF
+    is_about_raw = False
     #def allocate(self, decoder, index):
     #    raise NotImplementedError
     def equals(self, fieldnums):
@@ -461,7 +462,7 @@
 
     def debug_prints(self):
         raise NotImplementedError
-        
+
 
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
     def __init__(self, fielddescrs):
@@ -547,6 +548,7 @@
 
 class VRawBufferStateInfo(AbstractVirtualInfo):
     kind = INT
+    is_about_raw = True
     
     def __init__(self, size, offsets, descrs):
         self.size = size
@@ -772,7 +774,9 @@
         assert self.virtuals_cache is not None
         v = self.virtuals_cache.get_int(index)
         if not v:
-            v = self.rd_virtuals[index].allocate_int(self, index)
+            v = self.rd_virtuals[index]
+            assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
+            v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")
         return v
 
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -761,8 +761,6 @@
         cpu = jd.warmstate.cpu
 
         def ll_portal_runner(*args):
-            hook_for_tests(cpu) # usually it's empty, but tests can monkeypatch
-                                # it to fix the annotator
             start = True
             while 1:
                 try:
@@ -999,10 +997,3 @@
         graphs = self.translator.graphs
         for graph, block, i in find_force_quasi_immutable(graphs):
             self.replace_force_quasiimmut_with_direct_call(block.operations[i])
-
-def hook_for_tests(cpu):
-    """
-    This function is empty and does nothing. Its only role is to be
-    monkey-patched by tests to "fix" the annotator if needed (see
-    e.g. x86/test/test_ztranslation::test_external_exception_handling_translates
-    """

From noreply at buildbot.pypy.org  Fri Jun 14 15:26:34 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Jun 2013 15:26:34 +0200 (CEST)
Subject: [pypy-commit] pypy argsort-segfault: document branch
Message-ID: <20130614132634.300AE1C3354@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: argsort-segfault
Changeset: r64883:4bcabd580cba
Date: 2013-06-14 16:25 +0300
http://bitbucket.org/pypy/pypy/changeset/4bcabd580cba/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,3 +52,7 @@
 
 .. branch: ctypes-byref
 Add the '_obj' attribute on ctypes pointer() and byref() objects
+
+.. branch: argsort-segfault
+Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
+

From noreply at buildbot.pypy.org  Fri Jun 14 15:48:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 15:48:47 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130614134847.1546B1C0134@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r124:bbee0ad53806
Date: 2013-06-14 15:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/bbee0ad53806/

Log:	Next test

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -450,17 +450,30 @@
   return L;
 }
 
+static inline gcptr check_flag_write_barrier(gcptr W)
+{
+  if (W->h_tid & GCFLAG_WRITE_BARRIER)
+    {
+      struct tx_descriptor *d = thread_descriptor;
+      gcptrlist_insert(&d->private_old_pointing_to_young, W);
+      W->h_tid &= ~GCFLAG_WRITE_BARRIER;
+    }
+  return W;
+}
+
 gcptr stm_WriteBarrier(gcptr P)
 {
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+    return check_flag_write_barrier(P);
+
   gcptr R, W;
+  R = stm_read_barrier(P);
+  if (is_private(R))
+    return check_flag_write_barrier(R);
+
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
-  R = stm_read_barrier(P);
-
-  if (is_private(R))
-    return R;
-
   spinlock_acquire(d->public_descriptor->collection_lock, 'L');
   if (d->public_descriptor->stolen_objects.size != 0)
     stm_normalize_stolen_objects(d);
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -27,6 +27,7 @@
 
     gcptrlist_delete(&d->old_objects_to_trace);
     gcptrlist_delete(&d->public_to_young);
+    gcptrlist_delete(&d->private_old_pointing_to_young);
 }
 
 static char *collect_and_allocate_size(size_t size);  /* forward */
@@ -131,6 +132,13 @@
     gcptrlist_clear(&d->public_to_young);
 }
 
+static void mark_private_old_pointing_to_young(struct tx_descriptor *d)
+{
+    /* trace the objects recorded earlier by stmgc_write_barrier() */
+    gcptrlist_move(&d->old_objects_to_trace,
+                   &d->private_old_pointing_to_young);
+}
+
 static void visit_all_outside_objects(struct tx_descriptor *d)
 {
     while (gcptrlist_size(&d->old_objects_to_trace) > 0) {
@@ -177,9 +185,7 @@
 
     mark_public_to_young(d);
 
-#if 0
     mark_private_old_pointing_to_young(d);
-#endif
 
     visit_all_outside_objects(d);
 #if 0
@@ -216,8 +222,8 @@
     if (d->nursery_current == d->nursery_base /*&&
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
-        //assert(gcptrlist_size(&d->private_old_pointing_to_young) == 0);
-        //assert(gcptrlist_size(&d->public_to_young) == 0);
+        assert(gcptrlist_size(&d->private_old_pointing_to_young) == 0);
+        assert(gcptrlist_size(&d->public_to_young) == 0);
         return 0;
     }
     else {
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -11,7 +11,8 @@
     char *nursery_end;                                  \
     char *nursery_base;                                 \
     struct GcPtrList old_objects_to_trace;              \
-    struct GcPtrList public_to_young;
+    struct GcPtrList public_to_young;                   \
+    struct GcPtrList private_old_pointing_to_young;
 
 struct tx_descriptor;  /* from et.h */
 
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -87,8 +87,8 @@
 gcptr stm_write_barrier(gcptr obj)
 {
     /* XXX inline in the caller */
-    if (UNLIKELY((obj->h_revision != stm_private_rev_num) &
-                 ((obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) == 0)))
+    if (UNLIKELY((obj->h_revision != stm_private_rev_num) |
+                 ((obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)))
         obj = stm_WriteBarrier(obj);
     return obj;
 }
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -71,3 +71,24 @@
     p3 = lib.stm_read_barrier(p1)
     assert not lib.in_nursery(p3) and p3 != p2
     assert lib.rawgetlong(p3, 0) == -91467
+
+def test_outer2inner():   # test mark_private_old_pointing_to_young()
+    p1 = nalloc_refs(1)
+    lib.stm_push_root(p1)
+    minor_collect()
+    check_nursery_free(p1)
+    p1 = lib.stm_pop_root()
+    assert classify(p1) == "private"
+    p2 = nalloc(HDR + WORD)
+    lib.setlong(p2, 0, 8972981)
+    lib.setptr(p1, 0, p2)
+    #
+    lib.stm_push_root(p1)
+    minor_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    check_nursery_free(p2)
+    p2b = lib.getptr(p1b, 0)
+    assert p2b != p2
+    check_not_free(p2b)
+    assert lib.getlong(p2b, 0) == 8972981

From noreply at buildbot.pypy.org  Fri Jun 14 15:55:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 15:55:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: untabbify
Message-ID: <20130614135523.B1C2D1C0134@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64884:8d8aa8c2badb
Date: 2013-06-14 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/8d8aa8c2badb/

Log:	untabbify

diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -31,13 +31,13 @@
     char *bytes;
     int length;
     Tcl_ObjType *typePtr;
-    union {			/* The internal representation: */
-	long longValue;		/*   - an long integer value. */
-	double doubleValue;	/*   - a double-precision floating value. */
-	struct {		/*   - internal rep as two pointers. */
-	    void *ptr1;
-	    void *ptr2;
-	} twoPtrValue;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
     } internalRep;
     ...;
 } Tcl_Obj;

From noreply at buildbot.pypy.org  Fri Jun 14 16:54:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 16:54:26 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130614145426.76C501C3322@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r125:61b9c5a13a48
Date: 2013-06-14 16:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/61b9c5a13a48/

Log:	Next test

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -119,16 +119,75 @@
 
 static void mark_public_to_young(struct tx_descriptor *d)
 {
+    /* "public_to_young" contains ptrs to the public copies used as
+       key of "public_to_private", but only the ones that were added
+       since the last minor collection.  Once the transaction commit,
+       they stay in "public_to_young", and so they become public
+       objects whose h_revision is a public stub, which itself points
+       (originally) to a protected young object.
+
+       Be careful and accept more or less any object in the list, which
+       can show up because of aborted transactions.
+    */
     long i, size = d->public_to_young.size;
     gcptr *items = d->public_to_young.items;
 
     for (i = 0; i < size; i++) {
         gcptr P = items[i];
+        assert(P->h_tid & GCFLAG_PUBLIC);
+
+        revision_t v = ACCESS_ONCE(P->h_revision);
         wlog_t *item;
+        G2L_FIND(d->public_to_private, P, item, goto not_in_public_to_private);
 
-        G2L_FIND(d->public_to_private, P, item, continue);
+        if (!(v & 1)) {   // "is a pointer"
+            /* P is both a key in public_to_private and an outdated copy.
+               We are in a case where we know the transaction will not
+               be able to commit successfully.
+            */
+            abort();
+            AbortTransactionAfterCollect(d, ABRT_COLLECT_MINOR);
+            //...
+        }
+
         visit_if_young(&item->val);
+        continue;
+
+    not_in_public_to_private:
+        if (v & 1) {   // "is not a pointer"
+            /* P is neither a key in public_to_private nor outdated.
+               It must come from an older transaction that aborted.
+               Nothing to do now.
+            */
+            continue;
+        }
+
+        gcptr S = (gcptr)v;
+        revision_t w = ACCESS_ONCE(S->h_revision);
+        if ((w & 3) != 2) {
+            /* P has a ptr in h_revision, but this object is not a stub
+               with a protected pointer.  It has likely been the case
+               in the past, but someone made even more changes.
+               Nothing to do now.
+            */
+            continue;
+        }
+
+        if (STUB_THREAD(S) != d->public_descriptor) {
+            /* Bah, it's indeed a stub but for another thread.  Nothing
+               to do now.
+            */
+            continue;
+        }
+
+        /* It's a stub for us.  It cannot be un-stubbed under our
+           feet because we hold our own collection_lock.
+        */
+        gcptr L = (gcptr)(w - 2);
+        visit_if_young(&L);
+        S->h_revision = ((revision_t)L) | 2;
     }
+
     gcptrlist_clear(&d->public_to_young);
 }
 
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -92,3 +92,35 @@
     assert p2b != p2
     check_not_free(p2b)
     assert lib.getlong(p2b, 0) == 8972981
+
+def test_outer2inner_after_transaction_end():
+    p1 = palloc(HDR + WORD)
+    lib.rawsetlong(p1, 0, 420063)
+    p2 = lib.stm_write_barrier(p1)
+    lib.rawsetlong(p2, 0, -91467)
+    assert lib.in_nursery(p2)
+    lib.stm_push_root(p1)
+    print "committing..."
+    transaction_break()
+    print "done"
+
+    # first check that the situation is still the same in the next transaction
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    assert classify(p1b) == "public"
+    p2b = lib.stm_read_barrier(p1b)
+    assert lib.in_nursery(p2b)
+    assert p2b == p2
+    assert classify(p2) == "protected"
+    check_not_free(p2b)
+    lib.stm_push_root(p1b)
+    print 'ok'
+
+    # then do a minor collection
+    minor_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    # check that the link p1 -> p2 was kept alive by moving p2 outside
+    p2b = lib.stm_read_barrier(p1b)
+    assert not lib.in_nursery(p2b)
+    check_not_free(p2b)

From noreply at buildbot.pypy.org  Fri Jun 14 17:01:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 17:01:37 +0200 (CEST)
Subject: [pypy-commit] stmgc default: More tests that pass
Message-ID: <20130614150137.6686E1C029E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r126:60f02332d53f
Date: 2013-06-14 16:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/60f02332d53f/

Log:	More tests that pass

diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -124,3 +124,35 @@
     p2b = lib.stm_read_barrier(p1b)
     assert not lib.in_nursery(p2b)
     check_not_free(p2b)
+
+def test_minor_collection_at_thread_end():
+    p1 = palloc_refs(1)
+    p2 = nalloc(HDR)
+    setptr(p1, 0, p2)
+    lib.stm_finalize()
+    lib.stm_initialize_tests(0)
+    p1b = getptr(p1, 0)
+    assert p1b != p1
+    assert not lib.in_nursery(p1b)
+    check_not_free(p1b)
+
+def test_prebuilt_keeps_alive():
+    p0 = palloc_refs(1)
+    p1 = nalloc(HDR)
+    lib.setptr(p0, 0, p1)
+    minor_collect()
+    check_nursery_free(p1)
+    check_prebuilt(p0)
+    p2 = lib.getptr(p0, 0)
+    assert not lib.in_nursery(p2)
+    check_not_free(p2)
+
+def test_prebuilt_keeps_alive_at_thread_end():
+    p0 = palloc_refs(1)
+    p1 = nalloc(HDR)
+    lib.setptr(p0, 0, p1)
+    lib.stm_finalize()
+    lib.stm_initialize_tests(0)
+    check_prebuilt(p0)
+    p2 = lib.getptr(p0, 0)
+    check_not_free(p2)

From noreply at buildbot.pypy.org  Fri Jun 14 17:01:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 17:01:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Reintroduce oalloc()
Message-ID: <20130614150138.B67841C029E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r127:25521e54d778
Date: 2013-06-14 17:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/25521e54d778/

Log:	Reintroduce oalloc()

diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -54,6 +54,7 @@
     void stm_set_transaction_length(long length_max);
 
     /* extra non-public code */
+    void *stm_malloc(size_t size);
     //gcptr stmgcpage_malloc(size_t size);
     //void stmgcpage_free(gcptr obj);
     //long stmgcpage_count(int quantity);
@@ -390,20 +391,18 @@
 # ____________________________________________________________
 
 def oalloc(size):
-    "Allocate an 'old' object, i.e. outside any nursery"
-    p = lib.stmgcpage_malloc(size)
-    p.h_tid = GCFLAG_WRITE_BARRIER | GCFLAG_OLD
-    p.h_revision = lib.get_local_revision()
+    "Allocate an 'old' public object, outside any nursery"
+    p = ffi.cast("gcptr", lib.stm_malloc(size))
+    p.h_tid = GCFLAG_OLD | GCFLAG_PUBLIC
+    p.h_revision = 1
     lib.settid(p, 42 + size)
     return p
 
-#ofree = lib.stmgcpage_free
-
 def oalloc_refs(nrefs):
-    "Allocate an 'old' object, i.e. outside any nursery, with nrefs pointers"
-    p = lib.stmgcpage_malloc(HDR + WORD * nrefs)
-    p.h_tid = GCFLAG_WRITE_BARRIER | GCFLAG_OLD
-    p.h_revision = lib.get_local_revision()
+    "Allocate an 'old' public object, outside any nursery, with nrefs pointers"
+    p = ffi.cast("gcptr", lib.stm_malloc(HDR + WORD * nrefs))
+    p.h_tid = GCFLAG_OLD | GCFLAG_PUBLIC
+    p.h_revision = 1
     lib.settid(p, 421 + nrefs)
     for i in range(nrefs):
         rawsetptr(p, i, ffi.NULL)
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -126,7 +126,7 @@
     check_not_free(p2b)
 
 def test_minor_collection_at_thread_end():
-    p1 = palloc_refs(1)
+    p1 = oalloc_refs(1)
     p2 = nalloc(HDR)
     setptr(p1, 0, p2)
     lib.stm_finalize()

From noreply at buildbot.pypy.org  Fri Jun 14 17:04:03 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Jun 2013 17:04:03 +0200 (CEST)
Subject: [pypy-commit] pypy argsort-segfault: close branch about to be merged
Message-ID: <20130614150403.2BA2B1C029E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: argsort-segfault
Changeset: r64885:c7483245daca
Date: 2013-06-14 18:00 +0300
http://bitbucket.org/pypy/pypy/changeset/c7483245daca/

Log:	close branch about to be merged


From noreply at buildbot.pypy.org  Fri Jun 14 17:04:04 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Jun 2013 17:04:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge argsort-segfault which fixes
	issue 1510
Message-ID: <20130614150404.5EA611C029E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r64886:c9e36178fe49
Date: 2013-06-14 18:02 +0300
http://bitbucket.org/pypy/pypy/changeset/c9e36178fe49/

Log:	merge argsort-segfault which fixes issue 1510

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,3 +52,7 @@
 
 .. branch: ctypes-byref
 Add the '_obj' attribute on ctypes pointer() and byref() objects
+
+.. branch: argsort-segfault
+Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
+
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -20,7 +20,7 @@
 def make_sort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
-    
+
     class Repr(object):
         def __init__(self, index_stride_size, stride_size, size, values,
                      indexes, index_start, start):
@@ -71,11 +71,11 @@
         def __init__(self, index_stride_size, stride_size, size):
             start = 0
             dtype = interp_dtype.get_dtype_cache(space).w_longdtype
-            self.indexes = dtype.itemtype.malloc(size*dtype.get_size())
-            self.values = alloc_raw_storage(size * stride_size, 
+            indexes = dtype.itemtype.malloc(size*dtype.get_size())
+            values = alloc_raw_storage(size * stride_size,
                                             track_allocation=False)
-            Repr.__init__(self, index_stride_size, stride_size, 
-                          size, self.values, self.indexes, start, start)
+            Repr.__init__(self, dtype.get_size(), stride_size,
+                          size, values, indexes, start, start)
 
         def __del__(self):
             free_raw_storage(self.indexes, track_allocation=False)
@@ -96,7 +96,7 @@
         for i in range(stop-start):
             retval.setitem(i, lst.getitem(i+start))
         return retval
-    
+
     if count < 2:
         def arg_lt(a, b):
             # Does numpy do <= ?
@@ -108,7 +108,7 @@
                     return True
                 elif a[0][i] > b[0][i]:
                     return False
-            # Does numpy do True?    
+            # Does numpy do True?
             return False
 
     ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
@@ -180,7 +180,7 @@
 
 class SortCache(object):
     built = False
-    
+
     def __init__(self, space):
         if self.built:
             return
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2499,6 +2499,13 @@
         b = a.argsort()
         assert (b[:3] == [0, 100, 200]).all()
 
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
     def test_argsort_axis(self):
         from numpypy import array
         a = array([[4, 2], [1, 3]])

From noreply at buildbot.pypy.org  Fri Jun 14 17:04:05 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Jun 2013 17:04:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20130614150405.8FD421C029E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r64887:3ed53260e74c
Date: 2013-06-14 18:03 +0300
http://bitbucket.org/pypy/pypy/changeset/3ed53260e74c/

Log:	merge heads

diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -31,13 +31,13 @@
     char *bytes;
     int length;
     Tcl_ObjType *typePtr;
-    union {			/* The internal representation: */
-	long longValue;		/*   - an long integer value. */
-	double doubleValue;	/*   - a double-precision floating value. */
-	struct {		/*   - internal rep as two pointers. */
-	    void *ptr1;
-	    void *ptr2;
-	} twoPtrValue;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
     } internalRep;
     ...;
 } Tcl_Obj;

From noreply at buildbot.pypy.org  Fri Jun 14 18:45:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 18:45:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-the-middle-of-progress
Message-ID: <20130614164502.E3D491C1398@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r128:9cfc156e53f7
Date: 2013-06-14 18:44 +0200
http://bitbucket.org/pypy/stmgc/changeset/9cfc156e53f7/

Log:	in-the-middle-of-progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -33,7 +33,7 @@
 static int is_private(gcptr P)
 {
   return (P->h_revision == stm_private_rev_num) ||
-    (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+            gcflag_private_from_protected(P);
 }
 int _stm_is_private(gcptr P)
 {
@@ -80,7 +80,7 @@
   revision_t v;
 
  restart_all:
-  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+  if (gcflag_private_from_protected(P))
     {
       assert(!(P->h_revision & 1));   /* pointer to the backup copy */
 
@@ -95,14 +95,14 @@
   /* else, for the rest of this function, we can assume that P was not
      a private copy */
 
-  if (P->h_tid & GCFLAG_PUBLIC)
+  if (gcflag_public(P))
     {
       /* follow the chained list of h_revision's as long as they are
          regular pointers.  We will only find more public objects
          along this chain.
       */
     restart_all_public:
-      assert(P->h_tid & GCFLAG_PUBLIC);
+      assert(gcflag_public(P));
       v = ACCESS_ONCE(P->h_revision);
       if (!(v & 1))  // "is a pointer", i.e.
         {            //      "has a more recent revision"
@@ -112,7 +112,7 @@
 
           gcptr P_prev = P;
           P = (gcptr)v;
-          assert(P->h_tid & GCFLAG_PUBLIC);
+          assert(gcflag_public(P));
 
           v = ACCESS_ONCE(P->h_revision);
 
@@ -145,7 +145,7 @@
          because *we* have an entry in d->public_to_private.  (It might
          also be someone else.)
       */
-      if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
+      if (gcflag_public_to_private(P))
         {
           wlog_t *item;
         retry_public_to_private:;
@@ -154,7 +154,7 @@
           /* We have a key in 'public_to_private'.  The value is the
              corresponding private object. */
           P = item->val;
-          assert(!(P->h_tid & GCFLAG_PUBLIC));
+          assert(!gcflag_public(P));
           assert(is_private(P));
           fprintf(stderr, "read_barrier: %p -> %p public_to_private\n", G, P);
           return P;
@@ -229,7 +229,7 @@
       fprintf(stderr, "read_barrier: %p -> stealing %p...\n  ", G, P);
       stm_steal_stub(P);
 
-      assert(P->h_tid & GCFLAG_PUBLIC);
+      assert(gcflag_public(P));
       goto restart_all_public;
     }
 }
@@ -245,8 +245,8 @@
   if (pubobj == P || ((P->h_revision & 3) == 2 &&
                       pubobj->h_revision == P->h_revision))
     {
-      assert(!(org_pubobj->h_tid & GCFLAG_STUB));
-      assert(!(privobj->h_tid & GCFLAG_PUBLIC));
+      assert(!gcflag_stub(org_pubobj));
+      assert(!gcflag_public(privobj));
       assert(is_private(privobj));
       if (P != org_pubobj)
         fprintf(stderr, "| actually %p ", org_pubobj);
@@ -299,7 +299,7 @@
       return P;
     }
 
-  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+  if (gcflag_private_from_protected(P))
     {
       /* private too, with a backup copy */
       assert(!(P->h_revision & 1));
@@ -307,7 +307,7 @@
       return P;
     }
 
-  if (P->h_tid & GCFLAG_PUBLIC)
+  if (gcflag_public(P))
     {
       fprintf(stderr, "public ");
 
@@ -323,7 +323,7 @@
             }
 
           P = (gcptr)v;
-          assert(P->h_tid & GCFLAG_PUBLIC);
+          assert(gcflag_public(P));
           fprintf(stderr, "-> %p public ", P);
         }
 
@@ -354,12 +354,12 @@
     {
       P = (gcptr)(v - 2);
       fprintf(stderr, "-foreign-> %p ", P);
-      if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+      if (gcflag_private_from_protected(P))
         {
           P = (gcptr)P->h_revision;     /* the backup copy */
           fprintf(stderr, "-backup-> %p ", P);
         }
-      if (!(P->h_tid & GCFLAG_PUBLIC))
+      if (!gcflag_public(P))
         {
           fprintf(stderr, "protected by someone else!\n");
           return (gcptr)-1;
@@ -397,10 +397,10 @@
 
   assert(P->h_revision != stm_private_rev_num);
   assert(P->h_revision & 1);
-  assert(!(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
-  assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
-  assert(!(P->h_tid & GCFLAG_STUB));
-  assert(!(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
+  assert(!gcflag_public_to_private(P));
+  assert(!gcflag_backup_copy(P));
+  assert(!gcflag_stub(P));
+  assert(!gcflag_private_from_protected(P));
 
   B = stmgc_duplicate(P);
   B->h_tid |= GCFLAG_BACKUP_COPY;
@@ -410,12 +410,12 @@
 
   gcptrlist_insert(&d->private_from_protected, P);
 
-  return P;
+  return P;   /* always returns its arg: the object is converted in-place */
 }
 
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
-  assert(R->h_tid & GCFLAG_PUBLIC);
+  assert(gcflag_public(R));
 
 #ifdef _GC_DEBUG
   wlog_t *entry;
@@ -427,9 +427,9 @@
   R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
   gcptr L = stmgc_duplicate(R);
-  assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
-  assert(!(L->h_tid & GCFLAG_STUB));
-  assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
+  assert(!gcflag_backup_copy(L));
+  assert(!gcflag_stub(L));
+  assert(!gcflag_private_from_protected(L));
   L->h_tid &= ~(GCFLAG_OLD               |
                 GCFLAG_VISITED           |
                 GCFLAG_PUBLIC            |
@@ -439,7 +439,6 @@
                 0);
   L->h_revision = stm_private_rev_num;
   g2l_insert(&d->public_to_private, R, L);
-  gcptrlist_insert(&d->public_to_young, R);
   fprintf(stderr, "write_barrier: adding %p -> %p to public_to_private\n",
           R, L);
 
@@ -450,38 +449,64 @@
   return L;
 }
 
-static inline gcptr check_flag_write_barrier(gcptr W)
+static inline void record_write_barrier(gcptr P)
 {
-  if (W->h_tid & GCFLAG_WRITE_BARRIER)
+  if (gcflag_write_barrier(P))
     {
-      struct tx_descriptor *d = thread_descriptor;
-      gcptrlist_insert(&d->private_old_pointing_to_young, W);
-      W->h_tid &= ~GCFLAG_WRITE_BARRIER;
+      P->h_tid &= ~GCFLAG_WRITE_BARRIER;
+      gcptrlist_insert(&thread_descriptor->old_with_young_pointers_inside, P);
     }
-  return W;
 }
 
 gcptr stm_WriteBarrier(gcptr P)
 {
-  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
-    return check_flag_write_barrier(P);
+  if (is_private(P))
+    {
+      /* If we have GCFLAG_WRITE_BARRIER in P, then list it into
+         old_with_young_pointers_inside: it's a private object that may
+         be modified by the program after we return, and the mutation
+         may be to write young pointers (in fact it's a common case).
+      */
+      record_write_barrier(P);
+      return P;
+    }
 
   gcptr R, W;
   R = stm_read_barrier(P);
+
   if (is_private(R))
-    return check_flag_write_barrier(R);
+    {
+      record_write_barrier(P);
+      return P;
+    }
 
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
+  /* We need the collection_lock for the sequel; this is required notably
+     because we're about to edit flags on a protected object.
+  */
   spinlock_acquire(d->public_descriptor->collection_lock, 'L');
   if (d->public_descriptor->stolen_objects.size != 0)
     stm_normalize_stolen_objects(d);
 
-  if (R->h_tid & GCFLAG_PUBLIC)
-    W = LocalizePublic(d, R);
+  if (gcflag_public(R))
+    {
+      /* Make and return a new (young) private copy of the public R.
+         Add R into the list 'old_public_with_young_copy'.
+      */
+      assert(gcflag_old(R));
+      gcptrlist_insert(&d->old_public_with_young_copy, R);
+      W = LocalizePublic(d, R);
+    }
   else
-    W = LocalizeProtected(d, R);
+    {
+      /* Turn the protected copy in-place into a private copy.  If it's
+         an old object that still has GCFLAG_WRITE_BARRIER, then we must
+         also record it in the list 'old_with_young_pointers_inside'. */
+      W = LocalizeProtected(d, R);
+      record_write_barrier(W);
+    }
 
   spinlock_release(d->public_descriptor->collection_lock);
 
@@ -528,7 +553,7 @@
       v = ACCESS_ONCE(R->h_revision);
       if (!(v & 1))               // "is a pointer", i.e.
         {                         //   "has a more recent revision"
-          if (R->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+          if (gcflag_private_from_protected(R))
             {
               /* such an object R might be listed in list_of_read_objects
                  before it was turned from protected to private */
@@ -786,7 +811,7 @@
       gcptr R = item->addr;
       revision_t v;
     retry:
-      assert(R->h_tid & GCFLAG_PUBLIC);
+      assert(gcflag_public(R));
       v = ACCESS_ONCE(R->h_revision);
       if (!(v & 1))            // "is a pointer", i.e.
         {                      //   "has a more recent revision"
@@ -806,7 +831,7 @@
         goto retry;
 
       gcptr L = item->val;
-      assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED ?
+      assert(gcflag_private_from_protected(L) ?
              L->h_revision == (revision_t)R :
              L->h_revision == stm_private_rev_num);
       assert(v != stm_private_rev_num);
@@ -830,7 +855,7 @@
       gcptr L = item->val;
       revision_t expected, v = L->h_revision;
 
-      if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+      if (gcflag_private_from_protected(L))
         expected = (revision_t)R;
       else
         expected = stm_private_rev_num;
@@ -867,10 +892,10 @@
   G2L_LOOP_FORWARD(d->public_to_private, item)
     {
       gcptr L = item->val;
-      assert(!(L->h_tid & GCFLAG_VISITED));
-      assert(!(L->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
-      assert(!(L->h_tid & GCFLAG_PREBUILT_ORIGINAL));
-      assert(!(L->h_tid & GCFLAG_NURSERY_MOVED));
+      assert(!gcflag_visited(L));
+      assert(!gcflag_public_to_private(L));
+      assert(!gcflag_prebuilt_original(L));
+      assert(!gcflag_nursery_moved(L));
       assert(L->h_revision != localrev);   /* modified by AcquireLocks() */
 
 #ifdef DUMP_EXTRA
@@ -894,9 +919,9 @@
       gcptr R = item->addr;
       revision_t v = (revision_t)item->val;
 
-      assert(R->h_tid & GCFLAG_PUBLIC);
-      assert(R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
-      assert(!(R->h_tid & GCFLAG_NURSERY_MOVED));
+      assert(gcflag_public(R));
+      assert(gcflag_public_to_private(R));
+      assert(!gcflag_nursery_moved(R));
       assert(R->h_revision != localrev);
 
 #ifdef DUMP_EXTRA
@@ -907,7 +932,7 @@
       ACCESS_ONCE(R->h_revision) = v;
 
 #if 0
-      if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
+      if (gcflag_prebuilt_original(R))
         {
           /* cannot possibly get here more than once for a given value of R */
           pthread_mutex_lock(&mutex_prebuilt_gcroots);
@@ -932,7 +957,7 @@
   for (i = 0; i < size; i++)
     {
       gcptr P = items[i];
-      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+      assert(gcflag_private_from_protected(P));
       P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
 
       if (P->h_revision & 1)   // "is not a pointer"
@@ -947,7 +972,7 @@
       gcptr B = (gcptr)P->h_revision;
       P->h_revision = new_revision;
 
-      if (B->h_tid & GCFLAG_PUBLIC)
+      if (gcflag_public(B))
         {
           /* B was stolen */
           while (1)
@@ -976,20 +1001,20 @@
   for (i = 0; i < size; i++)
     {
       gcptr P = items[i];
-      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+      assert(gcflag_private_from_protected(P));
       assert(!(P->h_revision & 1));   // "is a pointer"
 
       gcptr B = (gcptr)P->h_revision;
-      if (B->h_tid & GCFLAG_PUBLIC)
+      if (gcflag_public(B))
         {
-          assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
+          assert(!gcflag_backup_copy(B));
           P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
           P->h_tid |= GCFLAG_PUBLIC;
           /* P becomes a public outdated object */
         }
       else
         {
-          assert(B->h_tid & GCFLAG_BACKUP_COPY);
+          assert(gcflag_backup_copy(B));
           memcpy(P, B, stmcb_size(P));
           P->h_tid &= ~GCFLAG_BACKUP_COPY;
         }
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -39,11 +39,12 @@
  * GCFLAG_PREBUILT_ORIGINAL is only set on the original version of
  * prebuilt objects.
  *
- * GCFLAG_WRITE_BARRIER is set on *old* objects to track old-to- young
- * pointers.  It may be left set on *public* objects but is ignored
- * there, because public objects are read-only.  The flag is removed
- * once a write occurs and the object is recorded in the list
- * 'old_pointing_to_young'; it is set again at the next minor
+ * GCFLAG_WRITE_BARRIER is set on *old* objects to track old-to-young
+ * pointers.  It is only useful on private objects, and on protected
+ * objects (which may be turned private again).  It may be left set on
+ * public objects but is ignored there, because such objects are read-only.
+ * The flag is removed once a write occurs and the object is recorded in
+ * the list 'old_pointing_to_young'; it is set again at the next minor
  * collection.
  *
  * GCFLAG_NURSERY_MOVED is used temporarily during minor collections.
@@ -82,6 +83,20 @@
                          "PRIVATE_FROM_PROTECTED", \
                          NULL }
 
+#define _DECLARE_FLAG(funcname, flagname)       \
+    static inline _Bool funcname(gcptr P) {     \
+        return (P->h_tid & flagname) != 0; }
+_DECLARE_FLAG(gcflag_old,                    GCFLAG_OLD)
+_DECLARE_FLAG(gcflag_visited,                GCFLAG_VISITED)
+_DECLARE_FLAG(gcflag_public,                 GCFLAG_PUBLIC)
+_DECLARE_FLAG(gcflag_prebuilt_original,      GCFLAG_PREBUILT_ORIGINAL)
+_DECLARE_FLAG(gcflag_public_to_private,      GCFLAG_PUBLIC_TO_PRIVATE)
+_DECLARE_FLAG(gcflag_write_barrier,          GCFLAG_WRITE_BARRIER)
+_DECLARE_FLAG(gcflag_nursery_moved,          GCFLAG_NURSERY_MOVED)
+_DECLARE_FLAG(gcflag_backup_copy,            GCFLAG_BACKUP_COPY)
+_DECLARE_FLAG(gcflag_stub,                   GCFLAG_STUB)
+_DECLARE_FLAG(gcflag_private_from_protected, GCFLAG_PRIVATE_FROM_PROTECTED)
+
 /************************************************************/
 
 #define ABRT_MANUAL               0
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -67,11 +67,11 @@
 
 static inline gcptr create_old_object_copy(gcptr obj)
 {
-    assert(!(obj->h_tid & GCFLAG_NURSERY_MOVED));
-    assert(!(obj->h_tid & GCFLAG_VISITED));
-    assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
-    assert(!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
-    assert(!(obj->h_tid & GCFLAG_OLD));
+    assert(!gcflag_nursery_moved(obj));
+    assert(!gcflag_visited(obj));
+    assert(!gcflag_write_barrier(obj));
+    assert(!gcflag_prebuilt_original(obj));
+    assert(!gcflag_old(obj));
 
     size_t size = stmcb_size(obj);
     gcptr fresh_old_copy = stm_malloc(size);
@@ -134,7 +134,7 @@
 
     for (i = 0; i < size; i++) {
         gcptr P = items[i];
-        assert(P->h_tid & GCFLAG_PUBLIC);
+        assert(gcflag_public(P));
 
         revision_t v = ACCESS_ONCE(P->h_revision);
         wlog_t *item;
@@ -145,11 +145,14 @@
                We are in a case where we know the transaction will not
                be able to commit successfully.
             */
+            fprintf(stderr, "public_to_young: %p was modified! abort!\n", P);
             abort();
             AbortTransactionAfterCollect(d, ABRT_COLLECT_MINOR);
             //...
         }
 
+        fprintf(stderr, "public_to_young: %p -> %p in public_to_private\n",
+                item->addr, item->val);
         visit_if_young(&item->val);
         continue;
 
@@ -159,6 +162,7 @@
                It must come from an older transaction that aborted.
                Nothing to do now.
             */
+            fprintf(stderr, "public_to_young: %p ignored\n", P);
             continue;
         }
 
@@ -170,6 +174,8 @@
                in the past, but someone made even more changes.
                Nothing to do now.
             */
+            fprintf(stderr, "public_to_young: %p -> %p not a stub, ignored\n",
+                    P, S);
             continue;
         }
 
@@ -177,6 +183,8 @@
             /* Bah, it's indeed a stub but for another thread.  Nothing
                to do now.
             */
+            fprintf(stderr, "public_to_young: %p -> %p stub wrong thread, "
+                    "ignored\n", P, S);
             continue;
         }
 
@@ -184,6 +192,9 @@
            feet because we hold our own collection_lock.
         */
         gcptr L = (gcptr)(w - 2);
+        fprintf(stderr, "public_to_young: %p -> %p stub -> %p\n",
+                P, S, L);
+
         visit_if_young(&L);
         S->h_revision = ((revision_t)L) | 2;
     }
@@ -203,8 +214,8 @@
     while (gcptrlist_size(&d->old_objects_to_trace) > 0) {
         gcptr obj = gcptrlist_pop(&d->old_objects_to_trace);
 
-        assert(obj->h_tid & GCFLAG_OLD);
-        assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
+        assert(gcflag_old(obj));
+        assert(!gcflag_write_barrier(obj));
         obj->h_tid |= GCFLAG_WRITE_BARRIER;
 
         stmcb_trace(obj, &visit_if_young);
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -11,8 +11,8 @@
     char *nursery_end;                                  \
     char *nursery_base;                                 \
     struct GcPtrList old_objects_to_trace;              \
-    struct GcPtrList public_to_young;                   \
-    struct GcPtrList private_old_pointing_to_young;
+    struct GcPtrList old_public_with_young_copy;        \
+    struct GcPtrList old_with_young_pointers_inside;
 
 struct tx_descriptor;  /* from et.h */
 
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -55,7 +55,7 @@
 static void replace_ptr_to_protected_with_stub(gcptr *pobj)
 {
     gcptr stub, obj = *pobj;
-    if (obj == NULL || (obj->h_tid & GCFLAG_PUBLIC) != 0)
+    if (obj == NULL || gcflag_public(obj))
         return;
 
     /* we use 'all_stubs', a dictionary, in order to try to avoid
@@ -98,7 +98,7 @@
     /* L might be a private_from_protected, or just a protected copy.
        To know which case it is, read GCFLAG_PRIVATE_FROM_PROTECTED.
     */
-    if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+    if (gcflag_private_from_protected(L)) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
 
         /* B is now a backup copy, i.e. a protected object, and we own
@@ -107,9 +107,9 @@
         */
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
 
-        if (B->h_tid & GCFLAG_PUBLIC_TO_PRIVATE) {
+        if (gcflag_public_to_private(B)) {
             /* already stolen */
-            assert(B->h_tid & GCFLAG_PUBLIC);
+            assert(gcflag_public(B));
             fprintf(stderr, "already stolen: %p -> %p <-> %p\n", P, L, B);
             L = B;
             goto already_stolen;
@@ -125,7 +125,7 @@
         }
     }
     else {
-        if (L->h_tid & GCFLAG_PUBLIC) {
+        if (gcflag_public(L)) {
             /* already stolen */
             fprintf(stderr, "already stolen: %p -> %p\n", P, L);
             goto already_stolen;
@@ -139,7 +139,7 @@
        thread's collection_lock, so we can read/write the flags.  Change
        it from protected to public.
     */
-    assert(!(L->h_tid & GCFLAG_PUBLIC));
+    assert(!gcflag_public(L));
     L->h_tid |= GCFLAG_PUBLIC;
 
     /* Note that all protected or backup copies have a h_revision that
@@ -192,8 +192,8 @@
         gcptr B = items[i];
         gcptr L = items[i + 1];
 
-        assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
-        assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
+        assert(gcflag_private_from_protected(L));
+        assert(!gcflag_backup_copy(B));  /* already removed */
 
         g2l_insert(&d->public_to_private, B, L);
 
@@ -218,7 +218,7 @@
         gcptr B = items[i];
         gcptr L = items[i + 1];
 
-        assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        assert(gcflag_private_from_protected(L));
         if (B == obj)
             return L;
     }
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -391,18 +391,19 @@
 # ____________________________________________________________
 
 def oalloc(size):
-    "Allocate an 'old' public object, outside any nursery"
+    "Allocate an 'old' protected object, outside any nursery"
     p = ffi.cast("gcptr", lib.stm_malloc(size))
-    p.h_tid = GCFLAG_OLD | GCFLAG_PUBLIC
-    p.h_revision = 1
+    p.h_tid = GCFLAG_OLD | GCFLAG_WRITE_BARRIER
+    p.h_revision = -sys.maxint
     lib.settid(p, 42 + size)
     return p
 
 def oalloc_refs(nrefs):
-    "Allocate an 'old' public object, outside any nursery, with nrefs pointers"
+    """Allocate an 'old' protected object, outside any nursery,
+    with nrefs pointers"""
     p = ffi.cast("gcptr", lib.stm_malloc(HDR + WORD * nrefs))
-    p.h_tid = GCFLAG_OLD | GCFLAG_PUBLIC
-    p.h_revision = 1
+    p.h_tid = GCFLAG_OLD | GCFLAG_WRITE_BARRIER
+    p.h_revision = -sys.maxint
     lib.settid(p, 421 + nrefs)
     for i in range(nrefs):
         rawsetptr(p, i, ffi.NULL)
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -126,7 +126,7 @@
     check_not_free(p2b)
 
 def test_minor_collection_at_thread_end():
-    p1 = oalloc_refs(1)
+    p1 = palloc_refs(1)
     p2 = nalloc(HDR)
     setptr(p1, 0, p2)
     lib.stm_finalize()
@@ -156,3 +156,31 @@
     check_prebuilt(p0)
     p2 = lib.getptr(p0, 0)
     check_not_free(p2)
+
+def test_old_protected_stay_alive():
+    p0 = oalloc(HDR + WORD)
+    assert classify(p0) == "protected"
+    lib.rawsetlong(p0, 0, 81211)
+    lib.stm_push_root(p0)
+    minor_collect()
+    p0b = lib.stm_pop_root()
+    assert p0b == p0
+    assert classify(p0) == "protected"
+    assert lib.rawgetlong(p0, 0) == 81211
+
+def test_old_private_from_protected_to_young_private():
+    p0 = oalloc_refs(1)
+    assert classify(p0) == "protected"
+    p1 = nalloc(HDR)
+    lib.setptr(p0, 0, p1)
+    assert classify(p0) == "private"   # private_from_protected
+    lib.stm_push_root(p0)
+    minor_collect()
+    p0b = lib.stm_pop_root()
+    assert p0b == p0
+    check_nursery_free(p1)
+    assert classify(p0) == "private"   # private_from_protected
+    p2 = lib.getptr(p0, 0)
+    assert not lib.in_nursery(p2)
+    check_not_free(p2)
+    assert classify(p2) == "private"

From noreply at buildbot.pypy.org  Fri Jun 14 18:55:47 2013
From: noreply at buildbot.pypy.org (andrewsmedina)
Date: Fri, 14 Jun 2013 18:55:47 +0200 (CEST)
Subject: [pypy-commit] pypy dtype-isnative: implemented dtype.isnative.
Message-ID: <20130614165547.3BCA81C1398@cobra.cs.uni-duesseldorf.de>

Author: Andrews Medina <andrewsmedina at gmail.com>
Branch: dtype-isnative
Changeset: r64888:ec63aaab8cf5
Date: 2013-06-14 13:47 -0300
http://bitbucket.org/pypy/pypy/changeset/ec63aaab8cf5/

Log:	implemented dtype.isnative.

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -133,6 +133,9 @@
     def descr_get_alignment(self, space):
         return space.wrap(self.itemtype.alignment)
 
+    def descr_get_isnative(self, space):
+        return space.wrap(self.native)
+
     def descr_get_base(self, space):
         return space.wrap(self.base)
 
@@ -439,6 +442,7 @@
     str = GetSetProperty(W_Dtype.descr_get_str),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
     alignment = GetSetProperty(W_Dtype.descr_get_alignment),
+    isnative = GetSetProperty(W_Dtype.descr_get_isnative),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
     name = interp_attrproperty('name', cls=W_Dtype),
     fields = GetSetProperty(W_Dtype.descr_get_fields),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -706,6 +706,11 @@
         from numpypy import dtype
         assert dtype('i4').alignment == 4
 
+    def test_isnative(self):
+        from numpypy import dtype
+        assert dtype('i4').isnative == True
+        assert dtype('>i8').isnative == False
+
     def test_any_all(self):
         import numpypy as numpy
         x = numpy.bool_(True)

From noreply at buildbot.pypy.org  Fri Jun 14 18:55:48 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Jun 2013 18:55:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged in
 andrewsmedina/numpypy/dtype-isnative (pull request #155)
Message-ID: <20130614165548.6239F1C1398@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64889:23314bb14a1f
Date: 2013-06-14 09:55 -0700
http://bitbucket.org/pypy/pypy/changeset/23314bb14a1f/

Log:	Merged in andrewsmedina/numpypy/dtype-isnative (pull request #155)

	implemented dtype.isnative.

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -133,6 +133,9 @@
     def descr_get_alignment(self, space):
         return space.wrap(self.itemtype.alignment)
 
+    def descr_get_isnative(self, space):
+        return space.wrap(self.native)
+
     def descr_get_base(self, space):
         return space.wrap(self.base)
 
@@ -439,6 +442,7 @@
     str = GetSetProperty(W_Dtype.descr_get_str),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
     alignment = GetSetProperty(W_Dtype.descr_get_alignment),
+    isnative = GetSetProperty(W_Dtype.descr_get_isnative),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
     name = interp_attrproperty('name', cls=W_Dtype),
     fields = GetSetProperty(W_Dtype.descr_get_fields),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -706,6 +706,11 @@
         from numpypy import dtype
         assert dtype('i4').alignment == 4
 
+    def test_isnative(self):
+        from numpypy import dtype
+        assert dtype('i4').isnative == True
+        assert dtype('>i8').isnative == False
+
     def test_any_all(self):
         import numpypy as numpy
         x = numpy.bool_(True)

From noreply at buildbot.pypy.org  Fri Jun 14 21:21:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 21:21:00 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Revert the gcflag_xyz() functions;
 instead found a way to keep the
Message-ID: <20130614192100.A9DDF1C0134@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r129:0fdcdfbbc387
Date: 2013-06-14 20:20 +0200
http://bitbucket.org/pypy/stmgc/changeset/0fdcdfbbc387/

Log:	Revert the gcflag_xyz() functions; instead found a way to keep the
	assertion error message readable, by using "static const" values.

	More progress.

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -22,6 +22,7 @@
     intptr_t align = ((intptr_t)p) & (PAGE_SIZE-1);
     p = ((char *)p) - align;
     sz += align;
+    fprintf(stderr, "dbgmem: %p, %ld, %d\n", p, (long)sz, prot);
     int err = mprotect(p, sz, prot);
     assert(err == 0);
 }
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -33,7 +33,7 @@
 static int is_private(gcptr P)
 {
   return (P->h_revision == stm_private_rev_num) ||
-            gcflag_private_from_protected(P);
+    (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
 }
 int _stm_is_private(gcptr P)
 {
@@ -80,7 +80,7 @@
   revision_t v;
 
  restart_all:
-  if (gcflag_private_from_protected(P))
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
       assert(!(P->h_revision & 1));   /* pointer to the backup copy */
 
@@ -95,14 +95,14 @@
   /* else, for the rest of this function, we can assume that P was not
      a private copy */
 
-  if (gcflag_public(P))
+  if (P->h_tid & GCFLAG_PUBLIC)
     {
       /* follow the chained list of h_revision's as long as they are
          regular pointers.  We will only find more public objects
          along this chain.
       */
     restart_all_public:
-      assert(gcflag_public(P));
+      assert(P->h_tid & GCFLAG_PUBLIC);
       v = ACCESS_ONCE(P->h_revision);
       if (!(v & 1))  // "is a pointer", i.e.
         {            //      "has a more recent revision"
@@ -112,7 +112,7 @@
 
           gcptr P_prev = P;
           P = (gcptr)v;
-          assert(gcflag_public(P));
+          assert(P->h_tid & GCFLAG_PUBLIC);
 
           v = ACCESS_ONCE(P->h_revision);
 
@@ -145,7 +145,7 @@
          because *we* have an entry in d->public_to_private.  (It might
          also be someone else.)
       */
-      if (gcflag_public_to_private(P))
+      if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
         {
           wlog_t *item;
         retry_public_to_private:;
@@ -154,7 +154,7 @@
           /* We have a key in 'public_to_private'.  The value is the
              corresponding private object. */
           P = item->val;
-          assert(!gcflag_public(P));
+          assert(!(P->h_tid & GCFLAG_PUBLIC));
           assert(is_private(P));
           fprintf(stderr, "read_barrier: %p -> %p public_to_private\n", G, P);
           return P;
@@ -229,7 +229,7 @@
       fprintf(stderr, "read_barrier: %p -> stealing %p...\n  ", G, P);
       stm_steal_stub(P);
 
-      assert(gcflag_public(P));
+      assert(P->h_tid & GCFLAG_PUBLIC);
       goto restart_all_public;
     }
 }
@@ -245,8 +245,8 @@
   if (pubobj == P || ((P->h_revision & 3) == 2 &&
                       pubobj->h_revision == P->h_revision))
     {
-      assert(!gcflag_stub(org_pubobj));
-      assert(!gcflag_public(privobj));
+      assert(!(org_pubobj->h_tid & GCFLAG_STUB));
+      assert(!(privobj->h_tid & GCFLAG_PUBLIC));
       assert(is_private(privobj));
       if (P != org_pubobj)
         fprintf(stderr, "| actually %p ", org_pubobj);
@@ -299,7 +299,7 @@
       return P;
     }
 
-  if (gcflag_private_from_protected(P))
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
       /* private too, with a backup copy */
       assert(!(P->h_revision & 1));
@@ -307,7 +307,7 @@
       return P;
     }
 
-  if (gcflag_public(P))
+  if (P->h_tid & GCFLAG_PUBLIC)
     {
       fprintf(stderr, "public ");
 
@@ -323,7 +323,7 @@
             }
 
           P = (gcptr)v;
-          assert(gcflag_public(P));
+          assert(P->h_tid & GCFLAG_PUBLIC);
           fprintf(stderr, "-> %p public ", P);
         }
 
@@ -354,12 +354,12 @@
     {
       P = (gcptr)(v - 2);
       fprintf(stderr, "-foreign-> %p ", P);
-      if (gcflag_private_from_protected(P))
+      if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
         {
           P = (gcptr)P->h_revision;     /* the backup copy */
           fprintf(stderr, "-backup-> %p ", P);
         }
-      if (!gcflag_public(P))
+      if (!(P->h_tid & GCFLAG_PUBLIC))
         {
           fprintf(stderr, "protected by someone else!\n");
           return (gcptr)-1;
@@ -397,10 +397,10 @@
 
   assert(P->h_revision != stm_private_rev_num);
   assert(P->h_revision & 1);
-  assert(!gcflag_public_to_private(P));
-  assert(!gcflag_backup_copy(P));
-  assert(!gcflag_stub(P));
-  assert(!gcflag_private_from_protected(P));
+  assert(!(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
+  assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
+  assert(!(P->h_tid & GCFLAG_STUB));
+  assert(!(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
 
   B = stmgc_duplicate(P);
   B->h_tid |= GCFLAG_BACKUP_COPY;
@@ -415,7 +415,7 @@
 
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
-  assert(gcflag_public(R));
+  assert(R->h_tid & GCFLAG_PUBLIC);
 
 #ifdef _GC_DEBUG
   wlog_t *entry;
@@ -427,9 +427,9 @@
   R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
   gcptr L = stmgc_duplicate(R);
-  assert(!gcflag_backup_copy(L));
-  assert(!gcflag_stub(L));
-  assert(!gcflag_private_from_protected(L));
+  assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
+  assert(!(L->h_tid & GCFLAG_STUB));
+  assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
   L->h_tid &= ~(GCFLAG_OLD               |
                 GCFLAG_VISITED           |
                 GCFLAG_PUBLIC            |
@@ -451,10 +451,10 @@
 
 static inline void record_write_barrier(gcptr P)
 {
-  if (gcflag_write_barrier(P))
+  if (P->h_tid & GCFLAG_WRITE_BARRIER)
     {
       P->h_tid &= ~GCFLAG_WRITE_BARRIER;
-      gcptrlist_insert(&thread_descriptor->old_with_young_pointers_inside, P);
+      gcptrlist_insert(&thread_descriptor->old_objects_to_trace, P);
     }
 }
 
@@ -463,9 +463,9 @@
   if (is_private(P))
     {
       /* If we have GCFLAG_WRITE_BARRIER in P, then list it into
-         old_with_young_pointers_inside: it's a private object that may
-         be modified by the program after we return, and the mutation
-         may be to write young pointers (in fact it's a common case).
+         old_objects_to_trace: it's a private object that may be
+         modified by the program after we return, and the mutation may
+         be to write young pointers (in fact it's a common case).
       */
       record_write_barrier(P);
       return P;
@@ -490,20 +490,20 @@
   if (d->public_descriptor->stolen_objects.size != 0)
     stm_normalize_stolen_objects(d);
 
-  if (gcflag_public(R))
+  if (R->h_tid & GCFLAG_PUBLIC)
     {
       /* Make and return a new (young) private copy of the public R.
-         Add R into the list 'old_public_with_young_copy'.
+         Add R into the list 'public_with_young_copy'.
       */
-      assert(gcflag_old(R));
-      gcptrlist_insert(&d->old_public_with_young_copy, R);
+      assert(R->h_tid & GCFLAG_OLD);
+      gcptrlist_insert(&d->public_with_young_copy, R);
       W = LocalizePublic(d, R);
     }
   else
     {
       /* Turn the protected copy in-place into a private copy.  If it's
          an old object that still has GCFLAG_WRITE_BARRIER, then we must
-         also record it in the list 'old_with_young_pointers_inside'. */
+         also record it in the list 'old_objects_to_trace'. */
       W = LocalizeProtected(d, R);
       record_write_barrier(W);
     }
@@ -553,7 +553,7 @@
       v = ACCESS_ONCE(R->h_revision);
       if (!(v & 1))               // "is a pointer", i.e.
         {                         //   "has a more recent revision"
-          if (gcflag_private_from_protected(R))
+          if (R->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
             {
               /* such an object R might be listed in list_of_read_objects
                  before it was turned from protected to private */
@@ -811,7 +811,7 @@
       gcptr R = item->addr;
       revision_t v;
     retry:
-      assert(gcflag_public(R));
+      assert(R->h_tid & GCFLAG_PUBLIC);
       v = ACCESS_ONCE(R->h_revision);
       if (!(v & 1))            // "is a pointer", i.e.
         {                      //   "has a more recent revision"
@@ -831,7 +831,7 @@
         goto retry;
 
       gcptr L = item->val;
-      assert(gcflag_private_from_protected(L) ?
+      assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED ?
              L->h_revision == (revision_t)R :
              L->h_revision == stm_private_rev_num);
       assert(v != stm_private_rev_num);
@@ -855,7 +855,7 @@
       gcptr L = item->val;
       revision_t expected, v = L->h_revision;
 
-      if (gcflag_private_from_protected(L))
+      if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
         expected = (revision_t)R;
       else
         expected = stm_private_rev_num;
@@ -892,10 +892,10 @@
   G2L_LOOP_FORWARD(d->public_to_private, item)
     {
       gcptr L = item->val;
-      assert(!gcflag_visited(L));
-      assert(!gcflag_public_to_private(L));
-      assert(!gcflag_prebuilt_original(L));
-      assert(!gcflag_nursery_moved(L));
+      assert(!(L->h_tid & GCFLAG_VISITED));
+      assert(!(L->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
+      assert(!(L->h_tid & GCFLAG_PREBUILT_ORIGINAL));
+      assert(!(L->h_tid & GCFLAG_NURSERY_MOVED));
       assert(L->h_revision != localrev);   /* modified by AcquireLocks() */
 
 #ifdef DUMP_EXTRA
@@ -919,9 +919,9 @@
       gcptr R = item->addr;
       revision_t v = (revision_t)item->val;
 
-      assert(gcflag_public(R));
-      assert(gcflag_public_to_private(R));
-      assert(!gcflag_nursery_moved(R));
+      assert(R->h_tid & GCFLAG_PUBLIC);
+      assert(R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
+      assert(!(R->h_tid & GCFLAG_NURSERY_MOVED));
       assert(R->h_revision != localrev);
 
 #ifdef DUMP_EXTRA
@@ -932,7 +932,7 @@
       ACCESS_ONCE(R->h_revision) = v;
 
 #if 0
-      if (gcflag_prebuilt_original(R))
+      if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
         {
           /* cannot possibly get here more than once for a given value of R */
           pthread_mutex_lock(&mutex_prebuilt_gcroots);
@@ -957,7 +957,7 @@
   for (i = 0; i < size; i++)
     {
       gcptr P = items[i];
-      assert(gcflag_private_from_protected(P));
+      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
       P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
 
       if (P->h_revision & 1)   // "is not a pointer"
@@ -972,7 +972,7 @@
       gcptr B = (gcptr)P->h_revision;
       P->h_revision = new_revision;
 
-      if (gcflag_public(B))
+      if (B->h_tid & GCFLAG_PUBLIC)
         {
           /* B was stolen */
           while (1)
@@ -987,7 +987,7 @@
         }
       else
         {
-          stm_free(B, stmcb_size(B));
+          //stm_free(B, stmcb_size(B));
         }
     };
   gcptrlist_clear(&d->private_from_protected);
@@ -1001,20 +1001,20 @@
   for (i = 0; i < size; i++)
     {
       gcptr P = items[i];
-      assert(gcflag_private_from_protected(P));
+      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
       assert(!(P->h_revision & 1));   // "is a pointer"
 
       gcptr B = (gcptr)P->h_revision;
-      if (gcflag_public(B))
+      if (B->h_tid & GCFLAG_PUBLIC)
         {
-          assert(!gcflag_backup_copy(B));
+          assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
           P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
           P->h_tid |= GCFLAG_PUBLIC;
           /* P becomes a public outdated object */
         }
       else
         {
-          assert(gcflag_backup_copy(B));
+          assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(P, B, stmcb_size(P));
           P->h_tid &= ~GCFLAG_BACKUP_COPY;
         }
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -44,7 +44,7 @@
  * objects (which may be turned private again).  It may be left set on
  * public objects but is ignored there, because such objects are read-only.
  * The flag is removed once a write occurs and the object is recorded in
- * the list 'old_pointing_to_young'; it is set again at the next minor
+ * the list 'old_objects_to_trace'; it is set again at the next minor
  * collection.
  *
  * GCFLAG_NURSERY_MOVED is used temporarily during minor collections.
@@ -54,16 +54,16 @@
  * that is == 2 (mod 4): in this case they point to a protected/private
  * object that belongs to the thread 'STUB_THREAD(p_stub)'.
  */
-#define GCFLAG_OLD               (STM_FIRST_GCFLAG << 0)
-#define GCFLAG_VISITED           (STM_FIRST_GCFLAG << 1)
-#define GCFLAG_PUBLIC            (STM_FIRST_GCFLAG << 2)
-#define GCFLAG_PREBUILT_ORIGINAL (STM_FIRST_GCFLAG << 3)
-#define GCFLAG_PUBLIC_TO_PRIVATE (STM_FIRST_GCFLAG << 4)
-#define GCFLAG_WRITE_BARRIER     (STM_FIRST_GCFLAG << 5)
-#define GCFLAG_NURSERY_MOVED     (STM_FIRST_GCFLAG << 6)
-#define GCFLAG_BACKUP_COPY       (STM_FIRST_GCFLAG << 7)   /* debugging */
-#define GCFLAG_STUB              (STM_FIRST_GCFLAG << 8)   /* debugging */
-#define GCFLAG_PRIVATE_FROM_PROTECTED (STM_FIRST_GCFLAG << 9)
+static const revision_t GCFLAG_OLD                    = STM_FIRST_GCFLAG << 0;
+static const revision_t GCFLAG_VISITED                = STM_FIRST_GCFLAG << 1;
+static const revision_t GCFLAG_PUBLIC                 = STM_FIRST_GCFLAG << 2;
+static const revision_t GCFLAG_PREBUILT_ORIGINAL      = STM_FIRST_GCFLAG << 3;
+static const revision_t GCFLAG_PUBLIC_TO_PRIVATE      = STM_FIRST_GCFLAG << 4;
+static const revision_t GCFLAG_WRITE_BARRIER          = STM_FIRST_GCFLAG << 5;
+static const revision_t GCFLAG_NURSERY_MOVED          = STM_FIRST_GCFLAG << 6;
+static const revision_t GCFLAG_BACKUP_COPY  /*debug*/ = STM_FIRST_GCFLAG << 7;
+static const revision_t GCFLAG_STUB         /*debug*/ = STM_FIRST_GCFLAG << 8;
+static const revision_t GCFLAG_PRIVATE_FROM_PROTECTED = STM_FIRST_GCFLAG << 9;
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
@@ -83,20 +83,6 @@
                          "PRIVATE_FROM_PROTECTED", \
                          NULL }
 
-#define _DECLARE_FLAG(funcname, flagname)       \
-    static inline _Bool funcname(gcptr P) {     \
-        return (P->h_tid & flagname) != 0; }
-_DECLARE_FLAG(gcflag_old,                    GCFLAG_OLD)
-_DECLARE_FLAG(gcflag_visited,                GCFLAG_VISITED)
-_DECLARE_FLAG(gcflag_public,                 GCFLAG_PUBLIC)
-_DECLARE_FLAG(gcflag_prebuilt_original,      GCFLAG_PREBUILT_ORIGINAL)
-_DECLARE_FLAG(gcflag_public_to_private,      GCFLAG_PUBLIC_TO_PRIVATE)
-_DECLARE_FLAG(gcflag_write_barrier,          GCFLAG_WRITE_BARRIER)
-_DECLARE_FLAG(gcflag_nursery_moved,          GCFLAG_NURSERY_MOVED)
-_DECLARE_FLAG(gcflag_backup_copy,            GCFLAG_BACKUP_COPY)
-_DECLARE_FLAG(gcflag_stub,                   GCFLAG_STUB)
-_DECLARE_FLAG(gcflag_private_from_protected, GCFLAG_PRIVATE_FROM_PROTECTED)
-
 /************************************************************/
 
 #define ABRT_MANUAL               0
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -26,8 +26,7 @@
     stm_free(d->nursery_base, GC_NURSERY);
 
     gcptrlist_delete(&d->old_objects_to_trace);
-    gcptrlist_delete(&d->public_to_young);
-    gcptrlist_delete(&d->private_old_pointing_to_young);
+    gcptrlist_delete(&d->public_with_young_copy);
 }
 
 static char *collect_and_allocate_size(size_t size);  /* forward */
@@ -67,11 +66,11 @@
 
 static inline gcptr create_old_object_copy(gcptr obj)
 {
-    assert(!gcflag_nursery_moved(obj));
-    assert(!gcflag_visited(obj));
-    assert(!gcflag_write_barrier(obj));
-    assert(!gcflag_prebuilt_original(obj));
-    assert(!gcflag_old(obj));
+    assert(!(obj->h_tid & GCFLAG_NURSERY_MOVED));
+    assert(!(obj->h_tid & GCFLAG_VISITED));
+    assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
+    assert(!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
+    assert(!(obj->h_tid & GCFLAG_OLD));
 
     size_t size = stmcb_size(obj);
     gcptr fresh_old_copy = stm_malloc(size);
@@ -119,27 +118,27 @@
 
 static void mark_public_to_young(struct tx_descriptor *d)
 {
-    /* "public_to_young" contains ptrs to the public copies used as
-       key of "public_to_private", but only the ones that were added
-       since the last minor collection.  Once the transaction commit,
-       they stay in "public_to_young", and so they become public
-       objects whose h_revision is a public stub, which itself points
-       (originally) to a protected young object.
+    /* "public_with_young_copy" lists the public copies that may have
+       a more recent (or in-progress) private or protected object that
+       is young.  Note that public copies themselves are always old.
 
-       Be careful and accept more or less any object in the list, which
-       can show up because of aborted transactions.
+       The list should only contain public objects, but beyong that, be
+       careful and ignore any strange object: it can show up because of
+       aborted transactions (and then some different changes).
     */
-    long i, size = d->public_to_young.size;
-    gcptr *items = d->public_to_young.items;
+    long i, size = d->public_with_young_copy.size;
+    gcptr *items = d->public_with_young_copy.items;
 
     for (i = 0; i < size; i++) {
         gcptr P = items[i];
-        assert(gcflag_public(P));
+        assert(P->h_tid & GCFLAG_PUBLIC);
 
         revision_t v = ACCESS_ONCE(P->h_revision);
         wlog_t *item;
         G2L_FIND(d->public_to_private, P, item, goto not_in_public_to_private);
 
+        /* found P in 'public_to_private' */
+
         if (!(v & 1)) {   // "is a pointer"
             /* P is both a key in public_to_private and an outdated copy.
                We are in a case where we know the transaction will not
@@ -153,6 +152,7 @@
 
         fprintf(stderr, "public_to_young: %p -> %p in public_to_private\n",
                 item->addr, item->val);
+        assert(_stm_is_private(item->val));
         visit_if_young(&item->val);
         continue;
 
@@ -199,14 +199,7 @@
         S->h_revision = ((revision_t)L) | 2;
     }
 
-    gcptrlist_clear(&d->public_to_young);
-}
-
-static void mark_private_old_pointing_to_young(struct tx_descriptor *d)
-{
-    /* trace the objects recorded earlier by stmgc_write_barrier() */
-    gcptrlist_move(&d->old_objects_to_trace,
-                   &d->private_old_pointing_to_young);
+    gcptrlist_clear(&d->public_with_young_copy);
 }
 
 static void visit_all_outside_objects(struct tx_descriptor *d)
@@ -214,8 +207,8 @@
     while (gcptrlist_size(&d->old_objects_to_trace) > 0) {
         gcptr obj = gcptrlist_pop(&d->old_objects_to_trace);
 
-        assert(gcflag_old(obj));
-        assert(!gcflag_write_barrier(obj));
+        assert(obj->h_tid & GCFLAG_OLD);
+        assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
         obj->h_tid |= GCFLAG_WRITE_BARRIER;
 
         stmcb_trace(obj, &visit_if_young);
@@ -225,8 +218,6 @@
 static void setup_minor_collect(struct tx_descriptor *d)
 {
     spinlock_acquire(d->public_descriptor->collection_lock, 'M');  /*minor*/
-    assert(gcptrlist_size(&d->old_objects_to_trace) == 0);
-
     if (d->public_descriptor->stolen_objects.size != 0)
         stm_normalize_stolen_objects(d);
 }
@@ -234,6 +225,7 @@
 static void teardown_minor_collect(struct tx_descriptor *d)
 {
     assert(gcptrlist_size(&d->old_objects_to_trace) == 0);
+    assert(gcptrlist_size(&d->public_with_young_copy) == 0);
     assert(gcptrlist_size(&d->public_descriptor->stolen_objects) == 0);
 
     spinlock_release(d->public_descriptor->collection_lock);
@@ -255,8 +247,6 @@
 
     mark_public_to_young(d);
 
-    mark_private_old_pointing_to_young(d);
-
     visit_all_outside_objects(d);
 #if 0
     fix_list_of_read_objects(d);
@@ -292,8 +282,7 @@
     if (d->nursery_current == d->nursery_base /*&&
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
-        assert(gcptrlist_size(&d->private_old_pointing_to_young) == 0);
-        assert(gcptrlist_size(&d->public_to_young) == 0);
+        assert(gcptrlist_size(&d->public_with_young_copy) == 0);
         return 0;
     }
     else {
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -11,8 +11,7 @@
     char *nursery_end;                                  \
     char *nursery_base;                                 \
     struct GcPtrList old_objects_to_trace;              \
-    struct GcPtrList old_public_with_young_copy;        \
-    struct GcPtrList old_with_young_pointers_inside;
+    struct GcPtrList public_with_young_copy;
 
 struct tx_descriptor;  /* from et.h */
 
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -55,7 +55,7 @@
 static void replace_ptr_to_protected_with_stub(gcptr *pobj)
 {
     gcptr stub, obj = *pobj;
-    if (obj == NULL || gcflag_public(obj))
+    if (obj == NULL || (obj->h_tid & GCFLAG_PUBLIC) != 0)
         return;
 
     /* we use 'all_stubs', a dictionary, in order to try to avoid
@@ -98,7 +98,7 @@
     /* L might be a private_from_protected, or just a protected copy.
        To know which case it is, read GCFLAG_PRIVATE_FROM_PROTECTED.
     */
-    if (gcflag_private_from_protected(L)) {
+    if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
 
         /* B is now a backup copy, i.e. a protected object, and we own
@@ -107,9 +107,9 @@
         */
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
 
-        if (gcflag_public_to_private(B)) {
+        if (B->h_tid & GCFLAG_PUBLIC_TO_PRIVATE) {
             /* already stolen */
-            assert(gcflag_public(B));
+            assert(B->h_tid & GCFLAG_PUBLIC);
             fprintf(stderr, "already stolen: %p -> %p <-> %p\n", P, L, B);
             L = B;
             goto already_stolen;
@@ -125,7 +125,7 @@
         }
     }
     else {
-        if (gcflag_public(L)) {
+        if (L->h_tid & GCFLAG_PUBLIC) {
             /* already stolen */
             fprintf(stderr, "already stolen: %p -> %p\n", P, L);
             goto already_stolen;
@@ -139,7 +139,7 @@
        thread's collection_lock, so we can read/write the flags.  Change
        it from protected to public.
     */
-    assert(!gcflag_public(L));
+    assert(!(L->h_tid & GCFLAG_PUBLIC));
     L->h_tid |= GCFLAG_PUBLIC;
 
     /* Note that all protected or backup copies have a h_revision that
@@ -192,8 +192,8 @@
         gcptr B = items[i];
         gcptr L = items[i + 1];
 
-        assert(gcflag_private_from_protected(L));
-        assert(!gcflag_backup_copy(B));  /* already removed */
+        assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
 
         g2l_insert(&d->public_to_private, B, L);
 
@@ -218,7 +218,7 @@
         gcptr B = items[i];
         gcptr L = items[i + 1];
 
-        assert(gcflag_private_from_protected(L));
+        assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
         if (B == obj)
             return L;
     }
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -502,14 +502,16 @@
     lib.AbortTransaction(lib.ABRT_MANUAL)
 
 def classify(p):
-    private = (p.h_revision == lib.get_private_rev_num() or
-               (p.h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) != 0)
+    private_from_protected = (p.h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) != 0
+    private_other = p.h_revision == lib.get_private_rev_num()
     public  = (p.h_tid & GCFLAG_PUBLIC) != 0
     backup  = (p.h_tid & GCFLAG_BACKUP_COPY) != 0
     stub    = (p.h_tid & GCFLAG_STUB) != 0
-    assert private + public + backup <= 1
+    assert private_from_protected + private_other + public + backup <= 1
     assert (public, stub) != (False, True)
-    if private:
+    if private_from_protected:
+        return "private_from_protected"
+    if private_other:
         return "private"
     if public:
         if stub:
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -52,7 +52,7 @@
     assert classify(p) == "protected"
     p2 = lib.stm_write_barrier(p)
     assert p2 == p       # does not move
-    assert classify(p) == "private"
+    assert classify(p) == "private_from_protected"
     pback = follow_revision(p)
     assert classify(pback) == "backup"
     assert list_of_private_from_protected() == [p]
@@ -65,7 +65,7 @@
     org_r = p.h_revision
     assert classify(p) == "protected"
     lib.setlong(p, 0, 927122)
-    assert classify(p) == "private"
+    assert classify(p) == "private_from_protected"
     pback = follow_revision(p)
     assert pback and pback != p
     assert pback.h_revision == org_r
@@ -73,7 +73,7 @@
                            GCFLAG_BACKUP_COPY)
     assert lib.rawgetlong(pback, 0) == 78927812
     assert lib.rawgetlong(p, 0) == 927122
-    assert classify(p) == "private"
+    assert classify(p) == "private_from_protected"
     assert classify(pback) == "backup"
 
 def test_prebuilt_is_public():
@@ -263,13 +263,13 @@
             lib.rawsetlong(p2, 0, -451112)
             pback = follow_revision(p1)
             pback_.append(pback)
-            assert classify(p1) == "private"
+            assert classify(p1) == "private_from_protected"
             assert classify(pback) == "backup"
             assert lib.stm_read_barrier(p) == p1
             assert lib.stm_read_barrier(p1) == p1
             assert pback.h_revision & 1
             r.wait_while_in_parallel()
-            assert classify(p1) == "private"
+            assert classify(p1) == "private_from_protected"
             assert classify(pback) == "public"
             assert pback.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
             assert lib.stm_read_barrier(p) == p1
@@ -322,7 +322,7 @@
         if c == 0:
             lib.setlong(p, 0, -38383)
             assert lib.getlong(p, 0) == -38383
-            assert classify(p) == "private"
+            assert classify(p) == "private_from_protected"
             abort_and_retry()
     perform_transaction(cb)
 
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -168,18 +168,29 @@
     assert classify(p0) == "protected"
     assert lib.rawgetlong(p0, 0) == 81211
 
+def test_old_private_from_protected():
+    p0 = oalloc(HDR + WORD)
+    assert classify(p0) == "protected"
+    lib.setlong(p0, 0, 29820298)
+    assert classify(p0) == "private_from_protected"
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert classify(p0) == "protected"
+    assert lib.getlong(p0, 0) == 29820298
+    assert classify(p0) == "protected"
+
 def test_old_private_from_protected_to_young_private():
     p0 = oalloc_refs(1)
     assert classify(p0) == "protected"
     p1 = nalloc(HDR)
     lib.setptr(p0, 0, p1)
-    assert classify(p0) == "private"   # private_from_protected
+    assert classify(p0) == "private_from_protected"
     lib.stm_push_root(p0)
     minor_collect()
     p0b = lib.stm_pop_root()
     assert p0b == p0
     check_nursery_free(p1)
-    assert classify(p0) == "private"   # private_from_protected
+    assert classify(p0) == "private_from_protected"
     p2 = lib.getptr(p0, 0)
     assert not lib.in_nursery(p2)
     check_not_free(p2)

From noreply at buildbot.pypy.org  Fri Jun 14 21:21:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 21:21:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Test passes
Message-ID: <20130614192101.CED5D1C0134@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r130:aa0f0a2a6f74
Date: 2013-06-14 20:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/aa0f0a2a6f74/

Log:	Test passes

diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -195,3 +195,24 @@
     assert not lib.in_nursery(p2)
     check_not_free(p2)
     assert classify(p2) == "private"
+
+def test_new_version():
+    p1 = oalloc(HDR)
+    assert lib.stm_write_barrier(p1) == p1
+    lib.stm_push_root(p1)
+    transaction_break()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    p2 = lib.stm_write_barrier(p1)
+    assert p2 == p1
+    assert not lib.in_nursery(p2)
+    check_not_free(p1)
+    lib.stm_push_root(p1)
+    minor_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    check_not_free(p1)
+    p2 = lib.stm_read_barrier(p1)
+    assert p2 == p1
+    assert not lib.in_nursery(p2)
+    assert classify(p2) == "private_from_protected"

From noreply at buildbot.pypy.org  Fri Jun 14 21:21:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 21:21:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Test and bug fix
Message-ID: <20130614192102.EDD831C0134@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r131:69b1046ea32d
Date: 2013-06-14 20:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/69b1046ea32d/

Log:	Test and bug fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -476,8 +476,8 @@
 
   if (is_private(R))
     {
-      record_write_barrier(P);
-      return P;
+      record_write_barrier(R);
+      return R;
     }
 
   struct tx_descriptor *d = thread_descriptor;
@@ -498,6 +498,7 @@
       assert(R->h_tid & GCFLAG_OLD);
       gcptrlist_insert(&d->public_with_young_copy, R);
       W = LocalizePublic(d, R);
+      assert(is_private(W));
     }
   else
     {
@@ -505,6 +506,7 @@
          an old object that still has GCFLAG_WRITE_BARRIER, then we must
          also record it in the list 'old_objects_to_trace'. */
       W = LocalizeProtected(d, R);
+      assert(is_private(W));
       record_write_barrier(W);
     }
 
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -216,3 +216,19 @@
     assert p2 == p1
     assert not lib.in_nursery(p2)
     assert classify(p2) == "private_from_protected"
+
+def test_prebuilt_version():
+    p1 = lib.pseudoprebuilt(HDR, 42 + HDR)
+    p2 = lib.stm_write_barrier(p1)
+    assert p2 != p1
+    check_prebuilt(p1)
+    check_not_free(p2)
+    minor_collect()
+    check_prebuilt(p1)
+    check_nursery_free(p2)
+    p2 = lib.stm_read_barrier(p1)
+    assert classify(p2) == "private"
+    p3 = lib.stm_write_barrier(p1)
+    assert classify(p3) == "private"
+    assert p3 == p2 != p1
+    assert not lib.in_nursery(p2)

From noreply at buildbot.pypy.org  Fri Jun 14 21:21:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Jun 2013 21:21:04 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix test
Message-ID: <20130614192104.0D9001C0134@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r132:c46d7e01ec15
Date: 2013-06-14 20:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/c46d7e01ec15/

Log:	Fix test

diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -189,7 +189,7 @@
     p2b = lib.stm_write_barrier(p)
     assert p2b == p2
     assert classify(p) == "public"
-    assert classify(p2) == "private"
+    assert classify(p2) == "private_from_protected"
     assert list_of_read_objects() == [p2]
     p3 = lib.stm_read_barrier(p)
     assert p3 == p2

From noreply at buildbot.pypy.org  Sat Jun 15 07:33:17 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 15 Jun 2013 07:33:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a jit_merge_point to list.find
Message-ID: <20130615053317.7CEA11C02BA@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r64890:87bfc8884702
Date: 2013-06-15 13:32 +0800
http://bitbucket.org/pypy/pypy/changeset/87bfc8884702/

Log:	Add a jit_merge_point to list.find

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -693,6 +693,7 @@
             raise OperationError(space.w_ValueError,
                                  space.wrap("list modified during sort"))
 
+find_jmp = jit.JitDriver(greens = [], reds = 'auto', name = 'list.find')
 
 class ListStrategy(object):
     sizehint = -1
@@ -717,6 +718,7 @@
         i = start
         # needs to be safe against eq_w mutating stuff
         while i < stop and i < w_list.length():
+            find_jmp.jit_merge_point()
             if space.eq_w(w_list.getitem(i), w_item):
                 return i
             i += 1

From noreply at buildbot.pypy.org  Sat Jun 15 13:36:29 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sat, 15 Jun 2013 13:36:29 +0200 (CEST)
Subject: [pypy-commit] buildbot buildbot-0.8.7: use got_revision here
Message-ID: <20130615113629.843391C1398@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: buildbot-0.8.7
Changeset: r826:d0caac6ed8bf
Date: 2013-06-15 13:36 +0200
http://bitbucket.org/pypy/buildbot/changeset/d0caac6ed8bf/

Log:	use got_revision here

diff --git a/bot2/pypybuildbot/test/test_builds.py b/bot2/pypybuildbot/test/test_builds.py
--- a/bot2/pypybuildbot/test/test_builds.py
+++ b/bot2/pypybuildbot/test/test_builds.py
@@ -62,7 +62,7 @@
 def test_pypy_upload():
     pth = py.test.ensuretemp('buildbot')
     inst = builds.PyPyUpload(slavesrc='slavesrc', masterdest=str(pth.join('mstr')),
-                             basename='base-%(final_file_name)s', workdir='.',
+                             basename='base-%(got_revision)s', workdir='.',
                              blocksize=100)
     factory = inst._getStepFactory().factory
     kw = inst._getStepFactory().kwargs
@@ -73,7 +73,7 @@
     rebuilt.start()
     assert pth.join('mstr').check(dir=True)
     assert rebuilt.masterdest == str(pth.join('mstr', 'trunk',
-                                              'base-123-ea5ca8'))
+                                              'base-123'))
     assert rebuilt.symlinkname == str(pth.join('mstr', 'trunk',
                                                'base-latest'))
 

From noreply at buildbot.pypy.org  Sat Jun 15 13:36:28 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sat, 15 Jun 2013 13:36:28 +0200 (CEST)
Subject: [pypy-commit] buildbot buildbot-0.8.7: merge default
Message-ID: <20130615113628.4E7DB1C0F1D@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: buildbot-0.8.7
Changeset: r825:41d58f6cac9e
Date: 2013-06-15 13:07 +0200
http://bitbucket.org/pypy/buildbot/changeset/41d58f6cac9e/

Log:	merge default

diff --git a/bbhook/hook.py b/bbhook/hook.py
--- a/bbhook/hook.py
+++ b/bbhook/hook.py
@@ -3,6 +3,8 @@
 import subprocess
 import sys
 import time
+import thread, Queue
+import traceback
 
 from .main import app
 from . import scm
@@ -39,7 +41,24 @@
         yield commit
 
 
-def handle(payload, test=False):
+
+def _handle_thread():
+    while True:
+        local_repo = payload = None
+        try:
+            local_repo, payload = queue.get()
+            _do_handle(local_repo, payload)
+        except:
+            traceback.print_exc()
+            print >> sys.stderr, 'payload:'
+            pprint.pprint(payload, sys.stderr)
+            print >> sys.stderr
+
+queue = Queue.Queue()
+thread.start_new_thread(_handle_thread, ())
+
+
+def handle(payload, test=True):
     path = payload['repository']['absolute_url']
     owner = payload['repository']['owner']
     local_repo = app.config['LOCAL_REPOS'].join(path)
@@ -47,6 +66,12 @@
     if not check_for_local_repo(local_repo, remote_repo, owner):
         print >> sys.stderr, 'Ignoring unknown repo', path
         return
+    if test:
+        _do_handle(local_repo, payload, test)
+    else:
+        queue.put((local_repo, payload))
+
+def _do_handle(local_repo, payload, test=False):
     scm.hg('pull', '-R', local_repo)
     for commit in get_commits(payload):
         for handler in HANDLERS:
diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py
--- a/bot2/pypybuildbot/arm_master.py
+++ b/bot2/pypybuildbot/arm_master.py
@@ -23,8 +23,7 @@
     translationArgs=crosstranslationargs + ['-O2'],
     platform='linux-armel',
     interpreter='pypy',
-    prefix=['schroot', '-c', 'armel'],
-    trigger='APPLVLLINUXARM_scheduler')
+    prefix=['schroot', '-c', 'armel'])
 
 pypyJITCrossTranslationFactoryARM = pypybuilds.NightlyBuild(
     translationArgs=(crosstranslationargs
@@ -51,6 +50,14 @@
     prefix=['schroot', '-c', 'raspbian'],
     trigger='JITLINUXARMHF_RASPBIAN_scheduler')
 
+pypyJITCrossTranslationFactoryRaringHF = pypybuilds.NightlyBuild(
+    translationArgs=(crosstranslationargs
+                        + jit_translation_args
+                        + crosstranslationjitargs),
+    platform='linux-armhf-raring',
+    interpreter='pypy',
+    prefix=['schroot', '-c', 'raring'])
+
 pypyARMJITTranslatedTestFactory = pypybuilds.TranslatedTests(
     translationArgs=(crosstranslationargs
                         + jit_translation_args
@@ -100,6 +107,7 @@
 BUILDJITLINUXARM = "build-pypy-c-jit-linux-armel"
 BUILDLINUXARMHF_RASPBIAN = "build-pypy-c-linux-armhf-raspbian"
 BUILDJITLINUXARMHF_RASPBIAN = "build-pypy-c-jit-linux-armhf-raspbian"
+BUILDJITLINUXARMHF_RARING = "build-pypy-c-jit-linux-armhf-raring"
 
 builderNames = [
     APPLVLLINUXARM,
@@ -121,12 +129,13 @@
     Nightly("nighly-arm-0-00", [
         BUILDJITLINUXARM,              # on hhu-cross-armel, uses 1 core
         BUILDJITLINUXARMHF_RASPBIAN,   # on hhu-cross-raspbianhf, uses 1 core
+        BUILDJITLINUXARMHF_RARING,     # on hhu-cross-raring-armhf, uses 1 core
 
         BUILDLINUXARM,                 # on hhu-cross-armel, uses 1 core
         BUILDLINUXARMHF_RASPBIAN,      # on hhu-cross-raspbianhf, uses 1 core
 
         JITBACKENDONLYLINUXARMEL,      # on hhu-imx.53
-        JITBACKENDONLYLINUXARMHF,      # on hhu-raspberry-pi
+        JITBACKENDONLYLINUXARMHF,
         JITBACKENDONLYLINUXARMHF_v7,   # on cubieboard-bob
         ], branch=None, hour=0, minute=0),
 
@@ -138,12 +147,12 @@
         JITLINUXARM,               # triggered by BUILDJITLINUXARM, on hhu-beagleboard
     ]),
     Triggerable("APPLVLLINUXARMHF_RASPBIAN_scheduler", [
-        APPLVLLINUXARMHF_RASPBIAN,  # triggered by BUILDLINUXARMHF_RASPBIAN, on hhu-raspberry-pi
+        APPLVLLINUXARMHF_RASPBIAN,  # triggered by BUILDLINUXARMHF_RASPBIAN
         APPLVLLINUXARMHF_v7,        # triggered by BUILDLINUXARMHF_RASPBIAN, on cubieboard-bob
     ]),
 
     Triggerable("JITLINUXARMHF_RASPBIAN_scheduler", [
-        JITLINUXARMHF_RASPBIAN,       # triggered by BUILDJITLINUXARMHF_RASPBIAN, on hhu-raspberry-pi
+        JITLINUXARMHF_RASPBIAN,       # triggered by BUILDJITLINUXARMHF_RASPBIAN
         JITLINUXARMHF_v7,             # triggered by BUILDJITLINUXARMHF_RASPBIAN, on cubieboard-bob
     ]),
 ]
@@ -162,7 +171,7 @@
   # armhf
   ## armv6
   {"name": JITBACKENDONLYLINUXARMHF,
-   "slavenames": ['hhu-raspberry-pi'],
+   "slavenames": ['hhu-raspberry-pi', 'hhu-pypy-pi', 'hhu-pypy-pi2'],
    "builddir": JITBACKENDONLYLINUXARMHF,
    "factory": pypyJitBackendOnlyOwnTestFactoryARM,
    "category": 'linux-armhf',
@@ -194,14 +203,14 @@
    },
   ## armv6 hardfloat
   {"name": APPLVLLINUXARMHF_RASPBIAN,
-   "slavenames": ["hhu-raspberry-pi"],
+   "slavenames": ['hhu-raspberry-pi', 'hhu-pypy-pi', 'hhu-pypy-pi2'],
    "builddir": APPLVLLINUXARMHF_RASPBIAN,
    "factory": pypyARMHF_RASPBIAN_TranslatedAppLevelTestFactory,
    "category": "linux-armhf",
    "locks": [ARMBoardLock.access('counting')],
    },
   {"name": JITLINUXARMHF_RASPBIAN,
-   "slavenames": ["hhu-raspberry-pi"],
+   "slavenames": ['hhu-raspberry-pi', 'hhu-pypy-pi', 'hhu-pypy-pi2'],
    'builddir': JITLINUXARMHF_RASPBIAN,
    'factory': pypyARMHF_RASPBIAN_JITTranslatedTestFactory,
    'category': 'linux-armhf',
@@ -251,4 +260,11 @@
    "category": 'linux-armhf',
    "locks": [ARMCrossLock.access('counting')],
   },
+  {"name": BUILDJITLINUXARMHF_RARING,
+   "slavenames": ['hhu-cross-raring'],
+   "builddir": BUILDJITLINUXARMHF_RARING,
+   "factory": pypyJITCrossTranslationFactoryRaringHF,
+   "category": 'linux-armhf',
+   "locks": [ARMCrossLock.access('counting')],
+  },
 ]
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -181,9 +181,12 @@
 CPYTHON_64 = "cpython-2-benchmark-x86-64"
 
 
-extra_opts= {'xerxes': {'keepalive_interval': 15},
+extra_opts = {'xerxes': {'keepalive_interval': 15},
              'aurora': {'max_builds': 1},
              'salsa': {'max_builds': 1},
+             'hhu-raspberry-pi': {'max_builds': 1},
+             'hhu-pypy-pi': {'max_builds': 1},
+             'hhu-pypy-pi2': {'max_builds': 1},
              }
 
 BuildmasterConfig = {
@@ -204,7 +207,7 @@
             APPLVLLINUX32,             # on tannit32, uses 1 core
             APPLVLLINUX64,             # on allegro64, uses 1 core
             # other platforms
-            MACOSX32,                  # on minime
+            #MACOSX32,                 # on minime
             JITWIN32,                  # on aurora
             JITFREEBSD764,             # on headless
             JITFREEBSD864,             # on ananke
diff --git a/bot2/pypybuildbot/pypylist.py b/bot2/pypybuildbot/pypylist.py
--- a/bot2/pypybuildbot/pypylist.py
+++ b/bot2/pypybuildbot/pypylist.py
@@ -28,6 +28,8 @@
         'linux64':   50,
         'osx':       30,
         'win32':     10,
+        'linux_armhf_raspbian': 7,
+        'linux_armhf_raring': 6,
         'linux_armel': 5,
         }
 
@@ -66,6 +68,8 @@
         name = self.filename.replace(ext, '')
         # remove the dash from linux-armel, else the split does not work
         name = name.replace('-armel', '_armel')
+        name = name.replace('-libc2', '_libc2')
+        name = name.replace('-armhf-ra', '_armhf_ra')
         dashes = name.count('-')
         if dashes == 4:
             # svn based
diff --git a/master/public_html/summary.css b/master/public_html/summary.css
--- a/master/public_html/summary.css
+++ b/master/public_html/summary.css
@@ -73,6 +73,10 @@
     border: 1px gray solid;
 }
 
+div.header {
+    margin-top: 15px;
+}
+
 div.footer {
     font-size: 80%;
 }
diff --git a/master/templates/summary.html b/master/templates/summary.html
--- a/master/templates/summary.html
+++ b/master/templates/summary.html
@@ -1,7 +1,7 @@
 {% extends "layout.html" %}
 
 {% block morehead %}
-<link rel="stylesheet" href="{{ path_to_root }}summary.css" type="text/css" />'
+<link rel="stylesheet" href="{{ path_to_root }}summary.css" type="text/css" />
 <script language=javascript type='text/javascript'>
     hiddenstates = [ ];
     function togglestate(a, c) {

From noreply at buildbot.pypy.org  Sat Jun 15 18:52:24 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 18:52:24 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130615165224.644441C0175@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r133:87748cb8c1c5
Date: 2013-06-15 11:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/87748cb8c1c5/

Log:	in-progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -82,12 +82,12 @@
  restart_all:
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
-      assert(!(P->h_revision & 1));   /* pointer to the backup copy */
+      assert(IS_POINTER(P->h_revision));   /* pointer to the backup copy */
 
       /* check P->h_revision->h_revision: if a pointer, then it means
          the backup copy has been stolen into a public object and then
          modified by some other thread.  Abort. */
-      if (!(((gcptr)P->h_revision)->h_revision & 1))
+      if (IS_POINTER(((gcptr)P->h_revision)->h_revision))
         AbortTransaction(ABRT_STOLEN_MODIFIED);
 
       goto add_in_recent_reads_cache;
@@ -104,8 +104,8 @@
     restart_all_public:
       assert(P->h_tid & GCFLAG_PUBLIC);
       v = ACCESS_ONCE(P->h_revision);
-      if (!(v & 1))  // "is a pointer", i.e.
-        {            //      "has a more recent revision"
+      if (IS_POINTER(v))  /* "is a pointer", "has a more recent revision" */
+        {
         retry:
           if (v & 2)
             goto follow_stub;
@@ -116,8 +116,8 @@
 
           v = ACCESS_ONCE(P->h_revision);
 
-          if (!(v & 1))  // "is a pointer", i.e.
-            {            //      "has a more recent revision"
+          if (IS_POINTER(v))
+            {
               if (v & 2)
                 goto follow_stub;
 
@@ -127,7 +127,7 @@
               P_prev->h_revision = v;
               P = (gcptr)v;
               v = ACCESS_ONCE(P->h_revision);
-              if (!(v & 1))  // "is a pointer", i.e. "has a more recent rev"
+              if (IS_POINTER(v))
                 goto retry;
             }
 
@@ -302,7 +302,7 @@
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
       /* private too, with a backup copy */
-      assert(!(P->h_revision & 1));
+      assert(IS_POINTER(P->h_revision));
       fprintf(stderr, "private_from_protected\n");
       return P;
     }
@@ -311,7 +311,7 @@
     {
       fprintf(stderr, "public ");
 
-      while (v = P->h_revision, !(v & 1))
+      while (v = P->h_revision, IS_POINTER(v))
         {
           if (v & 2)
             {
@@ -553,8 +553,8 @@
       revision_t v;
     retry:
       v = ACCESS_ONCE(R->h_revision);
-      if (!(v & 1))               // "is a pointer", i.e.
-        {                         //   "has a more recent revision"
+      if (IS_POINTER(v))  /* "is a pointer", i.e. has a more recent revision */
+        {
           if (R->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
             {
               /* such an object R might be listed in list_of_read_objects
@@ -815,8 +815,8 @@
     retry:
       assert(R->h_tid & GCFLAG_PUBLIC);
       v = ACCESS_ONCE(R->h_revision);
-      if (!(v & 1))            // "is a pointer", i.e.
-        {                      //   "has a more recent revision"
+      if (IS_POINTER(v))     /* "has a more recent revision" */
+        {
           assert(v != 0);
           AbortTransaction(ABRT_COMMIT);
         }
@@ -907,7 +907,7 @@
       L->h_revision = new_revision;
 
       gcptr stub = stm_stub_malloc(d->public_descriptor);
-      stub->h_tid = GCFLAG_PUBLIC | GCFLAG_STUB;
+      stub->h_tid = GCFLAG_PUBLIC | GCFLAG_STUB | GCFLAG_OLD;
       stub->h_revision = ((revision_t)L) | 2;
       item->val = stub;
 
@@ -962,7 +962,7 @@
       assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
       P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
 
-      if (P->h_revision & 1)   // "is not a pointer"
+      if (!IS_POINTER(P->h_revision))
         {
           /* This case occurs when a GCFLAG_PRIVATE_FROM_PROTECTED object
              is stolen: it ends up as a value in 'public_to_private'.
@@ -980,7 +980,7 @@
           while (1)
             {
               revision_t v = ACCESS_ONCE(B->h_revision);
-              if (!(v & 1))    // "is a pointer", i.e. "was modified"
+              if (IS_POINTER(v))    /* "was modified" */
                 AbortTransaction(ABRT_STOLEN_MODIFIED);
 
               if (bool_cas(&B->h_revision, v, (revision_t)P))
@@ -1004,14 +1004,14 @@
     {
       gcptr P = items[i];
       assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
-      assert(!(P->h_revision & 1));   // "is a pointer"
+      assert(IS_POINTER(P->h_revision));
 
       gcptr B = (gcptr)P->h_revision;
       if (B->h_tid & GCFLAG_PUBLIC)
         {
           assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
           P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
-          P->h_tid |= GCFLAG_PUBLIC;
+          P->h_tid |= GCFLAG_PUBLIC;      abort();
           /* P becomes a public outdated object */
         }
       else
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -83,6 +83,8 @@
                          "PRIVATE_FROM_PROTECTED", \
                          NULL }
 
+#define IS_POINTER(v)    (!((v) & 1))   /* even-valued number */
+
 /************************************************************/
 
 #define ABRT_MANUAL               0
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -139,7 +139,7 @@
 
         /* found P in 'public_to_private' */
 
-        if (!(v & 1)) {   // "is a pointer"
+        if (IS_POINTER(v)) {
             /* P is both a key in public_to_private and an outdated copy.
                We are in a case where we know the transaction will not
                be able to commit successfully.
@@ -157,7 +157,7 @@
         continue;
 
     not_in_public_to_private:
-        if (v & 1) {   // "is not a pointer"
+        if (!IS_POINTER(v)) {
             /* P is neither a key in public_to_private nor outdated.
                It must come from an older transaction that aborted.
                Nothing to do now.
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -74,7 +74,8 @@
  not_found:
     stub = stm_stub_malloc(sd->foreign_pd);
     stub->h_tid = (obj->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
-                                                   | GCFLAG_STUB;
+                                                   | GCFLAG_STUB
+                                                   | GCFLAG_OLD;
     stub->h_revision = ((revision_t)obj) | 2;
     g2l_insert(&sd->all_stubs, obj, stub);
 
@@ -156,6 +157,17 @@
        odd number that is also valid on a public up-to-date object.
     */
 
+    /* Move the object out of the other thread's nursery, if needed */
+    if (!(L->h_tid & GCFLAG_OLD)) {
+        size_t size = stmcb_size(L);
+        gcptr O = stm_malloc(size);
+        memcpy(O, L, size);
+        L->h_revision = (revision_t)O;
+        L->h_tid |= GCFLAG_NURSERY_MOVED;
+        O->h_tid |= GCFLAG_OLD;
+        L = O;
+    }
+
     /* Fix the content of the object: we need to change all pointers
        that reference protected copies into pointers that reference
        stub copies.
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -188,8 +188,18 @@
     int in_nursery(gcptr obj)
     {
         struct tx_descriptor *d = thread_descriptor;
-        return (d->nursery_base <= (char*)obj &&
-                ((char*)obj) < d->nursery_end);
+        int result1 = (d->nursery_base <= (char*)obj &&
+                       ((char*)obj) < d->nursery_end);
+        if (obj->h_tid & GCFLAG_OLD) {
+            assert(result1 == 0);
+        }
+        else {
+            /* this assert() also fails if "obj" is in another nursery than
+               the one of the current thread.  This is ok, because we
+               should not see such pointers. */
+            assert(result1 == 1);
+        }
+        return result1;
     }
 
     gcptr pseudoprebuilt(size_t size, int tid)
@@ -515,8 +525,15 @@
         return "private"
     if public:
         if stub:
+            assert not lib.in_nursery(p)
             return "stub"
-        return "public"
+        else:
+            # public objects usually never live in the nursery, but
+            # if stealing makes one, it has GCFLAG_NURSERY_MOVED.
+            if lib.in_nursery(p):
+                assert p.h_tid & GCFLAG_NURSERY_MOVED
+                assert not (p.h_revision & 1)   # "is a pointer"
+            return "public"
     if backup:
         return "backup"
     else:
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -198,7 +198,7 @@
     assert p4 == p2
     assert list_of_read_objects() == [p2]
 
-def test_stealing():
+def test_stealing_old():
     p = palloc(HDR + WORD)
     plist = [p]
     def f1(r):
@@ -207,6 +207,10 @@
         assert p1 != p
         assert classify(p) == "public"
         assert classify(p1) == "private"
+        minor_collect()
+        check_nursery_free(p1)
+        p1 = lib.stm_read_barrier(p)
+        assert p1.h_tid & GCFLAG_OLD
         assert p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
         lib.rawsetlong(p1, 0, 2782172)
         lib.stm_commit_transaction()
@@ -235,6 +239,51 @@
         r.set(3)
     run_parallel(f1, f2)
 
+def test_stealing_young():
+    p = palloc(HDR + WORD)
+    plist = [p]
+    def f1(r):
+        assert (p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE) == 0
+        p1 = lib.stm_write_barrier(p)   # private copy
+        assert p1 != p
+        assert classify(p) == "public"
+        assert classify(p1) == "private"
+        assert p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
+        lib.rawsetlong(p1, 0, 2782172)
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        assert classify(p) == "public"
+        assert classify(p1) == "protected"
+        plist.append(p1)     # now p's most recent revision is protected
+        assert classify(follow_revision(p)) == "stub"
+        assert p1.h_revision & 1
+        r.set(2)
+        r.wait(3)
+        assert classify(p1) == "public"
+        assert (p1.h_revision & 1) == 0    # outdated
+        p2 = ffi.cast("gcptr", p1.h_revision)
+        assert lib.in_nursery(p1)
+        assert not lib.in_nursery(p2)
+        assert lib.stm_read_barrier(p) == p2
+        assert lib.stm_read_barrier(p1) == p2
+        assert lib.stm_read_barrier(p2) == p2
+    def f2(r):
+        r.wait(2)
+        p2 = lib.stm_read_barrier(p)    # steals
+        assert classify(p2) == "public"
+        assert lib.rawgetlong(p2, 0) == 2782172
+        assert p2 == lib.stm_read_barrier(p)    # short-circuit h_revision
+        if SHORTCUT:
+            assert p.h_revision == int(ffi.cast("revision_t", p2))
+        assert p2 == lib.stm_read_barrier(p)
+        assert p2 != plist[-1]   # p2 is a public moved-out-of-nursery
+        assert plist[-1].h_tid & GCFLAG_PUBLIC
+        assert plist[-1].h_tid & GCFLAG_NURSERY_MOVED
+        assert plist[-1].h_revision == int(ffi.cast("revision_t", p2))
+        assert classify(p2) == "public"
+        r.set(3)
+    run_parallel(f1, f2)
+
 def test_stealing_while_modifying(aborting=False):
     p = palloc(HDR + WORD)
 
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -232,3 +232,32 @@
     assert classify(p3) == "private"
     assert p3 == p2 != p1
     assert not lib.in_nursery(p2)
+
+def _public_and_protected_in_nursery():
+    p1 = palloc(HDR + WORD)
+    lib.rawsetlong(p1, 0, 1000000)
+    p2 = lib.stm_write_barrier(p1)
+    assert lib.in_nursery(p2)
+    lib.setlong(p1, 0, 2000000)
+    assert lib.rawgetlong(p2, 0) == 2000000
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert classify(p1) == "public"
+    assert classify(p2) == "protected"
+    check_not_free(p2)
+    return p1, p2
+
+def test_public_not_in_nursery():
+    p1, p2 = _public_and_protected_in_nursery()
+    plist = []
+    def f1(_):
+        p3 = lib.stm_read_barrier(p1)
+        assert classify(p3) == "public"
+        assert not lib.in_nursery(p3)
+        assert p3 != p2    # not in-place, because p2 is in the nursery
+        plist.append(p3)
+    run_parallel(f1)
+    p3 = lib.stm_read_barrier(p1)
+    assert plist == [p3]
+    assert classify(p3) == "public"
+    assert not lib.in_nursery(p3)

From noreply at buildbot.pypy.org  Sat Jun 15 18:52:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 18:52:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Test fixes
Message-ID: <20130615165225.9BE7F1C029E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r134:1f6dfc74a907
Date: 2013-06-15 18:04 +0200
http://bitbucket.org/pypy/stmgc/changeset/1f6dfc74a907/

Log:	Test fixes

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -67,6 +67,8 @@
 
 void stm_free(void *p, size_t sz)
 {
+    assert(((intptr_t)((char *)p + sz) & (PAGE_SIZE-1)) == 0);
+
     size_t nb_pages = (sz + PAGE_SIZE - 1) / PAGE_SIZE + 1;
     long i, base = ((char *)p - zone_start) / PAGE_SIZE;
     assert(0 <= base && base < (MMAP_TOTAL / PAGE_SIZE));
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -402,7 +402,7 @@
   assert(!(P->h_tid & GCFLAG_STUB));
   assert(!(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
 
-  B = stmgc_duplicate(P);
+  B = stmgc_duplicate_old(P);
   B->h_tid |= GCFLAG_BACKUP_COPY;
 
   P->h_tid |= GCFLAG_PRIVATE_FROM_PROTECTED;
@@ -989,7 +989,7 @@
         }
       else
         {
-          //stm_free(B, stmcb_size(B));
+          stm_free(B, stmcb_size(B));
         }
     };
   gcptrlist_clear(&d->private_from_protected);
@@ -1011,8 +1011,8 @@
         {
           assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
           P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
-          P->h_tid |= GCFLAG_PUBLIC;      abort();
-          /* P becomes a public outdated object */
+          P->h_tid |= GCFLAG_PUBLIC;
+          /* P becomes a public (possibly young) outdated object */
         }
       else
         {
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -62,6 +62,15 @@
     return L;
 }
 
+gcptr stmgc_duplicate_old(gcptr P)
+{
+    size_t size = stmcb_size(P);
+    gcptr L = (gcptr)stm_malloc(size);
+    memcpy(L, P, size);
+    L->h_tid |= GCFLAG_OLD;
+    return L;
+}
+
 /************************************************************/
 
 static inline gcptr create_old_object_copy(gcptr obj)
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -20,5 +20,6 @@
 void stmgc_minor_collect(void);
 int stmgc_minor_collect_anything_to_do(struct tx_descriptor *);
 gcptr stmgc_duplicate(gcptr);
+gcptr stmgc_duplicate_old(gcptr);
 
 #endif
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -70,7 +70,7 @@
     assert pback and pback != p
     assert pback.h_revision == org_r
     assert pback.h_tid == ((p.h_tid & ~GCFLAG_PRIVATE_FROM_PROTECTED) |
-                           GCFLAG_BACKUP_COPY)
+                           GCFLAG_BACKUP_COPY | GCFLAG_OLD)
     assert lib.rawgetlong(pback, 0) == 78927812
     assert lib.rawgetlong(p, 0) == 927122
     assert classify(p) == "private_from_protected"
@@ -292,6 +292,10 @@
         assert classify(p) == "public"
         assert classify(p1) == "private"
         lib.rawsetlong(p1, 0, 2782172)
+        minor_collect()
+        check_nursery_free(p1)
+        p1 = lib.stm_read_barrier(p)
+        assert p1.h_tid & GCFLAG_OLD
         pback_ = []
 
         def cb(c):
@@ -378,17 +382,21 @@
 def test_abort_stealing_while_modifying():
     test_stealing_while_modifying(aborting=True)
 
-def test_stub_for_refs_from_stolen():
+def test_stub_for_refs_from_stolen(old=False):
     p = palloc_refs(1)
     qlist = []
     def f1(r):
+        q1 = nalloc(HDR + WORD)
+        if old:
+            lib.stm_push_root(q1)
+            minor_collect()
+            q1 = lib.stm_pop_root()
         assert (p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE) == 0
         p1 = lib.stm_write_barrier(p)   # private copy
         assert p1 != p
         assert classify(p) == "public"
         assert classify(p1) == "private"
         assert p.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
-        q1 = nalloc(HDR + WORD)
         qlist.append(q1)
         lib.setlong(q1, 0, -29187)
         lib.setptr(p1, 0, q1)
@@ -412,7 +420,11 @@
         assert q2.h_revision % 4 == 2
         q3 = lib.stm_read_barrier(q2)
         assert q3 != q2
-        assert q3 == qlist[0]
+        if old:
+            assert q3 == qlist[0]
         assert classify(q3) == "public"   # has been stolen
         assert lib.getlong(q3, 0) == -29187
     run_parallel(f1, f2)
+
+def test_stub_for_refs_from_stolen_old():
+    test_stub_for_refs_from_stolen(old=True)

From noreply at buildbot.pypy.org  Sat Jun 15 18:52:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 18:52:26 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tweaks tweaks.
Message-ID: <20130615165226.D859D1C02BA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r135:450808530645
Date: 2013-06-15 18:52 +0200
http://bitbucket.org/pypy/stmgc/changeset/450808530645/

Log:	Tweaks tweaks.

diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt
--- a/c4/doc-objects.txt
+++ b/c4/doc-objects.txt
@@ -270,3 +270,26 @@
 - public copies: must be changed carefully: `h_tid` is only modified to
 add GCFLAG_PUBLIC_TO_PRIVATE; and `h_revision` changes are done with
 bool_cas() in a thread-controlled way.
+
+
+
+Nursery and minor collections
+-----------------------------
+
+Every thread has its own nursery.  When it's full, the thread does a
+minor collection and proceed.
+
+Most object copies are allocated "young", i.e. from the nursery of its
+thread, with the following exceptions.
+
+Any public object (GCFLAG_PUBLIC) must be old.  This is a consequence of
+the fact that public objects may be accessed concurrently by any thead,
+but young objects are moved during a minor collection.  The rule admits
+one exception: a young object may be turned GCFLAG_PUBLIC during its
+existence (together with GCFLAG_NURSERY_MOVED) in order to get the
+behavior of public objects, but only if we know that no other thread
+will access it anyway.
+
+The backup copy of a GCFLAG_PRIVATE_FROM_PROTECTED copy is allocated
+old, and explicitly freed when the thread commits (unless it was
+stolen).
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1007,12 +1007,20 @@
       assert(IS_POINTER(P->h_revision));
 
       gcptr B = (gcptr)P->h_revision;
+      assert(B->h_tid & GCFLAG_OLD);
+
       if (B->h_tid & GCFLAG_PUBLIC)
         {
           assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
           P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
           P->h_tid |= GCFLAG_PUBLIC;
-          /* P becomes a public (possibly young) outdated object */
+          if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
+          /* P becomes a public outdated object.  It may create an
+             exception documented in doc-objects.txt: a public but young
+             object.  It's still fine because it should only be seen by
+             other threads during stealing, and as it's outdated,
+             stealing will follow its h_revision (to B).
+          */
         }
       else
         {
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -81,10 +81,7 @@
     assert(!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
     assert(!(obj->h_tid & GCFLAG_OLD));
 
-    size_t size = stmcb_size(obj);
-    gcptr fresh_old_copy = stm_malloc(size);
-    memcpy(fresh_old_copy, obj, size);
-    fresh_old_copy->h_tid |= GCFLAG_OLD;
+    gcptr fresh_old_copy = stmgc_duplicate_old(obj);
 
     fprintf(stderr, "minor: %p is copied to %p\n", obj, fresh_old_copy);
     return fresh_old_copy;
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -104,8 +104,9 @@
 
         /* B is now a backup copy, i.e. a protected object, and we own
            the foreign thread's collection_lock, so we can read/write the
-           flags
+           flags.  B is old, like all backup copies.
         */
+        assert(B->h_tid & GCFLAG_OLD);
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
 
         if (B->h_tid & GCFLAG_PUBLIC_TO_PRIVATE) {
@@ -129,11 +130,27 @@
         if (L->h_tid & GCFLAG_PUBLIC) {
             /* already stolen */
             fprintf(stderr, "already stolen: %p -> %p\n", P, L);
+
+            /* note that we should follow h_revision at least one more
+               step: it is necessary if L is public but young (and then
+               has GCFLAG_NURSERY_MOVED), but it is fine to do it more
+               generally. */
+            v = ACCESS_ONCE(L->h_revision);
+            if (IS_POINTER(v))
+                L = (gcptr)v;
             goto already_stolen;
         }
         else {
             fprintf(stderr, "stolen: %p -> %p\n", P, L);
         }
+
+        /* Copy the object out of the other thread's nursery, if needed */
+        if (!(L->h_tid & GCFLAG_OLD)) {
+            gcptr O = stmgc_duplicate_old(L);
+            L->h_revision = (revision_t)O;
+            L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
+            L = O;
+        }
     }
 
     /* Here L is a protected (or backup) copy, and we own the foreign
@@ -157,17 +174,6 @@
        odd number that is also valid on a public up-to-date object.
     */
 
-    /* Move the object out of the other thread's nursery, if needed */
-    if (!(L->h_tid & GCFLAG_OLD)) {
-        size_t size = stmcb_size(L);
-        gcptr O = stm_malloc(size);
-        memcpy(O, L, size);
-        L->h_revision = (revision_t)O;
-        L->h_tid |= GCFLAG_NURSERY_MOVED;
-        O->h_tid |= GCFLAG_OLD;
-        L = O;
-    }
-
     /* Fix the content of the object: we need to change all pointers
        that reference protected copies into pointers that reference
        stub copies.
@@ -188,6 +194,8 @@
     smp_wmb();
 
  already_stolen:
+    assert(L->h_tid & GCFLAG_OLD);
+
     /* update the original P->h_revision to point directly to L */
     P->h_revision = (revision_t)L;
 

From noreply at buildbot.pypy.org  Sat Jun 15 19:03:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 19:03:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next important behavior of minor
 collections (see test).
Message-ID: <20130615170341.2E2ED1C0175@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r136:ed12efb977c7
Date: 2013-06-15 19:03 +0200
http://bitbucket.org/pypy/stmgc/changeset/ed12efb977c7/

Log:	Next important behavior of minor collections (see test).

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -97,7 +97,14 @@
         /* not a nursery object */
     }
     else {
-        /* a nursery object */
+        /* it's a nursery object.  Was it already moved? */
+
+        if (UNLIKELY(obj->h_tid & GCFLAG_NURSERY_MOVED)) {
+            /* yes: just fix the ref. */
+            *root = (gcptr)obj->h_revision;
+            return;
+        }
+
         /* make a copy of it outside */
         fresh_old_copy = create_old_object_copy(obj);
         obj->h_tid |= GCFLAG_NURSERY_MOVED;
@@ -126,11 +133,12 @@
 {
     /* "public_with_young_copy" lists the public copies that may have
        a more recent (or in-progress) private or protected object that
-       is young.  Note that public copies themselves are always old.
+       is young.  Note that public copies themselves are always old
+       (short of a few exceptions that don't end up in this list).
 
-       The list should only contain public objects, but beyong that, be
-       careful and ignore any strange object: it can show up because of
-       aborted transactions (and then some different changes).
+       The list should only contain old public objects, but beyong that,
+       be careful and ignore any strange object: it can show up because
+       of aborted transactions (and then some different changes).
     */
     long i, size = d->public_with_young_copy.size;
     gcptr *items = d->public_with_young_copy.items;
@@ -138,6 +146,7 @@
     for (i = 0; i < size; i++) {
         gcptr P = items[i];
         assert(P->h_tid & GCFLAG_PUBLIC);
+        assert(P->h_tid & GCFLAG_OLD);
 
         revision_t v = ACCESS_ONCE(P->h_revision);
         wlog_t *item;
@@ -247,10 +256,6 @@
 
     mark_young_roots(d);
 
-#if 0
-    mark_private_from_protected(d);
-#endif
-
     mark_public_to_young(d);
 
     visit_all_outside_objects(d);
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -261,3 +261,14 @@
     assert plist == [p3]
     assert classify(p3) == "public"
     assert not lib.in_nursery(p3)
+
+def test_move_to_one_place():
+    p1 = nalloc(HDR)
+    lib.stm_push_root(p1)
+    lib.stm_push_root(p1)
+    lib.stm_push_root(p1)
+    minor_collect()
+    p2a = lib.stm_pop_root()
+    p2b = lib.stm_pop_root()
+    p2c = lib.stm_pop_root()
+    assert p2a == p2b == p2c
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -207,7 +207,7 @@
         return lib._stm_nonrecord_barrier(ptr)
 
     def is_private(self, ptr):
-        return classify(ptr) == "private"
+        return classify(ptr) in ("private", "private_from_protected")
 
     def check_valid(self, lst):
         lst = list(lst)

From noreply at buildbot.pypy.org  Sat Jun 15 22:46:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 22:46:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130615204641.A14C71C0F1D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r137:f0d8de30d1d4
Date: 2013-06-15 22:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/f0d8de30d1d4/

Log:	in-progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -409,6 +409,7 @@
   P->h_revision = (revision_t)B;
 
   gcptrlist_insert(&d->private_from_protected, P);
+  fprintf(stderr, "private_from_protected: insert %p\n", P);
 
   return P;   /* always returns its arg: the object is converted in-place */
 }
@@ -779,6 +780,7 @@
   }
   assert(d->list_of_read_objects.size == 0);
   assert(d->private_from_protected.size == 0);
+  assert(d->num_private_from_protected_known_old == 0);
   assert(!g2l_any_entry(&d->public_to_private));
   assert(d->public_descriptor->stolen_objects.size == 0);
 
@@ -993,6 +995,8 @@
         }
     };
   gcptrlist_clear(&d->private_from_protected);
+  d->num_private_from_protected_known_old = 0;
+  fprintf(stderr, "private_from_protected: clear (commit)\n");
 }
 
 void AbortPrivateFromProtected(struct tx_descriptor *d)
@@ -1005,6 +1009,7 @@
       gcptr P = items[i];
       assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
       assert(IS_POINTER(P->h_revision));
+      P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
 
       gcptr B = (gcptr)P->h_revision;
       assert(B->h_tid & GCFLAG_OLD);
@@ -1012,7 +1017,6 @@
       if (B->h_tid & GCFLAG_PUBLIC)
         {
           assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
-          P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
           P->h_tid |= GCFLAG_PUBLIC;
           if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
           /* P becomes a public outdated object.  It may create an
@@ -1024,12 +1028,20 @@
         }
       else
         {
+          /* copy the backup copy B back over the now-protected object P,
+             and then free B, which will not be used any more. */
+          size_t size = stmcb_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
-          memcpy(P, B, stmcb_size(P));
-          P->h_tid &= ~GCFLAG_BACKUP_COPY;
+          memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
+                 ((char *)B) + offsetof(struct stm_object_s, h_revision),
+                 size - offsetof(struct stm_object_s, h_revision));
+          assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
+          stm_free(B, size);
         }
     };
   gcptrlist_clear(&d->private_from_protected);
+  d->num_private_from_protected_known_old = 0;
+  fprintf(stderr, "private_from_protected: clear (abort)\n");
 }
 
 void CommitTransaction(void)
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -129,6 +129,18 @@
     }
 }
 
+static void mark_private_from_protected(struct tx_descriptor *d)
+{
+    long i, size = d->public_with_young_copy.size;
+    gcptr *items = d->public_with_young_copy.items;
+
+    for (i = d->num_private_from_protected_known_old; i < size; i++) {
+        visit_if_young(&items[i]);
+    }
+
+    d->num_private_from_protected_known_old = size;
+}
+
 static void mark_public_to_young(struct tx_descriptor *d)
 {
     /* "public_with_young_copy" lists the public copies that may have
@@ -256,6 +268,8 @@
 
     mark_young_roots(d);
 
+    mark_private_from_protected(d);
+
     mark_public_to_young(d);
 
     visit_all_outside_objects(d);
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -11,7 +11,8 @@
     char *nursery_end;                                  \
     char *nursery_base;                                 \
     struct GcPtrList old_objects_to_trace;              \
-    struct GcPtrList public_with_young_copy;
+    struct GcPtrList public_with_young_copy;            \
+    long num_private_from_protected_known_old;
 
 struct tx_descriptor;  /* from et.h */
 
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -411,7 +411,7 @@
     tester.run_single_thread()
 
 def test_more_single_thread():
-    py.test.skip("more random tests")
+    #py.test.skip("more random tests")
     for i in range(100):
         yield test_single_thread, i + 3900
 

From noreply at buildbot.pypy.org  Sat Jun 15 22:46:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 22:46:42 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fix
Message-ID: <20130615204642.BE69F1C1398@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r138:440f456cecdc
Date: 2013-06-15 22:19 +0200
http://bitbucket.org/pypy/stmgc/changeset/440f456cecdc/

Log:	fix

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -131,8 +131,8 @@
 
 static void mark_private_from_protected(struct tx_descriptor *d)
 {
-    long i, size = d->public_with_young_copy.size;
-    gcptr *items = d->public_with_young_copy.items;
+    long i, size = d->private_from_protected.size;
+    gcptr *items = d->private_from_protected.items;
 
     for (i = d->num_private_from_protected_known_old; i < size; i++) {
         visit_if_young(&items[i]);

From noreply at buildbot.pypy.org  Sat Jun 15 22:46:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 22:46:43 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix: dont call stmgc_duplicate() when
 we hold the collection_lock.
Message-ID: <20130615204643.E21541C0F1D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r139:cb8059368058
Date: 2013-06-15 22:46 +0200
http://bitbucket.org/pypy/stmgc/changeset/cb8059368058/

Log:	Fix: dont call stmgc_duplicate() when we hold the collection_lock.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -414,7 +414,7 @@
   return P;   /* always returns its arg: the object is converted in-place */
 }
 
-static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
+static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R, gcptr L)
 {
   assert(R->h_tid & GCFLAG_PUBLIC);
 
@@ -427,7 +427,6 @@
 
   R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
-  gcptr L = stmgc_duplicate(R);
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(L->h_tid & GCFLAG_STUB));
   assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
@@ -472,7 +471,7 @@
       return P;
     }
 
-  gcptr R, W;
+  gcptr R, L, W;
   R = stm_read_barrier(P);
 
   if (is_private(R))
@@ -484,6 +483,13 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
+  L = NULL;
+  if (R->h_tid & GCFLAG_PUBLIC)
+    {
+    retry:
+      L = stmgc_duplicate(R);
+    }
+
   /* We need the collection_lock for the sequel; this is required notably
      because we're about to edit flags on a protected object.
   */
@@ -493,12 +499,19 @@
 
   if (R->h_tid & GCFLAG_PUBLIC)
     {
+      if (L == NULL)
+        {
+          /* Oups, the flags on R changed while we where waiting for
+             the collection_lock. */
+          spinlock_release(d->public_descriptor->collection_lock);
+          goto retry;
+        }
       /* Make and return a new (young) private copy of the public R.
          Add R into the list 'public_with_young_copy'.
       */
       assert(R->h_tid & GCFLAG_OLD);
       gcptrlist_insert(&d->public_with_young_copy, R);
-      W = LocalizePublic(d, R);
+      W = LocalizePublic(d, R, L);
       assert(is_private(W));
     }
   else

From noreply at buildbot.pypy.org  Sat Jun 15 22:51:07 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Jun 2013 22:51:07 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130615205107.B4BB11C1398@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r140:b429233c5724
Date: 2013-06-15 22:50 +0200
http://bitbucket.org/pypy/stmgc/changeset/b429233c5724/

Log:	Fix

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -266,12 +266,14 @@
     /* acquire the "collection lock" first */
     setup_minor_collect(d);
 
+    /* first do this, which asserts that some objects are private ---
+       which fails if they have already been GCFLAG_NURSERY_MOVED */
+    mark_public_to_young(d);
+
     mark_young_roots(d);
 
     mark_private_from_protected(d);
 
-    mark_public_to_young(d);
-
     visit_all_outside_objects(d);
 #if 0
     fix_list_of_read_objects(d);

From noreply at buildbot.pypy.org  Sun Jun 16 09:03:21 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Jun 2013 09:03:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: some random small cleanups
Message-ID: <20130616070321.A8D3D1C01C0@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64891:4e6c24fe2fd0
Date: 2013-06-16 00:02 -0700
http://bitbucket.org/pypy/pypy/changeset/4e6c24fe2fd0/

Log:	some random small cleanups

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -1,13 +1,12 @@
+import sys
+
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.platform import platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.unroll import unrolling_iterable
 
-import sys, os
 
-link_files = []
-include_dirs = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32', 'zlib1',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -24,7 +23,7 @@
     includes = []
 
 includes += [
-    'openssl/ssl.h', 
+    'openssl/ssl.h',
     'openssl/err.h',
     'openssl/rand.h',
     'openssl/evp.h',
@@ -33,16 +32,14 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
-    link_files = link_files,
     includes = includes,
-    include_dirs = include_dirs,
     export_symbols = [],
     post_include_bits = [
         # Unnamed structures are not supported by rffi_platform.
         # So we replace an attribute access with a macro call.
         '#define pypy_GENERAL_NAME_dirn(name) (name->d.dirn)',
-        ],
-    )
+    ],
+)
 
 eci = rffi_platform.configure_external_library(
     'openssl', eci,
@@ -114,7 +111,7 @@
     X509_extension_st = rffi_platform.Struct(
         'struct X509_extension_st',
         [('value', ASN1_STRING)])
-    X509V3_EXT_D2I = lltype.FuncType([rffi.VOIDP, rffi.CCHARPP, rffi.LONG], 
+    X509V3_EXT_D2I = lltype.FuncType([rffi.VOIDP, rffi.CCHARPP, rffi.LONG],
                                      rffi.VOIDP)
     v3_ext_method = rffi_platform.Struct(
         'struct v3_ext_method',
@@ -137,7 +134,7 @@
         'OBJ_NAME',
         [('alias', rffi.INT),
          ('name', rffi.CCHARP),
-         ]) 
+         ])
 
 
 for k, v in rffi_platform.configure(CConfig).items():
@@ -259,11 +256,11 @@
 ssl_external('ASN1_STRING_to_UTF8', [rffi.CCHARPP, ASN1_STRING], rffi.INT)
 ssl_external('ASN1_TIME_print', [BIO, ASN1_TIME], rffi.INT)
 ssl_external('i2a_ASN1_INTEGER', [BIO, ASN1_INTEGER], rffi.INT)
-ssl_external('ASN1_item_d2i', 
+ssl_external('ASN1_item_d2i',
              [rffi.VOIDP, rffi.CCHARPP, rffi.LONG, ASN1_ITEM], rffi.VOIDP)
-if OPENSSL_EXPORT_VAR_AS_FUNCTION:             
+if OPENSSL_EXPORT_VAR_AS_FUNCTION:
     ssl_external('ASN1_ITEM_ptr', [ASN1_ITEM_EXP], ASN1_ITEM, macro=True)
-else:    
+else:
     ssl_external('ASN1_ITEM_ptr', [rffi.VOIDP], ASN1_ITEM, macro=True)
 
 ssl_external('sk_GENERAL_NAME_num', [GENERAL_NAMES], rffi.INT,

From noreply at buildbot.pypy.org  Sun Jun 16 10:06:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 10:06:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Buildbot failed for three days because
 of this. I think I'll go for
Message-ID: <20130616080637.BE55E1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64892:642665958a4d
Date: 2013-06-16 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/642665958a4d/

Log:	Buildbot failed for three days because of this. I think I'll go for
	this technique more generally: any failure that stays 3 days on the
	buildbot is going to be reverted/crudely skipped/whatever.

diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -1,3 +1,5 @@
+raise ImportError("please fix the buildbot tests, at least by raising ImportError if tcl.h/tk.h are not there")
+
 # _tkinter package -- low-level interface to libtk and libtcl.
 #
 # This is an internal module, applications should "import Tkinter" instead.

From noreply at buildbot.pypy.org  Sun Jun 16 11:15:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 11:15:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix,
 going in another direction than cb8059368058: stmgc_duplicate()
Message-ID: <20130616091538.EE18B1C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r141:337b49208d45
Date: 2013-06-16 11:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/337b49208d45/

Log:	Fix, going in another direction than cb8059368058: stmgc_duplicate()
	should not cause any minor collection --- the write barrier should
	not.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -414,7 +414,7 @@
   return P;   /* always returns its arg: the object is converted in-place */
 }
 
-static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R, gcptr L)
+static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
   assert(R->h_tid & GCFLAG_PUBLIC);
 
@@ -427,11 +427,13 @@
 
   R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
+  /* note that stmgc_duplicate() usually returns a young object, but may
+     return an old one if the nursery is full at this moment. */
+  gcptr L = stmgc_duplicate(R);
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(L->h_tid & GCFLAG_STUB));
   assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
-  L->h_tid &= ~(GCFLAG_OLD               |
-                GCFLAG_VISITED           |
+  L->h_tid &= ~(GCFLAG_VISITED           |
                 GCFLAG_PUBLIC            |
                 GCFLAG_PREBUILT_ORIGINAL |
                 GCFLAG_PUBLIC_TO_PRIVATE |
@@ -471,7 +473,7 @@
       return P;
     }
 
-  gcptr R, L, W;
+  gcptr R, W;
   R = stm_read_barrier(P);
 
   if (is_private(R))
@@ -483,13 +485,6 @@
   struct tx_descriptor *d = thread_descriptor;
   assert(d->active >= 1);
 
-  L = NULL;
-  if (R->h_tid & GCFLAG_PUBLIC)
-    {
-    retry:
-      L = stmgc_duplicate(R);
-    }
-
   /* We need the collection_lock for the sequel; this is required notably
      because we're about to edit flags on a protected object.
   */
@@ -499,20 +494,21 @@
 
   if (R->h_tid & GCFLAG_PUBLIC)
     {
-      if (L == NULL)
-        {
-          /* Oups, the flags on R changed while we where waiting for
-             the collection_lock. */
-          spinlock_release(d->public_descriptor->collection_lock);
-          goto retry;
-        }
       /* Make and return a new (young) private copy of the public R.
-         Add R into the list 'public_with_young_copy'.
+         Add R into the list 'public_with_young_copy', unless W is
+         actually an old object, in which case we need to record W.
       */
       assert(R->h_tid & GCFLAG_OLD);
-      gcptrlist_insert(&d->public_with_young_copy, R);
-      W = LocalizePublic(d, R, L);
+      W = LocalizePublic(d, R);
       assert(is_private(W));
+
+      if (W->h_tid & GCFLAG_OLD)
+        {
+          W->h_tid |= GCFLAG_WRITE_BARRIER;
+          record_write_barrier(W);
+        }
+      else
+        gcptrlist_insert(&d->public_with_young_copy, R);
     }
   else
     {
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -31,14 +31,20 @@
 
 static char *collect_and_allocate_size(size_t size);  /* forward */
 
-inline static char *allocate_nursery(size_t size)
+inline static char *allocate_nursery(size_t size, int can_collect)
 {
     struct tx_descriptor *d = thread_descriptor;
     char *cur = d->nursery_current;
     char *end = cur + size;
     d->nursery_current = end;
     if (end > d->nursery_end) {
-        cur = collect_and_allocate_size(size);
+        if (can_collect) {
+            cur = collect_and_allocate_size(size);
+        }
+        else {
+            d->nursery_current = cur;
+            cur = NULL;
+        }
     }
     return cur;
 }
@@ -46,7 +52,7 @@
 gcptr stm_allocate(size_t size, unsigned long tid)
 {
     /* XXX inline the fast path */
-    gcptr P = (gcptr)allocate_nursery(size);
+    gcptr P = (gcptr)allocate_nursery(size, 1);
     assert(tid == (tid & STM_USER_TID_MASK));
     P->h_tid = tid;
     P->h_revision = stm_private_rev_num;
@@ -56,7 +62,10 @@
 gcptr stmgc_duplicate(gcptr P)
 {
     size_t size = stmcb_size(P);
-    gcptr L = (gcptr)allocate_nursery(size);
+    gcptr L = (gcptr)allocate_nursery(size, 0);
+    if (L == NULL)
+        return stmgc_duplicate_old(P);
+
     memcpy(L, P, size);
     L->h_tid &= ~GCFLAG_OLD;
     return L;
@@ -290,7 +299,14 @@
     teardown_minor_collect(d);
 
     /* clear the nursery */
+#if defined(_GC_DEBUG) && _GC_DEBUG >= 2
+    stm_free(d->nursery_base, GC_NURSERY);
+    d->nursery_base = stm_malloc(GC_NURSERY);
     memset(d->nursery_base, 0, GC_NURSERY);
+    d->nursery_end = d->nursery_base + GC_NURSERY;
+#else
+    memset(d->nursery_base, 0, GC_NURSERY);
+#endif
     d->nursery_current = d->nursery_base;
 
     assert(!stmgc_minor_collect_anything_to_do(d));
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -464,7 +464,8 @@
     assert 42 < (p.h_tid & 0xFFFF) < 521
 
 def check_nursery_free(p):
-    assert p.h_tid == p.h_revision == 0
+    #assert p.h_tid == p.h_revision == 0
+    assert not lib._stm_can_access_memory(p)
 
 def check_inaccessible(p):
     assert not lib._stm_can_access_memory(p)
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -2,6 +2,9 @@
 from support import *
 
 
+COLLECT_MOVES_NURSERY = True
+
+
 def setup_function(f):
     lib.stm_clear_between_tests()
     lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
@@ -30,7 +33,8 @@
         p1 = lib.stm_pop_root()
         check_not_free(p1)
         check_not_free(p3)
-    assert p2 in seen    # the pointer location was reused
+    if not COLLECT_MOVES_NURSERY:
+        assert p2 in seen    # the pointer location was reused
 
 def test_nursery_follows():
     p1 = nalloc_refs(1)

From noreply at buildbot.pypy.org  Sun Jun 16 11:51:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 11:51:15 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130616095115.906F61C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r143:4420ecc1548e
Date: 2013-06-16 11:50 +0200
http://bitbucket.org/pypy/stmgc/changeset/4420ecc1548e/

Log:	Fix

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -144,7 +144,12 @@
     gcptr *items = d->private_from_protected.items;
 
     for (i = d->num_private_from_protected_known_old; i < size; i++) {
+        assert(items[i]->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        assert(IS_POINTER(items[i]->h_revision));
+
         visit_if_young(&items[i]);
+
+        stmcb_trace((gcptr)items[i]->h_revision, &visit_if_young);
     }
 
     d->num_private_from_protected_known_old = size;

From noreply at buildbot.pypy.org  Sun Jun 16 11:51:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 11:51:14 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test failure
Message-ID: <20130616095114.533E71C3000@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r142:3f512083c4c1
Date: 2013-06-16 11:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/3f512083c4c1/

Log:	Next test failure

diff --git a/c4/test/model.py b/c4/test/model.py
--- a/c4/test/model.py
+++ b/c4/test/model.py
@@ -179,3 +179,10 @@
     def __init__(self, current_revision, numrefs):
         self.created_in_revision = current_revision
         current_revision.content[self] = [None] * numrefs
+
+    def __repr__(self):
+        if hasattr(self, 'identity') and hasattr(self, 'ffi'):
+            return '<StmObject 0x%x>' % (
+                int(self.ffi.cast('intptr_t', self.identity)),)
+        else:
+            return '<StmObject at 0x%x>' % (id(self),)
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -276,3 +276,23 @@
     p2b = lib.stm_pop_root()
     p2c = lib.stm_pop_root()
     assert p2a == p2b == p2c
+
+def test_backup_ptr_update():
+    p1 = nalloc_refs(1)
+    p2 = nalloc(HDR + WORD)
+    lib.setlong(p2, 0, 389719)
+    lib.setptr(p1, 0, p2)
+    lib.stm_push_root(p1)
+    assert lib.in_nursery(p1)
+
+    @perform_transaction
+    def run(retry_counter):
+        if retry_counter == 0:
+            lib.stm_write_barrier(p1)
+            minor_collect()
+            abort_and_retry()
+
+    p1 = lib.stm_pop_root()
+    assert not lib.in_nursery(p1)
+    p2 = lib.getptr(p1, 0)
+    assert lib.getlong(p2, 0) == 389719
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -57,7 +57,7 @@
         text = '%d.%d$ %s\n' % (self.seed, self.counter, text)
         sys.stderr.write(text)
         self.counter += 1
-        #if text.startswith('261225.987$'):
+        #if text.startswith('3970.539$'):
         #    import pdb; pdb.set_trace()
 
     def check_not_free(self, ptr):
@@ -78,6 +78,7 @@
         assert self.current_rev is not None
         obj = model.StmObject(self.current_rev, 2)
         obj.identity = pid
+        obj.ffi = ffi
         self.sync.id2stmobj[pid] = obj
         return pair(obj, nptr)
 
@@ -216,6 +217,7 @@
             p = lst.pop()
             if p == emptypair:
                 continue
+            self.dump(repr(p))
             self.check(p)
 
             ptr = self.nonrecord_barrier(p.ptr)
@@ -241,10 +243,12 @@
                 qobj = content[i]
                 qptr = lib.rawgetptr(ptr, i)
                 q = pair(qobj, qptr)
+                self.dump('[%d] = %r' % (i, q))
                 self.check(q)
                 if q not in seen:
                     lst.append(q)
                     seen.add(q)
+        self.dump('ok')
 
     def transaction_break(self):
         if self.interruptible_transaction:
@@ -412,7 +416,7 @@
 
 def test_more_single_thread():
     #py.test.skip("more random tests")
-    for i in range(100):
+    for i in range(70, 100):
         yield test_single_thread, i + 3900
 
 

From noreply at buildbot.pypy.org  Sun Jun 16 11:56:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 11:56:36 +0200 (CEST)
Subject: [pypy-commit] stmgc default: single-threaded tests seem to pass now.
Message-ID: <20130616095636.D947A1C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r144:e33b431fa84c
Date: 2013-06-16 11:56 +0200
http://bitbucket.org/pypy/stmgc/changeset/e33b431fa84c/

Log:	single-threaded tests seem to pass now.

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -217,7 +217,7 @@
             p = lst.pop()
             if p == emptypair:
                 continue
-            self.dump(repr(p))
+            #self.dump(repr(p))
             self.check(p)
 
             ptr = self.nonrecord_barrier(p.ptr)
@@ -243,12 +243,12 @@
                 qobj = content[i]
                 qptr = lib.rawgetptr(ptr, i)
                 q = pair(qobj, qptr)
-                self.dump('[%d] = %r' % (i, q))
+                #self.dump('[%d] = %r' % (i, q))
                 self.check(q)
                 if q not in seen:
                     lst.append(q)
                     seen.add(q)
-        self.dump('ok')
+        #self.dump('ok')
 
     def transaction_break(self):
         if self.interruptible_transaction:
@@ -415,9 +415,9 @@
     tester.run_single_thread()
 
 def test_more_single_thread():
-    #py.test.skip("more random tests")
-    for i in range(70, 100):
-        yield test_single_thread, i + 3900
+    py.test.skip("more random tests")
+    for i in range(200):
+        yield test_single_thread, i + 3800
 
 
 class Sync(object):

From noreply at buildbot.pypy.org  Sun Jun 16 13:06:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 13:06:10 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130616110610.6ADC51C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r145:293c7843ae68
Date: 2013-06-16 12:40 +0200
http://bitbucket.org/pypy/stmgc/changeset/293c7843ae68/

Log:	Fix

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -224,6 +224,10 @@
            must appear in list_of_read_objects */
         fprintf(stderr, "n.readobj: %p\n", B);
         gcptrlist_insert(&d->list_of_read_objects, B);
+
+        /* must also list it here, in case the next minor collect moves it */
+        if (!(L->h_tid & GCFLAG_OLD))
+            gcptrlist_insert(&d->public_with_young_copy, B);
     }
     gcptrlist_clear(&d->public_descriptor->stolen_objects);
 }

From noreply at buildbot.pypy.org  Sun Jun 16 13:06:12 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 13:06:12 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130616110612.1BBD01C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r146:7635c4202e9b
Date: 2013-06-16 12:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/7635c4202e9b/

Log:	Fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -866,6 +866,9 @@
     {
       gcptr R = item->addr;
       gcptr L = item->val;
+      if (L == NULL)
+        continue;
+
       revision_t expected, v = L->h_revision;
 
       if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -186,9 +186,9 @@
                be able to commit successfully.
             */
             fprintf(stderr, "public_to_young: %p was modified! abort!\n", P);
-            abort();
+            item->val = NULL;
             AbortTransactionAfterCollect(d, ABRT_COLLECT_MINOR);
-            //...
+            continue;
         }
 
         fprintf(stderr, "public_to_young: %p -> %p in public_to_private\n",

From noreply at buildbot.pypy.org  Sun Jun 16 13:06:13 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 13:06:13 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix the test.
Message-ID: <20130616110613.A75F81C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r147:891faa27d62d
Date: 2013-06-16 13:02 +0200
http://bitbucket.org/pypy/stmgc/changeset/891faa27d62d/

Log:	Fix the test.

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -41,6 +41,7 @@
         self.sync_wait = sync_wait
         self.counter = 0
         self.current_rev = None
+        self.awaiting_abort = False
         #self.expecting_gap_in_commit_time = 2
         self.dump('run')
         #
@@ -100,7 +101,7 @@
         self.current_rev = None
 
     def cancel_expected_abort(self):
-        lib.stm_set_max_aborts(0)
+        lib.stm_set_max_aborts(int(self.awaiting_abort))
         self.expected_conflict = False
         self.current_rev = self.aborted_rev
         del self.aborted_rev
@@ -248,6 +249,16 @@
                 if q not in seen:
                     lst.append(q)
                     seen.add(q)
+
+            if self.is_private(ptr) and (p.obj.created_in_revision is not
+                                         self.current_rev):
+                try:
+                    self.current_rev.check_not_outdated(p.obj)
+                except model.Deleted:
+                    self.dump('POTENTIAL ABORT: %s' % (p,))
+                    self.awaiting_abort = True
+                    lib.stm_set_max_aborts(1)
+
         #self.dump('ok')
 
     def transaction_break(self):
@@ -273,9 +284,12 @@
         #
         if retry_counter == 0:
             assert not self.expected_conflict
+            assert not self.awaiting_abort
         else:
-            assert self.expected_conflict
+            assert self.expected_conflict or self.awaiting_abort
             self.expected_conflict = False
+            if self.awaiting_abort: self.current_rev = None
+            self.awaiting_abort = False
         #
         self.roots = self.roots_outside_perform[:]
         self.startrev()
@@ -330,6 +344,8 @@
 
     def commit(self):
         assert self.current_rev.start_time == lib.get_start_time()
+        if self.awaiting_abort:
+            raise model.Conflict("awaiting abort")
         self.current_rev.commit_transaction()
         self.current_rev = None
 

From noreply at buildbot.pypy.org  Sun Jun 16 17:29:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 17:29:00 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Copy this logic from c3.
Message-ID: <20130616152900.704621C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r149:921a07e0b869
Date: 2013-06-16 16:19 +0200
http://bitbucket.org/pypy/stmgc/changeset/921a07e0b869/

Log:	Copy this logic from c3.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -790,6 +790,7 @@
   assert(d->list_of_read_objects.size == 0);
   assert(d->private_from_protected.size == 0);
   assert(d->num_private_from_protected_known_old == 0);
+  assert(d->num_read_objects_known_old == 0);
   assert(!g2l_any_entry(&d->public_to_private));
   assert(d->public_descriptor->stolen_objects.size == 0);
 
@@ -1008,6 +1009,7 @@
     };
   gcptrlist_clear(&d->private_from_protected);
   d->num_private_from_protected_known_old = 0;
+  d->num_read_objects_known_old = 0;
   fprintf(stderr, "private_from_protected: clear (commit)\n");
 }
 
@@ -1053,6 +1055,7 @@
     };
   gcptrlist_clear(&d->private_from_protected);
   d->num_private_from_protected_known_old = 0;
+  d->num_read_objects_known_old = 0;
   fprintf(stderr, "private_from_protected: clear (abort)\n");
 }
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -269,6 +269,31 @@
     }
 }
 
+static void fix_list_of_read_objects(struct tx_descriptor *d)
+{
+    long i, limit = d->num_read_objects_known_old;
+    gcptr *items = d->list_of_read_objects.items;
+    assert(d->list_of_read_objects.size >= limit);
+
+    for (i = d->list_of_read_objects.size - 1; i >= limit; --i) {
+        gcptr obj = items[i];
+
+        if (!is_in_nursery(d, obj)) {
+            /* non-young or visited young objects are kept */
+            continue;
+        }
+        else if (obj->h_tid & GCFLAG_NURSERY_MOVED) {
+            /* visited nursery objects are kept and updated */
+            items[i] = (gcptr)obj->h_revision;
+            continue;
+        }
+        /* The listed object was not visited.  Unlist it. */
+        items[i] = items[--d->list_of_read_objects.size];
+    }
+    d->num_read_objects_known_old = d->list_of_read_objects.size;
+    fxcache_clear(&d->recent_reads_cache);
+}
+
 static void setup_minor_collect(struct tx_descriptor *d)
 {
     spinlock_acquire(d->public_descriptor->collection_lock, 'M');  /*minor*/
@@ -304,18 +329,13 @@
     mark_private_from_protected(d);
 
     visit_all_outside_objects(d);
-#if 0
+
     fix_list_of_read_objects(d);
 
     /* now all surviving nursery objects have been moved out, and all
        surviving young-but-outside-the-nursery objects have been flagged
-       with GCFLAG_OLD */
-    finish_public_to_young(d);
-
-    if (g2l_any_entry(&d->young_objects_outside_nursery))
-        free_unvisited_young_objects_outside_nursery(d);
-#endif
-
+       with GCFLAG_OLD
+    */
     teardown_minor_collect(d);
 
     /* clear the nursery */
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -12,7 +12,8 @@
     char *nursery_base;                                 \
     struct GcPtrList old_objects_to_trace;              \
     struct GcPtrList public_with_young_copy;            \
-    long num_private_from_protected_known_old;
+    long num_private_from_protected_known_old;          \
+    long num_read_objects_known_old;
 
 struct tx_descriptor;  /* from et.h */
 

From noreply at buildbot.pypy.org  Sun Jun 16 17:28:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 17:28:59 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130616152859.48F051C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r148:d7c0041a6fa2
Date: 2013-06-16 15:32 +0200
http://bitbucket.org/pypy/stmgc/changeset/d7c0041a6fa2/

Log:	in-progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1363,6 +1363,7 @@
     assert(d->active == 0);
     assert(d->public_descriptor->stolen_objects.size == 0);
     gcptrlist_delete(&d->public_descriptor->stolen_objects);
+    gcptrlist_delete(&d->public_descriptor->stolen_young_stubs);
 
     spinlock_acquire(descriptor_array_lock, 1);
     i = d->public_descriptor_index;
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -112,6 +112,7 @@
   struct stub_block_s *stub_blocks;
   gcptr stub_free_list;
   struct GcPtrList stolen_objects;
+  struct GcPtrList stolen_young_stubs;
   revision_t free_list_next;
   /* xxx gcpage data here */
 };
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -155,6 +155,48 @@
     d->num_private_from_protected_known_old = size;
 }
 
+static void trace_stub(struct tx_descriptor *d, gcptr S)
+{
+    revision_t w = ACCESS_ONCE(S->h_revision);
+    if ((w & 3) != 2) {
+        /* P has a ptr in h_revision, but this object is not a stub
+           with a protected pointer.  It has likely been the case
+           in the past, but someone made even more changes.
+           Nothing to do now.
+        */
+        fprintf(stderr, "trace_stub: %p not a stub, ignored\n", S);
+        return;
+    }
+
+    assert(S->h_tid & GCFLAG_STUB);
+    if (STUB_THREAD(S) != d->public_descriptor) {
+        /* Bah, it's indeed a stub but for another thread.  Nothing
+           to do now.
+        */
+        fprintf(stderr, "trace_stub: %p stub wrong thread, ignored\n", S);
+        return;
+    }
+
+    /* It's a stub for us.  It cannot be un-stubbed under our
+       feet because we hold our own collection_lock.
+    */
+    gcptr L = (gcptr)(w - 2);
+    fprintf(stderr, "trace_stub: %p stub -> %p\n", S, L);
+    visit_if_young(&L);
+    S->h_revision = ((revision_t)L) | 2;
+}
+
+static void mark_stolen_young_stubs(struct tx_descriptor *d)
+{
+    long i, size = d->public_descriptor->stolen_young_stubs.size;
+    gcptr *items = d->public_descriptor->stolen_young_stubs.items;
+
+    for (i = 0; i < size; i++) {
+        trace_stub(d, items[i]);
+    }
+    gcptrlist_clear(&d->public_descriptor->stolen_young_stubs);
+}
+
 static void mark_public_to_young(struct tx_descriptor *d)
 {
     /* "public_with_young_copy" lists the public copies that may have
@@ -207,37 +249,8 @@
             continue;
         }
 
-        gcptr S = (gcptr)v;
-        revision_t w = ACCESS_ONCE(S->h_revision);
-        if ((w & 3) != 2) {
-            /* P has a ptr in h_revision, but this object is not a stub
-               with a protected pointer.  It has likely been the case
-               in the past, but someone made even more changes.
-               Nothing to do now.
-            */
-            fprintf(stderr, "public_to_young: %p -> %p not a stub, ignored\n",
-                    P, S);
-            continue;
-        }
-
-        if (STUB_THREAD(S) != d->public_descriptor) {
-            /* Bah, it's indeed a stub but for another thread.  Nothing
-               to do now.
-            */
-            fprintf(stderr, "public_to_young: %p -> %p stub wrong thread, "
-                    "ignored\n", P, S);
-            continue;
-        }
-
-        /* It's a stub for us.  It cannot be un-stubbed under our
-           feet because we hold our own collection_lock.
-        */
-        gcptr L = (gcptr)(w - 2);
-        fprintf(stderr, "public_to_young: %p -> %p stub -> %p\n",
-                P, S, L);
-
-        visit_if_young(&L);
-        S->h_revision = ((revision_t)L) | 2;
+        fprintf(stderr, "public_to_young: %p -> ", P);
+        trace_stub(d, (gcptr)v);
     }
 
     gcptrlist_clear(&d->public_with_young_copy);
@@ -286,6 +299,8 @@
 
     mark_young_roots(d);
 
+    mark_stolen_young_stubs(d);
+
     mark_private_from_protected(d);
 
     visit_all_outside_objects(d);
@@ -309,6 +324,8 @@
     d->nursery_base = stm_malloc(GC_NURSERY);
     memset(d->nursery_base, 0, GC_NURSERY);
     d->nursery_end = d->nursery_base + GC_NURSERY;
+    fprintf(stderr, "minor: nursery moved to [%p to %p]\n", d->nursery_base,
+            d->nursery_end);
 #else
     memset(d->nursery_base, 0, GC_NURSERY);
 #endif
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -79,6 +79,9 @@
     stub->h_revision = ((revision_t)obj) | 2;
     g2l_insert(&sd->all_stubs, obj, stub);
 
+    if (!(obj->h_tid & GCFLAG_OLD))
+        gcptrlist_insert(&sd->foreign_pd->stolen_young_stubs, stub);
+
  done:
     *pobj = stub;
     fprintf(stderr, "  stolen: fixing *%p: %p -> %p\n", pobj, obj, stub);
@@ -116,15 +119,14 @@
             L = B;
             goto already_stolen;
         }
-        else {
-            B->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
-            /* add {B: L} in 'public_to_private', but lazily, because we
-               don't want to walk over the feet of the foreign thread
-            */
-            gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
-            fprintf(stderr, "stolen: %p -> %p <-> %p\n", P, L, B);
-            L = B;
-        }
+
+        B->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
+        /* add {B: L} in 'public_to_private', but lazily, because we
+           don't want to walk over the feet of the foreign thread
+        */
+        gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
+        fprintf(stderr, "stolen: %p -> %p <-> %p\n", P, L, B);
+        L = B;
     }
     else {
         if (L->h_tid & GCFLAG_PUBLIC) {
@@ -140,9 +142,8 @@
                 L = (gcptr)v;
             goto already_stolen;
         }
-        else {
-            fprintf(stderr, "stolen: %p -> %p\n", P, L);
-        }
+
+        fprintf(stderr, "stolen: %p -> %p\n", P, L);
 
         /* Copy the object out of the other thread's nursery, if needed */
         if (!(L->h_tid & GCFLAG_OLD)) {
@@ -151,6 +152,7 @@
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
             L = O;
         }
+        assert(L->h_tid & GCFLAG_OLD);
     }
 
     /* Here L is a protected (or backup) copy, and we own the foreign

From noreply at buildbot.pypy.org  Sun Jun 16 17:29:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 17:29:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Progress
Message-ID: <20130616152901.9D2861C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r150:ae17752628f7
Date: 2013-06-16 16:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/ae17752628f7/

Log:	Progress

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -753,6 +753,7 @@
 {
   if (d->active >= 0)
     {
+      fprintf(stderr, "abort %d after collect!\n", reason);
       assert(d->active == 1);   /* not 2, which means inevitable */
       d->active = -reason;
     }
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -378,6 +378,8 @@
         self._r = self.roots[num]
         assert isinstance(self._r, Pair)
         k = self.rnd.randrange(0, 13)
+        self.dump('{')
+        self.check_valid(self.roots + [p])
         self.dump('%4s%4s' % (k, num))
         #
         if k == 0:     # remove a root
@@ -421,7 +423,7 @@
                 abort_and_retry()
         elif k == 12:   # only do an stm_read_barrier
             p = self.read_barrier(p)
-        self.dump('-')
+        self.dump('}')
         self.check_valid(self.roots + [p])
         return p
 
@@ -503,6 +505,6 @@
 
 
 def test_more_multi_thread():
-    py.test.skip("more random tests")
-    for i in range(580//2, 1000):
-        yield test_multi_thread, i
+    #py.test.skip("more random tests")
+    for i in range(100):
+        yield test_multi_thread, 1200 + i

From noreply at buildbot.pypy.org  Sun Jun 16 17:29:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 17:29:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Bug fix (GCFLAG_PUBLIC -> GCFLAG_OLD,
 in the middle of debugging prints)
Message-ID: <20130616152902.B18A51C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r151:502b592672e6
Date: 2013-06-16 16:58 +0200
http://bitbucket.org/pypy/stmgc/changeset/502b592672e6/

Log:	Bug fix (GCFLAG_PUBLIC -> GCFLAG_OLD, in the middle of debugging
	prints)

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -283,6 +283,23 @@
   return NULL;
 }
 
+static void _check_flags(gcptr P)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  int is_old = (P->h_tid & GCFLAG_OLD) != 0;
+  int in_nurs = (d->nursery_base <= (char*)P && ((char*)P) < d->nursery_end);
+  if (in_nurs)
+    {
+      assert(!is_old);
+      fprintf(stderr, "Y ");
+    }
+  else
+    {
+      assert(is_old);
+      fprintf(stderr, "O ");
+    }
+}
+
 gcptr _stm_nonrecord_barrier(gcptr P)
 {
   /* follows the logic in stm_DirectReadBarrier() */
@@ -290,6 +307,7 @@
   revision_t v;
 
   fprintf(stderr, "_stm_nonrecord_barrier: %p ", P);
+  _check_flags(P);
 
  restart_all:
   if (P->h_revision == stm_private_rev_num)
@@ -313,6 +331,9 @@
 
       while (v = P->h_revision, IS_POINTER(v))
         {
+          if (P->h_tid & GCFLAG_NURSERY_MOVED)
+            fprintf(stderr, "nursery_moved ");
+
           if (v & 2)
             {
               fprintf(stderr, "stub ");
@@ -323,6 +344,7 @@
             }
 
           P = (gcptr)v;
+          _check_flags(P);
           assert(P->h_tid & GCFLAG_PUBLIC);
           fprintf(stderr, "-> %p public ", P);
         }
@@ -348,15 +370,18 @@
   if (STUB_THREAD(P) == d->public_descriptor)
     {
       P = (gcptr)(v - 2);
+      _check_flags(P);
       fprintf(stderr, "-> %p ", P);
     }
   else
     {
       P = (gcptr)(v - 2);
+      /* cannot _check_flags(P): foreign! */
       fprintf(stderr, "-foreign-> %p ", P);
       if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
         {
           P = (gcptr)P->h_revision;     /* the backup copy */
+          /* cannot _check_flags(P): foreign! */
           fprintf(stderr, "-backup-> %p ", P);
         }
       if (!(P->h_tid & GCFLAG_PUBLIC))
@@ -365,6 +390,7 @@
           return (gcptr)-1;
         }
     }
+  /* cannot _check_flags(P): foreign! */
   goto restart_all;
 }
 
@@ -409,7 +435,7 @@
   P->h_revision = (revision_t)B;
 
   gcptrlist_insert(&d->private_from_protected, P);
-  fprintf(stderr, "private_from_protected: insert %p\n", P);
+  fprintf(stderr, "private_from_protected: insert %p (backup %p)\n", P, B);
 
   return P;   /* always returns its arg: the object is converted in-place */
 }
@@ -1006,6 +1032,7 @@
       else
         {
           stm_free(B, stmcb_size(B));
+          fprintf(stderr, "commit: free backup at %p\n", B);
         }
     };
   gcptrlist_clear(&d->private_from_protected);
@@ -1052,6 +1079,7 @@
                  size - offsetof(struct stm_object_s, h_revision));
           assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
           stm_free(B, size);
+          fprintf(stderr, "abort: free backup at %p\n", B);
         }
     };
   gcptrlist_clear(&d->private_from_protected);
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -17,6 +17,9 @@
     memset(d->nursery_base, 0, GC_NURSERY);
     d->nursery_end = d->nursery_base + GC_NURSERY;
     d->nursery_current = d->nursery_base;
+
+    fprintf(stderr, "minor: nursery is at [%p to %p]\n", d->nursery_base,
+            d->nursery_end);
 }
 
 void stmgc_done_nursery(void)
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -55,7 +55,8 @@
 static void replace_ptr_to_protected_with_stub(gcptr *pobj)
 {
     gcptr stub, obj = *pobj;
-    if (obj == NULL || (obj->h_tid & GCFLAG_PUBLIC) != 0)
+    if (obj == NULL || (obj->h_tid & (GCFLAG_PUBLIC | GCFLAG_OLD)) ==
+                        (GCFLAG_PUBLIC | GCFLAG_OLD))
         return;
 
     /* we use 'all_stubs', a dictionary, in order to try to avoid
@@ -138,8 +139,10 @@
                has GCFLAG_NURSERY_MOVED), but it is fine to do it more
                generally. */
             v = ACCESS_ONCE(L->h_revision);
-            if (IS_POINTER(v))
+            if (IS_POINTER(v)) {
                 L = (gcptr)v;
+                fprintf(stderr, "\t---> %p\n", L);
+            }
             goto already_stolen;
         }
 
@@ -151,6 +154,7 @@
             L->h_revision = (revision_t)O;
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
             L = O;
+            fprintf(stderr, "\t---> %p\n", L);
         }
         assert(L->h_tid & GCFLAG_OLD);
     }

From noreply at buildbot.pypy.org  Sun Jun 16 17:29:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 17:29:03 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130616152903.C12A01C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r152:9507e1b05a3c
Date: 2013-06-16 17:17 +0200
http://bitbucket.org/pypy/stmgc/changeset/9507e1b05a3c/

Log:	Fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -286,6 +286,10 @@
 static void _check_flags(gcptr P)
 {
   struct tx_descriptor *d = thread_descriptor;
+  if (P->h_tid & GCFLAG_STUB)
+    {
+      fprintf(stderr, "S");
+    }
   int is_old = (P->h_tid & GCFLAG_OLD) != 0;
   int in_nurs = (d->nursery_base <= (char*)P && ((char*)P) < d->nursery_end);
   if (in_nurs)
@@ -344,9 +348,9 @@
             }
 
           P = (gcptr)v;
-          _check_flags(P);
           assert(P->h_tid & GCFLAG_PUBLIC);
           fprintf(stderr, "-> %p public ", P);
+          _check_flags(P);
         }
 
       gcptr L = _find_public_to_private(P);
@@ -370,8 +374,8 @@
   if (STUB_THREAD(P) == d->public_descriptor)
     {
       P = (gcptr)(v - 2);
+      fprintf(stderr, "-> %p ", P);
       _check_flags(P);
-      fprintf(stderr, "-> %p ", P);
     }
   else
     {
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -112,7 +112,24 @@
         /* it's a nursery object.  Was it already moved? */
 
         if (UNLIKELY(obj->h_tid & GCFLAG_NURSERY_MOVED)) {
-            /* yes: just fix the ref. */
+
+            /* yes, but was it actually a public object in the nursery?
+               (such objects are always NURSERY_MOVED) */
+            if (obj->h_tid & GCFLAG_PUBLIC) {
+
+                /* follow the chain of revisions.  Necessary, otherwise
+                   we could end up with 'obj' being an incomplete stub. */
+                while (1) {
+                    revision_t v = ACCESS_ONCE(obj->h_revision);
+                    if (!IS_POINTER(v))
+                        break;
+                    obj = (gcptr)v;
+                }
+                *root = obj;
+                return;
+            }
+
+            /* common case: multiple refs, just fix the ref. */
             *root = (gcptr)obj->h_revision;
             return;
         }
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -506,5 +506,5 @@
 
 def test_more_multi_thread():
     #py.test.skip("more random tests")
-    for i in range(100):
+    for i in range(45, 100):
         yield test_multi_thread, 1200 + i

From noreply at buildbot.pypy.org  Sun Jun 16 17:52:14 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 16 Jun 2013 17:52:14 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fix for the alignment calculation of
 anonymous and zero sized bitfields on ARM.
Message-ID: <20130616155214.8D2761C01C0@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r1268:09f85e97e9d3
Date: 2013-06-16 10:18 -0500
http://bitbucket.org/cffi/cffi/changeset/09f85e97e9d3/

Log:	Fix for the alignment calculation of anonymous and zero sized
	bitfields on ARM.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3503,6 +3503,7 @@
             goto error;
 
         do_align = 1;
+#ifndef __arm__
         if (fbitsize >= 0) {
             if (!(sflags & SF_MSVC_BITFIELDS)) {
                 /* GCC: anonymous bitfields (of any size) don't cause alignment */
@@ -3513,6 +3514,7 @@
                 do_align = fbitsize > 0;
             }
         }
+#endif
         if (alignment < falign && do_align)
             alignment = falign;
 
diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py
--- a/testing/test_ffi_backend.py
+++ b/testing/test_ffi_backend.py
@@ -1,4 +1,5 @@
-import py, sys
+import py, sys, platform
+import pytest
 from testing import backend_tests, test_function, test_ownlib
 from cffi import FFI
 import _cffi_backend
@@ -121,6 +122,7 @@
         self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8)
         self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4)
 
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_anonymous_no_align(self):
         L = FFI().alignof("long long")
         self.check("char y; int :1;", 0, 1, 2)
@@ -133,6 +135,20 @@
         self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
         self.check("char x; long long  :57; char y;", L + 8, 1, L + 9)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_anonymous_align_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :1;", 0, 4, 4)
+        self.check("char x; int z:1; char y;", 2, 4, 4)
+        self.check("char x; int  :1; char y;", 2, 4, 4)
+        self.check("char x; long long z:48; char y;", 7, L, 8)
+        self.check("char x; long long  :48; char y;", 7, 8, 8)
+        self.check("char x; long long z:56; char y;", 8, L, 8 + L)
+        self.check("char x; long long  :56; char y;", 8, L, 8 + L)
+        self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
+        self.check("char x; long long  :57; char y;", L + 8, L, L + 8 + L)
+
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_zero(self):
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
@@ -143,6 +159,17 @@
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
         self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_zero_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :0;", 0, 4, 4)
+        self.check("char x; int :0; char y;", 4, 4, 8)
+        self.check("char x; int :0; int :0; char y;", 4, 4, 8)
+        self.check("char x; long long :0; char y;", L, 8, L + 8)
+        self.check("short x, y; int :0; int :0;", 2, 4, 4)
+        self.check("char x; int :0; short b:1; char y;", 5, 4, 8)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
+
     def test_error_cases(self):
         ffi = FFI()
         py.test.raises(TypeError,

From noreply at buildbot.pypy.org  Sun Jun 16 18:03:21 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 16 Jun 2013 18:03:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for the alignment calculation of
 anonymous and zero width bitfields on ARM
Message-ID: <20130616160321.C95751C0547@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64893:45b0965e4d99
Date: 2013-06-16 10:48 -0500
http://bitbucket.org/pypy/pypy/changeset/45b0965e4d99/

Log:	Fix for the alignment calculation of anonymous and zero width
	bitfields on ARM

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 
 from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr,
     ctypearray, ctypestruct, ctypevoid, ctypeenum)
@@ -121,6 +122,7 @@
 else:
     DEFAULT_SFLAGS = 0
 
+_ARM = rffi_platform.getdefined('__arm__', '')
 
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
@@ -180,7 +182,7 @@
         # field is an anonymous bitfield
         falign = ftype.alignof()
         do_align = True
-        if fbitsize >= 0:
+        if fbitsize >= 0 and not _ARM:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
                 # GCC: anonymous bitfields (of any size) don't cause alignment
                 do_align = (fname != '')

From noreply at buildbot.pypy.org  Sun Jun 16 18:03:23 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 16 Jun 2013 18:03:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: import tests from cffi
Message-ID: <20130616160323.3AD231C0547@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64894:8d3a39a0e619
Date: 2013-06-16 10:56 -0500
http://bitbucket.org/pypy/pypy/changeset/8d3a39a0e619/

Log:	import tests from cffi

diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -1,5 +1,6 @@
 # Generated by pypy/tool/import_cffi.py
-import py, sys
+import py, sys, platform
+import pytest
 from pypy.module.test_lib_pypy.cffi_tests import backend_tests, test_function, test_ownlib
 from cffi import FFI
 import _cffi_backend
@@ -122,6 +123,7 @@
         self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8)
         self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4)
 
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_anonymous_no_align(self):
         L = FFI().alignof("long long")
         self.check("char y; int :1;", 0, 1, 2)
@@ -134,6 +136,20 @@
         self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
         self.check("char x; long long  :57; char y;", L + 8, 1, L + 9)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_anonymous_align_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :1;", 0, 4, 4)
+        self.check("char x; int z:1; char y;", 2, 4, 4)
+        self.check("char x; int  :1; char y;", 2, 4, 4)
+        self.check("char x; long long z:48; char y;", 7, L, 8)
+        self.check("char x; long long  :48; char y;", 7, 8, 8)
+        self.check("char x; long long z:56; char y;", 8, L, 8 + L)
+        self.check("char x; long long  :56; char y;", 8, L, 8 + L)
+        self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
+        self.check("char x; long long  :57; char y;", L + 8, L, L + 8 + L)
+
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_zero(self):
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
@@ -144,6 +160,17 @@
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
         self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_zero_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :0;", 0, 4, 4)
+        self.check("char x; int :0; char y;", 4, 4, 8)
+        self.check("char x; int :0; int :0; char y;", 4, 4, 8)
+        self.check("char x; long long :0; char y;", L, 8, L + 8)
+        self.check("short x, y; int :0; int :0;", 2, 4, 4)
+        self.check("char x; int :0; short b:1; char y;", 5, 4, 8)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
+
     def test_error_cases(self):
         ffi = FFI()
         py.test.raises(TypeError,

From noreply at buildbot.pypy.org  Sun Jun 16 18:15:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:15:17 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next fix
Message-ID: <20130616161517.64F5A1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r153:dc86e7cd8452
Date: 2013-06-16 18:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/dc86e7cd8452/

Log:	Next fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -601,6 +601,13 @@
                  before it was turned from protected to private */
               continue;
             }
+          else if ((R->h_tid & (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
+                            == (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
+            {
+              /* such an object is identical to the one it points to */
+              R = (gcptr)v;
+              goto retry;
+            }
           else
             {
               fprintf(stderr, "validation failed: "
@@ -611,20 +618,15 @@
       if (v >= LOCKED)            // locked
         {
           if (!during_commit)
+            assert(v != d->my_lock);    // we don't hold any lock
+
+          if (v != d->my_lock)
             {
-              assert(v != d->my_lock);    // we don't hold any lock
+              /* not locked by me: spinloop until the other thread releases
+                 its lock */
               SpinLoop(SPLP_LOCKED_VALIDATE);
               goto retry;
             }
-          else
-            {
-              if (v != d->my_lock)         // not locked by me: conflict
-                {
-                  fprintf(stderr, "validation failed: "
-                          "%p is locked by another thread\n", R);
-                  return 0;
-                }
-            }
         }
     }
   return 1;
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -87,6 +87,7 @@
 
 static inline gcptr create_old_object_copy(gcptr obj)
 {
+    assert(!(obj->h_tid & GCFLAG_PUBLIC));
     assert(!(obj->h_tid & GCFLAG_NURSERY_MOVED));
     assert(!(obj->h_tid & GCFLAG_VISITED));
     assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -506,5 +506,5 @@
 
 def test_more_multi_thread():
     #py.test.skip("more random tests")
-    for i in range(45, 100):
+    for i in range(46, 100):
         yield test_multi_thread, 1200 + i

From noreply at buildbot.pypy.org  Sun Jun 16 18:16:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:16:10 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Don't use tcolor=37, which is white
Message-ID: <20130616161610.CAD931C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r154:9e594643aa52
Date: 2013-06-16 18:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/9e594643aa52/

Log:	Don't use tcolor=37, which is white

diff --git a/c4/fprintcolor.c b/c4/fprintcolor.c
--- a/c4/fprintcolor.c
+++ b/c4/fprintcolor.c
@@ -15,7 +15,7 @@
             if (bool_cas(&tnextid, tcolor, tcolor + 1))
                 break;
         }
-        tcolor = 31 + tcolor % 7;
+        tcolor = 31 + tcolor % 6;
     }
     int size = (int)sprintf(buffer, "\033[%dm", (int)tcolor);
     assert(size >= 0);

From noreply at buildbot.pypy.org  Sun Jun 16 18:21:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:21:42 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Revert the change done in a recent
 commit and fix it differently
Message-ID: <20130616162142.C284F1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r155:5a94c3c1a159
Date: 2013-06-16 18:21 +0200
http://bitbucket.org/pypy/stmgc/changeset/5a94c3c1a159/

Log:	Revert the change done in a recent commit and fix it differently
	(there are cases where it was still wrong).

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -955,7 +955,9 @@
       L->h_revision = new_revision;
 
       gcptr stub = stm_stub_malloc(d->public_descriptor);
-      stub->h_tid = GCFLAG_PUBLIC | GCFLAG_STUB | GCFLAG_OLD;
+      stub->h_tid = (L->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
+                                                   | GCFLAG_STUB
+                                                   | GCFLAG_OLD;
       stub->h_revision = ((revision_t)L) | 2;
       item->val = stub;
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -113,24 +113,9 @@
         /* it's a nursery object.  Was it already moved? */
 
         if (UNLIKELY(obj->h_tid & GCFLAG_NURSERY_MOVED)) {
-
-            /* yes, but was it actually a public object in the nursery?
-               (such objects are always NURSERY_MOVED) */
-            if (obj->h_tid & GCFLAG_PUBLIC) {
-
-                /* follow the chain of revisions.  Necessary, otherwise
-                   we could end up with 'obj' being an incomplete stub. */
-                while (1) {
-                    revision_t v = ACCESS_ONCE(obj->h_revision);
-                    if (!IS_POINTER(v))
-                        break;
-                    obj = (gcptr)v;
-                }
-                *root = obj;
-                return;
-            }
-
-            /* common case: multiple refs, just fix the ref. */
+            /* yes.  Such an object can be a public object in the nursery
+               too (such objects are always NURSERY_MOVED).  For all cases,
+               we can just fix the ref. */
             *root = (gcptr)obj->h_revision;
             return;
         }
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -506,5 +506,5 @@
 
 def test_more_multi_thread():
     #py.test.skip("more random tests")
-    for i in range(46, 100):
+    for i in range(100):
         yield test_multi_thread, 1200 + i

From noreply at buildbot.pypy.org  Sun Jun 16 18:29:02 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 16 Jun 2013 18:29:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: "import _tkinter" now raises
 ImportError when tk.h and other development files are not installed.
Message-ID: <20130616162902.6A3DB1C3000@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r64895:b5cea7cd6c02
Date: 2013-06-16 18:28 +0200
http://bitbucket.org/pypy/pypy/changeset/b5cea7cd6c02/

Log:	"import _tkinter" now raises ImportError when tk.h and other
	development files are not installed.

diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -1,5 +1,3 @@
-raise ImportError("please fix the buildbot tests, at least by raising ImportError if tcl.h/tk.h are not there")
-
 # _tkinter package -- low-level interface to libtk and libtcl.
 #
 # This is an internal module, applications should "import Tkinter" instead.
@@ -10,7 +8,12 @@
 class TclError(Exception):
     pass
 
-from .tklib import tklib, tkffi
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
 from .app import TkApp
 
 TK_VERSION = tkffi.string(tklib.get_tk_version())

From noreply at buildbot.pypy.org  Sun Jun 16 18:46:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:46:47 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130616164647.276871C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r157:c6e6c1abc9e2
Date: 2013-06-16 18:37 +0200
http://bitbucket.org/pypy/stmgc/changeset/c6e6c1abc9e2/

Log:	Fix

diff --git a/c4/atomic_ops.h b/c4/atomic_ops.h
--- a/c4/atomic_ops.h
+++ b/c4/atomic_ops.h
@@ -96,13 +96,17 @@
 #endif
 
 
-#define spinlock_acquire(lock, targetvalue)                     \
-    do { if (bool_cas(&(lock), 0, (targetvalue))) break;        \
-         do { smp_spinloop(); } while (ACCESS_ONCE(lock));      \
+#define spinlock_acquire(lock, targetvalue)                             \
+    do { if (bool_cas(&(lock), 0, (targetvalue))) {                     \
+             fprintf(stderr, "<<< locked %d\n", (int)targetvalue);      \
+             break;                                                     \
+         }                                                              \
+         do { smp_spinloop(); } while (ACCESS_ONCE(lock));              \
     } while (1)
 
-#define spinlock_release(lock)                                  \
-    do { smp_wmb(); assert((lock) != 0); (lock) = 0; } while (0)
+#define spinlock_release(lock)                          \
+    do { fprintf(stderr, "unlocked >>>\n"); smp_wmb();  \
+         assert((lock) != 0); (lock) = 0; } while (0)
 
 
 #endif  /* _SRCSTM_ATOMIC_OPS_ */
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -112,7 +112,8 @@
 
           gcptr P_prev = P;
           P = (gcptr)v;
-          assert(P->h_tid & GCFLAG_PUBLIC);
+          assert((P->h_tid & GCFLAG_PUBLIC) ||
+                 (P_prev->h_tid & GCFLAG_NURSERY_MOVED));
 
           v = ACCESS_ONCE(P->h_revision);
 

From noreply at buildbot.pypy.org  Sun Jun 16 18:46:46 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:46:46 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix the Makefile
Message-ID: <20130616164646.183341C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r156:633c13cdae59
Date: 2013-06-16 18:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/633c13cdae59/

Log:	Fix the Makefile

diff --git a/c4/Makefile b/c4/Makefile
--- a/c4/Makefile
+++ b/c4/Makefile
@@ -9,8 +9,8 @@
 
 
 H_FILES = atomic_ops.h stmgc.h stmimpl.h \
-          dbgmem.h et.h fprintcolor.h lists.h steal.h stmsync.h
-C_FILES = dbgmem.c et.c fprintcolor.c lists.c steal.c stmsync.c
+	  et.h lists.h steal.h nursery.h stmsync.h dbgmem.h fprintcolor.h
+C_FILES = et.c lists.c steal.c nursery.c stmsync.c dbgmem.c fprintcolor.c
 
 DEBUG = -g -DGC_NURSERY=0x10000  #-D_GC_DEBUG=1
 

From noreply at buildbot.pypy.org  Sun Jun 16 18:46:48 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:46:48 +0200 (CEST)
Subject: [pypy-commit] stmgc default: These fprints are really too much on
	some examples
Message-ID: <20130616164648.413F41C300E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r158:b7f5e39b46f9
Date: 2013-06-16 18:46 +0200
http://bitbucket.org/pypy/stmgc/changeset/b7f5e39b46f9/

Log:	These fprints are really too much on some examples

diff --git a/c4/atomic_ops.h b/c4/atomic_ops.h
--- a/c4/atomic_ops.h
+++ b/c4/atomic_ops.h
@@ -96,17 +96,25 @@
 #endif
 
 
-#define spinlock_acquire(lock, targetvalue)                             \
+#if 0    /* fprinting versions */
+# define spinlock_acquire(lock, targetvalue)                            \
     do { if (bool_cas(&(lock), 0, (targetvalue))) {                     \
              fprintf(stderr, "<<< locked %d\n", (int)targetvalue);      \
              break;                                                     \
          }                                                              \
          do { smp_spinloop(); } while (ACCESS_ONCE(lock));              \
     } while (1)
-
-#define spinlock_release(lock)                          \
-    do { fprintf(stderr, "unlocked >>>\n"); smp_wmb();  \
+# define spinlock_release(lock)                                         \
+    do { fprintf(stderr, "unlocked >>>\n"); smp_wmb();                  \
          assert((lock) != 0); (lock) = 0; } while (0)
+#else
+# define spinlock_acquire(lock, targetvalue)                    \
+    do { if (bool_cas(&(lock), 0, (targetvalue))) break;        \
+         do { smp_spinloop(); } while (ACCESS_ONCE(lock));      \
+    } while (1)
+# define spinlock_release(lock)                                 \
+    do { smp_wmb(); assert((lock) != 0); (lock) = 0; } while (0)
+#endif
 
 
 #endif  /* _SRCSTM_ATOMIC_OPS_ */

From noreply at buildbot.pypy.org  Sun Jun 16 18:46:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:46:49 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix.
Message-ID: <20130616164649.4E7B81C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r159:b1fc519ee04b
Date: 2013-06-16 18:46 +0200
http://bitbucket.org/pypy/stmgc/changeset/b1fc519ee04b/

Log:	Fix.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -448,6 +448,7 @@
 static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
 {
   assert(R->h_tid & GCFLAG_PUBLIC);
+  assert(!(R->h_tid & GCFLAG_NURSERY_MOVED));
 
 #ifdef _GC_DEBUG
   wlog_t *entry;
@@ -529,6 +530,14 @@
          Add R into the list 'public_with_young_copy', unless W is
          actually an old object, in which case we need to record W.
       */
+      if (R->h_tid & GCFLAG_NURSERY_MOVED)
+        {
+          /* Bah, the object turned into this kind of stub while we
+             were waiting for the collection_lock, because it was
+             stolen by someone else.  Use R->h_revision instead. */
+          assert(IS_POINTER(R->h_revision));
+          R = (gcptr)R->h_revision;
+        }
       assert(R->h_tid & GCFLAG_OLD);
       W = LocalizePublic(d, R);
       assert(is_private(W));

From noreply at buildbot.pypy.org  Sun Jun 16 18:58:16 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:58:16 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Skip again these extra tests,
	they seem to pass
Message-ID: <20130616165816.E17681C0547@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r160:8da924453f29
Date: 2013-06-16 18:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/8da924453f29/

Log:	Skip again these extra tests, they seem to pass

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -505,6 +505,6 @@
 
 
 def test_more_multi_thread():
-    #py.test.skip("more random tests")
+    py.test.skip("more random tests")
     for i in range(100):
         yield test_multi_thread, 1200 + i

From noreply at buildbot.pypy.org  Sun Jun 16 18:58:18 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:58:18 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Document the progress
Message-ID: <20130616165818.07EE91C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r161:6b668352cf79
Date: 2013-06-16 18:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/6b668352cf79/

Log:	Document the progress

diff --git a/README.txt b/README.txt
--- a/README.txt
+++ b/README.txt
@@ -9,6 +9,10 @@
 
 The library interface is in "c4/stmgc.h".
 
+Progress (these revisions are roughly stable versions, pick the latest):
+- f1ccf5bbcb6f: first step, working but with no GC
+- 8da924453f29: minor collection seems to be working, no major GC
+
 The file "c4/doc-objects.txt" contains some low-level explanations.
 
 Run tests with "py.test".

From noreply at buildbot.pypy.org  Sun Jun 16 18:58:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Jun 2013 18:58:19 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Remove this tag,
	documented explicitly in README.txt.
Message-ID: <20130616165819.1BD521C300E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r162:92e3e69f428b
Date: 2013-06-16 18:58 +0200
http://bitbucket.org/pypy/stmgc/changeset/92e3e69f428b/

Log:	Remove this tag, documented explicitly in README.txt.

diff --git a/.hgtags b/.hgtags
deleted file mode 100644
--- a/.hgtags
+++ /dev/null
@@ -1,1 +0,0 @@
-f1ccf5bbcb6f01fb995622f2b569e9858fbea08a c4-without-gc

From noreply at buildbot.pypy.org  Sun Jun 16 21:36:26 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 16 Jun 2013 21:36:26 +0200 (CEST)
Subject: [pypy-commit] cffi default: extend and fix test_c.py for ARM
Message-ID: <20130616193626.070B61C3000@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r1270:ec08e04d0ec1
Date: 2013-06-16 14:01 -0500
http://bitbucket.org/cffi/cffi/changeset/ec08e04d0ec1/

Log:	extend and fix test_c.py for ARM

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2778,10 +2778,10 @@
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
@@ -2794,7 +2794,10 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
@@ -2806,10 +2809,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2818,6 +2826,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy

From noreply at buildbot.pypy.org  Sun Jun 16 21:36:24 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 16 Jun 2013 21:36:24 +0200 (CEST)
Subject: [pypy-commit] cffi default: check for ARM using a flag like it is
	done for MSVC
Message-ID: <20130616193624.D786A1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r1269:58dc7182ce4d
Date: 2013-06-16 13:44 -0500
http://bitbucket.org/cffi/cffi/changeset/58dc7182ce4d/

Log:	check for ARM using a flag like it is done for MSVC

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3425,6 +3425,7 @@
 }
 
 #define SF_MSVC_BITFIELDS 1
+#define SF_GCC_ARM_BITFIELDS 2
 
 static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args)
 {
@@ -3439,7 +3440,11 @@
 #ifdef MS_WIN32
     int sflags = SF_MSVC_BITFIELDS;
 #else
+# ifdef __arm__
+    int sflags = SF_GCC_ARM_BITFIELDS;
+# else
     int sflags = 0;
+# endif
 #endif
 
     if (!PyArg_ParseTuple(args, "O!O!|Onii:complete_struct_or_union",
@@ -3503,8 +3508,7 @@
             goto error;
 
         do_align = 1;
-#ifndef __arm__
-        if (fbitsize >= 0) {
+        if (!(sflags & SF_GCC_ARM_BITFIELDS) && fbitsize >= 0) {
             if (!(sflags & SF_MSVC_BITFIELDS)) {
                 /* GCC: anonymous bitfields (of any size) don't cause alignment */
                 do_align = PyText_GetSize(fname) > 0;
@@ -3514,7 +3518,6 @@
                 do_align = fbitsize > 0;
             }
         }
-#endif
         if (alignment < falign && do_align)
             alignment = falign;
 

From noreply at buildbot.pypy.org  Sun Jun 16 21:51:18 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 16 Jun 2013 21:51:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: import updated tests from
	cffi:c/test_c.py
Message-ID: <20130616195118.09D111C01C0@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64896:c761d6824704
Date: 2013-06-16 14:41 -0500
http://bitbucket.org/pypy/pypy/changeset/c761d6824704/

Log:	import updated tests from cffi:c/test_c.py

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2767,10 +2767,10 @@
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
@@ -2783,7 +2783,10 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
@@ -2795,10 +2798,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2807,6 +2815,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy

From noreply at buildbot.pypy.org  Sun Jun 16 21:51:19 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 16 Jun 2013 21:51:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: check for ARM using a flag like it is
	done for MSVC
Message-ID: <20130616195119.83E771C01C0@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64897:b4b26fd9fe5b
Date: 2013-06-16 14:41 -0500
http://bitbucket.org/pypy/pypy/changeset/b4b26fd9fe5b/

Log:	check for ARM using a flag like it is done for MSVC

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -116,14 +116,15 @@
 # ____________________________________________________________
 
 SF_MSVC_BITFIELDS = 1
+SF_GCC_ARM_BITFIELDS = 2
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+elif rffi_platform.getdefined('__arm__', ''):
+    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
     DEFAULT_SFLAGS = 0
 
-_ARM = rffi_platform.getdefined('__arm__', '')
-
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -182,7 +183,7 @@
         # field is an anonymous bitfield
         falign = ftype.alignof()
         do_align = True
-        if fbitsize >= 0 and not _ARM:
+        if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
                 # GCC: anonymous bitfields (of any size) don't cause alignment
                 do_align = (fname != '')

From noreply at buildbot.pypy.org  Mon Jun 17 00:33:18 2013
From: noreply at buildbot.pypy.org (Flavio Percoco)
Date: Mon, 17 Jun 2013 00:33:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Return level's constant when level's
	name is passed.
Message-ID: <20130616223318.279C51C0547@cobra.cs.uni-duesseldorf.de>

Author: Flavio Percoco <flaper87 at gmail.com>
Branch: 
Changeset: r64898:26a150624b95
Date: 2013-06-16 23:58 +0200
http://bitbucket.org/pypy/pypy/changeset/26a150624b95/

Log:	Return level's constant when level's name is passed.

	After splitting _levelNames dictionary into 2 separate dicts, the
	getLevelName function stopped returning the level's integer value
	when the level's name is passed as argument. This patch restores
	that behavior.

	Fixes bug: 1517

diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,6 +134,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -168,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -278,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""

From noreply at buildbot.pypy.org  Mon Jun 17 09:57:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 09:57:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test_whatsnew
Message-ID: <20130617075732.0F7E61C03D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64899:81aa4bb4d0b6
Date: 2013-06-17 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/81aa4bb4d0b6/

Log:	fix test_whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -56,3 +56,4 @@
 .. branch: argsort-segfault
 Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
 
+.. branch: dtype-isnative

From noreply at buildbot.pypy.org  Mon Jun 17 11:14:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 11:14:45 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Document this flag
Message-ID: <20130617091445.1AA2F1C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r163:a7951703474b
Date: 2013-06-17 11:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/a7951703474b/

Log:	Document this flag

diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -53,6 +53,11 @@
  * by major collections.  'p_stub->h_revision' might be a value
  * that is == 2 (mod 4): in this case they point to a protected/private
  * object that belongs to the thread 'STUB_THREAD(p_stub)'.
+ *
+ * GCFLAG_PRIVATE_FROM_PROTECTED is set on objects that are private
+ * but converted from a protected.  These are precisely the objects
+ * that have a backup copy (in h_revision), which gives a copy of the
+ * original protected object.
  */
 static const revision_t GCFLAG_OLD                    = STM_FIRST_GCFLAG << 0;
 static const revision_t GCFLAG_VISITED                = STM_FIRST_GCFLAG << 1;

From noreply at buildbot.pypy.org  Mon Jun 17 13:15:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 13:15:56 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Comments
Message-ID: <20130617111556.02F801C1001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r164:a1263ec3c083
Date: 2013-06-17 13:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/a1263ec3c083/

Log:	Comments

diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -6,15 +6,35 @@
 #endif
 
 
-#define NURSERY_FIELDS_DECL                             \
-    char *nursery_current;                              \
-    char *nursery_end;                                  \
-    char *nursery_base;                                 \
-    struct GcPtrList old_objects_to_trace;              \
-    struct GcPtrList public_with_young_copy;            \
-    long num_private_from_protected_known_old;          \
+#define NURSERY_FIELDS_DECL                                             \
+    /* the nursery */                                                   \
+    char *nursery_current;                                              \
+    char *nursery_end;                                                  \
+    char *nursery_base;                                                 \
+                                                                        \
+    /* Between collections, we add to 'old_objects_to_trace' the        \
+       private objects that are old but may contain pointers to         \
+       young objects.  During minor collections the same list is        \
+       used to record all other old objects pending tracing; in         \
+       other words minor collection is a process that works             \
+       until the list is empty again. */                                \
+    struct GcPtrList old_objects_to_trace;                              \
+                                                                        \
+    /* 'public_with_young_copy' is a list of all public objects         \
+       that are outdated and whose 'h_revision' points to a             \
+       young object. */                                                 \
+    struct GcPtrList public_with_young_copy;                            \
+                                                                        \
+    /* These numbers are initially zero, but after a minor              \
+       collection, they are set to the size of the two lists            \
+       'private_from_protected' and 'list_of_read_objects'.             \
+       It's used on the following minor collection, if we're            \
+       still in the same transaction, to know that the initial          \
+       part of the lists cannot contain young objects any more. */      \
+    long num_private_from_protected_known_old;                          \
     long num_read_objects_known_old;
 
+
 struct tx_descriptor;  /* from et.h */
 
 void stmgc_init_nursery(void);

From noreply at buildbot.pypy.org  Mon Jun 17 13:35:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 13:35:51 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Start importing "c3/gcpage".
Message-ID: <20130617113551.8BD711C1001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r165:04275ee055bf
Date: 2013-06-17 13:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/04275ee055bf/

Log:	Start importing "c3/gcpage".

diff --git a/c4/Makefile b/c4/Makefile
--- a/c4/Makefile
+++ b/c4/Makefile
@@ -9,8 +9,11 @@
 
 
 H_FILES = atomic_ops.h stmgc.h stmimpl.h \
-	  et.h lists.h steal.h nursery.h stmsync.h dbgmem.h fprintcolor.h
-C_FILES = et.c lists.c steal.c nursery.c stmsync.c dbgmem.c fprintcolor.c
+	  et.h lists.h steal.h nursery.h gcpage.h \
+          stmsync.h dbgmem.h fprintcolor.h
+
+C_FILES = et.c lists.c steal.c nursery.c gcpage.c \
+          stmsync.c dbgmem.c fprintcolor.c
 
 DEBUG = -g -DGC_NURSERY=0x10000  #-D_GC_DEBUG=1
 
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -13,7 +13,6 @@
 
 #define MAX_THREADS         1024
 #define LOCKED              (INTPTR_MAX - 2*(MAX_THREADS-1))
-#define WORD                sizeof(gcptr)
 
 /* Description of the flags
  * ------------------------
@@ -119,7 +118,8 @@
   struct GcPtrList stolen_objects;
   struct GcPtrList stolen_young_stubs;
   revision_t free_list_next;
-  /* xxx gcpage data here */
+
+  GCPAGE_FIELDS_DECL
 };
 
 /* this struct contains all thread-local data that is never accessed
diff --git a/c4/gcpage.c b/c4/gcpage.c
new file mode 100644
--- /dev/null
+++ b/c4/gcpage.c
@@ -0,0 +1,15 @@
+#include "stmimpl.h"
+
+
+/* For statistics */
+static uintptr_t count_global_pages;
+
+/* For tests */
+long stmgcpage_count(int quantity)
+{
+    switch (quantity) {
+    case 0: return count_global_pages;
+    case 1: return LOCAL_GCPAGES()->count_pages;
+    default: return -1;
+    }
+}
diff --git a/c4/gcpage.h b/c4/gcpage.h
new file mode 100644
--- /dev/null
+++ b/c4/gcpage.h
@@ -0,0 +1,78 @@
+#ifndef _SRCSTM_GCPAGE_H
+#define _SRCSTM_GCPAGE_H
+
+
+#define WORD                sizeof(gcptr)
+#define GC_SMALL_REQUESTS   36
+
+#define DEBUG_WORD(char)    (revision_t)(((char) *                      \
+                                      (WORD >= 8 ? 0x0101010101010101ULL \
+                                                 : 0x01010101ULL)))
+
+
+/* Linux's glibc is good at 'malloc(1023*WORD)': the blocks ("pages") it
+   returns are exactly 1024 words apart, reserving only one extra word
+   for its internal data.  Here we assume that even on other systems it
+   will not use more than three words. */
+#ifndef GC_PAGE_SIZE
+#define GC_PAGE_SIZE   (1021 * WORD)
+#endif
+
+/* This is the largest size that we will map to our internal "pages"
+   structures. */
+#define GC_SMALL_REQUEST_THRESHOLD   ((GC_SMALL_REQUESTS-1) * WORD)
+
+/* The minimum heap size: never run a full collection with a smaller heap. */
+#ifndef GC_MIN
+#define GC_MIN    (16 * 1024 * 1024)
+#endif
+
+/* The minimum expansion factor: never run a full collection before we have
+   allocated this much space. */
+#ifndef GC_EXPAND
+#define GC_EXPAND    (7 * 1024 * 1024)
+#endif
+
+/* The heap is collected when it reaches 1.82 times what it had before. */
+#define GC_MAJOR_COLLECT    1.82
+
+
+typedef struct page_header_s {
+    struct page_header_s *next_page;
+} page_header_t;
+
+
+/* These fields are in tx_public_descriptor rather than tx_descriptor.
+   The indirection allows us to keep around the lists of pages even
+   after the thread finishes, until the next major collection.
+*/
+#define GCPAGE_FIELDS_DECL                                              \
+    /* The array 'pages_for_size' contains GC_SMALL_REQUESTS            \
+       chained lists of pages currently managed by this thread.         \
+       For each size N between WORD and SMALL_REQUEST_THRESHOLD         \
+       (included), the corresponding chained list contains pages        \
+       which store objects of size N. */                                \
+    struct page_header_s *pages_for_size[GC_SMALL_REQUESTS];            \
+                                                                        \
+    /* This array contains GC_SMALL_REQUESTS chained lists of           \
+       free locations. */                                               \
+    gcptr free_loc_for_size[GC_SMALL_REQUESTS];                         \
+                                                                        \
+    /* For statistics */                                                \
+    uintptr_t count_pages;
+
+
+#define LOCAL_GCPAGES()  (thread_descriptor->public_descriptor)
+
+
+void stmgcpage_init_tls(void);
+void stmgcpage_done_tls(void);
+gcptr stmgcpage_malloc(size_t size);
+void stmgcpage_free(gcptr obj);
+void stmgcpage_add_prebuilt_root(gcptr obj);
+void stmgcpage_possibly_major_collect(int force);
+long stmgcpage_count(int quantity);
+
+extern struct GcPtrList stm_prebuilt_gcroots;
+
+#endif
diff --git a/c4/stmimpl.h b/c4/stmimpl.h
--- a/c4/stmimpl.h
+++ b/c4/stmimpl.h
@@ -30,6 +30,7 @@
 #include "fprintcolor.h"
 #include "lists.h"
 #include "dbgmem.h"
+#include "gcpage.h"
 #include "nursery.h"
 #include "et.h"
 #include "steal.h"
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -5,11 +5,11 @@
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 header_files = [os.path.join(parent_dir, _n) for _n in
-                "et.h lists.h steal.h nursery.h "
+                "et.h lists.h steal.h nursery.h gcpage.h "
                 "stmsync.h dbgmem.h fprintcolor.h "
                 "stmgc.h stmimpl.h atomic_ops.h".split()]
 source_files = [os.path.join(parent_dir, _n) for _n in
-                "et.c lists.c steal.c nursery.c "
+                "et.c lists.c steal.c nursery.c gcpage.c "
                 "stmsync.c dbgmem.c fprintcolor.c".split()]
 
 _pycache_ = os.path.join(parent_dir, 'test', '__pycache__')
@@ -57,7 +57,7 @@
     void *stm_malloc(size_t size);
     //gcptr stmgcpage_malloc(size_t size);
     //void stmgcpage_free(gcptr obj);
-    //long stmgcpage_count(int quantity);
+    long stmgcpage_count(int quantity);
     //void stmgcpage_possibly_major_collect(int);
     revision_t stm_global_cur_time(void);
     //void stmgcpage_add_prebuilt_root(gcptr);
@@ -116,7 +116,6 @@
 
     //extern gcptr stmgcpage_malloc(size_t size);
     //extern void stmgcpage_free(gcptr obj);
-    //extern long stmgcpage_count(int quantity);
     //extern void stmgcpage_possibly_major_collect(int);
     extern revision_t stm_global_cur_time(void);
     //extern void stmgcpage_add_prebuilt_root(gcptr);
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
new file mode 100644
--- /dev/null
+++ b/c4/test/test_gcpage.py
@@ -0,0 +1,34 @@
+import py
+from support import *
+
+
+def setup_function(f):
+    lib.stm_clear_between_tests()
+    lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
+
+def teardown_function(_):
+    lib.stm_finalize()
+
+
+def test_HDR():
+    import struct
+    assert HDR == struct.calcsize("PP")
+
+def test_malloc_simple():
+    assert count_pages() == 0
+    p1 = lib.stmgcpage_malloc(HDR)
+    print p1
+    p2 = lib.stmgcpage_malloc(HDR)
+    print p2
+    p3 = lib.stmgcpage_malloc(HDR)
+    print p3
+    assert count_pages() == 1
+    p4 = lib.stmgcpage_malloc(HDR + 1)
+    print p4
+    assert count_pages() == 2
+    p5 = lib.stmgcpage_malloc(HDR + 1)
+    print p5
+    assert distance(p1, p2) == HDR
+    assert distance(p2, p3) == HDR
+    assert abs(distance(p3, p4)) > PAGE_ROOM / 2
+    assert distance(p4, p5) == HDR + WORD

From noreply at buildbot.pypy.org  Mon Jun 17 13:35:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 13:35:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Copy some more code from c3.
Message-ID: <20130617113552.ED0531C1001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r166:e47c6c61565d
Date: 2013-06-17 13:35 +0200
http://bitbucket.org/pypy/stmgc/changeset/e47c6c61565d/

Log:	Copy some more code from c3.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1344,7 +1344,6 @@
 
 struct tx_public_descriptor *stm_descriptor_array[MAX_THREADS] = {0};
 static revision_t descriptor_array_free_list = 0;
-static revision_t descriptor_array_lock = 0;
 
 int DescriptorInit(void)
 {
@@ -1360,7 +1359,7 @@
       revision_t i;
       struct tx_descriptor *d = stm_malloc(sizeof(struct tx_descriptor));
       memset(d, 0, sizeof(struct tx_descriptor));
-      spinlock_acquire(descriptor_array_lock, 1);
+      stmgcpage_acquire_global_lock();
 
       struct tx_public_descriptor *pd;
       i = descriptor_array_free_list;
@@ -1398,7 +1397,8 @@
       fprintf(stderr, "[%lx] pthread %lx starting\n",
               (long)d->public_descriptor_index, (long)pthread_self());
 
-      spinlock_release(descriptor_array_lock);
+      stmgcpage_init_tls();
+      stmgcpage_release_global_lock();
       return 1;
     }
   else
@@ -1415,12 +1415,13 @@
     gcptrlist_delete(&d->public_descriptor->stolen_objects);
     gcptrlist_delete(&d->public_descriptor->stolen_young_stubs);
 
-    spinlock_acquire(descriptor_array_lock, 1);
+    stmgcpage_acquire_global_lock();
+    stmgcpage_done_tls();
     i = d->public_descriptor_index;
     assert(stm_descriptor_array[i] == d->public_descriptor);
     d->public_descriptor->free_list_next = descriptor_array_free_list;
     descriptor_array_free_list = i;
-    spinlock_release(descriptor_array_lock);
+    stmgcpage_release_global_lock();
 
     thread_descriptor = NULL;
 
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -1,6 +1,16 @@
 #include "stmimpl.h"
 
 
+/* This maps each small request size to the number of blocks of this size
+   that fit in a page. */
+static int nblocks_for_size[GC_SMALL_REQUESTS];
+
+/* A mutex for major collections and other global operations */
+static pthread_mutex_t mutex_gc_lock = PTHREAD_MUTEX_INITIALIZER;
+
+/* A count-down: when it reaches 0, run the next major collection */
+static revision_t countdown_next_major_coll = GC_MIN;
+
 /* For statistics */
 static uintptr_t count_global_pages;
 
@@ -13,3 +23,155 @@
     default: return -1;
     }
 }
+
+
+/***** Support code *****/
+
+void stmgcpage_acquire_global_lock(void)
+{
+    int err = pthread_mutex_lock(&mutex_gc_lock);
+    assert(err == 0);
+}
+
+void stmgcpage_release_global_lock(void)
+{
+    int err = pthread_mutex_unlock(&mutex_gc_lock);
+    assert(err == 0);
+}
+
+
+/***** Initialization logic *****/
+
+static void init_global_data(void)
+{
+    int i;
+    for (i = 1; i < GC_SMALL_REQUESTS; i++) {
+        nblocks_for_size[i] =
+            (GC_PAGE_SIZE - sizeof(page_header_t)) / (WORD * i);
+    }
+}
+
+void stmgcpage_init_tls(void)
+{
+    if (nblocks_for_size[1] == 0)
+        init_global_data();
+}
+
+void stmgcpage_done_tls(void)
+{
+    /* Send to the shared area all my pages.  For now we don't extract
+       the information about which locations are free or not; we just
+       leave it to the next major GC to figure them out. */
+#if 0
+    struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
+
+    gcp->gcp_next = finished_thread_gcpages;
+    finished_thread_gcpages = gcp;
+    count_global_pages += gcp->count_pages;*/
+#endif
+}
+
+
+/***** Thread-local allocator *****/
+
+void stmgcpage_reduce_threshold(size_t size)
+{
+    revision_t next, target;
+ restart:
+    next = ACCESS_ONCE(countdown_next_major_coll);
+    if (next >= size)
+        target = next - size;
+    else
+        target = 0;
+    if (!bool_cas(&countdown_next_major_coll, next, target))
+        goto restart;
+}
+
+static gcptr allocate_new_page(int size_class)
+{
+    /* Adjust the threshold; the caller is responsible for detecting the
+       condition that the threshold reached 0. */
+    stmgcpage_reduce_threshold(GC_PAGE_SIZE);
+
+    /* Allocate and return a new page for the given size_class. */
+    page_header_t *page = (page_header_t *)stm_malloc(GC_PAGE_SIZE);
+    if (!page) {
+        fprintf(stderr, "allocate_new_page: out of memory!\n");
+        abort();
+    }
+    struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
+    gcp->count_pages++;
+
+    /* Initialize the fields of the resulting page */
+    page->next_page = gcp->pages_for_size[size_class];
+    gcp->pages_for_size[size_class] = page;
+
+    /* Initialize the chained list in the page */
+    gcptr head = (gcptr)(page + 1);
+    gcptr current, next;
+    int count = nblocks_for_size[size_class];
+    int nsize = size_class * WORD;
+    int i;
+    current = head;
+    for (i = 0; i < count - 1; i++) {
+        next = (gcptr)(((char *)current) + nsize);
+        assert(!(GCFLAG_VISITED & DEBUG_WORD(0xDD)));
+        current->h_tid = DEBUG_WORD(0xDD);  /*anything without GCFLAG_VISITED*/
+        current->h_revision = (revision_t)next;
+        //stm_dbgmem_not_used(current, nsize, 0);
+        current = next;
+    }
+    current->h_tid = DEBUG_WORD(0xDD);
+    current->h_revision = (revision_t)gcp->free_loc_for_size[size_class];
+    //stm_dbgmem_not_used(current, nsize, 0);
+    gcp->free_loc_for_size[size_class] = head;
+    return head;
+}
+
+gcptr stmgcpage_malloc(size_t size)
+{
+    /* Allocates an object of the given 'size'.  This will never run
+       a collection: you need to call stmgcpage_possibly_major_collect(0)
+       when you know you're at a safe point. */
+    if (size <= GC_SMALL_REQUEST_THRESHOLD) {
+        gcptr result;
+        struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
+        int size_class = (size + WORD - 1) / WORD;
+        assert(0 < size_class && size_class < GC_SMALL_REQUESTS);
+
+        /* The result is simply 'free_loc_for_size[size_class]' */
+        result = gcp->free_loc_for_size[size_class];
+        if (!result) {
+            result = allocate_new_page(size_class);
+        }
+        gcp->free_loc_for_size[size_class] = (gcptr)result->h_revision;
+        //stm_dbgmem_used_again(result, size_class * WORD, 0);
+        return result;
+    }
+    else {
+        fprintf(stderr, "XXX stmgcpage_malloc: too big!\n");
+        abort();
+    }
+}
+
+static unsigned char random_char = 42;
+
+void stmgcpage_free(gcptr obj)
+{
+    size_t size = stmcb_size(obj);
+    if (size <= GC_SMALL_REQUEST_THRESHOLD) {
+        struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
+        int size_class = (size + WORD - 1) / WORD;
+        assert(0 < size_class && size_class < GC_SMALL_REQUESTS);
+
+        /* We simply re-add the object to the right chained list */
+        assert(obj->h_tid = DEBUG_WORD(random_char++));
+        obj->h_revision = (revision_t)gcp->free_loc_for_size[size_class];
+        gcp->free_loc_for_size[size_class] = obj;
+        //stm_dbgmem_not_used(obj, size_class * WORD, 0);
+    }
+    else {
+        fprintf(stderr, "XXX stmgcpage_free: too big!\n");
+        abort();
+    }
+}
diff --git a/c4/gcpage.h b/c4/gcpage.h
--- a/c4/gcpage.h
+++ b/c4/gcpage.h
@@ -13,7 +13,8 @@
 /* Linux's glibc is good at 'malloc(1023*WORD)': the blocks ("pages") it
    returns are exactly 1024 words apart, reserving only one extra word
    for its internal data.  Here we assume that even on other systems it
-   will not use more than three words. */
+   will not use more than three words.  It results in pages of 1020
+   WORDs, which is a number that divides well by many small values. */
 #ifndef GC_PAGE_SIZE
 #define GC_PAGE_SIZE   (1021 * WORD)
 #endif
@@ -65,8 +66,10 @@
 #define LOCAL_GCPAGES()  (thread_descriptor->public_descriptor)
 
 
-void stmgcpage_init_tls(void);
-void stmgcpage_done_tls(void);
+void stmgcpage_acquire_global_lock(void);
+void stmgcpage_release_global_lock(void);
+void stmgcpage_init_tls(void);   /* must hold the global lock */
+void stmgcpage_done_tls(void);   /* must hold the global lock */
 gcptr stmgcpage_malloc(size_t size);
 void stmgcpage_free(gcptr obj);
 void stmgcpage_add_prebuilt_root(gcptr obj);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -55,8 +55,8 @@
 
     /* extra non-public code */
     void *stm_malloc(size_t size);
-    //gcptr stmgcpage_malloc(size_t size);
-    //void stmgcpage_free(gcptr obj);
+    gcptr stmgcpage_malloc(size_t size);
+    void stmgcpage_free(gcptr obj);
     long stmgcpage_count(int quantity);
     //void stmgcpage_possibly_major_collect(int);
     revision_t stm_global_cur_time(void);
@@ -107,7 +107,7 @@
     #define GCFLAG_STUB              ...
     #define GCFLAG_PRIVATE_FROM_PROTECTED  ...
     #define ABRT_MANUAL              ...
-    //typedef struct { ...; } page_header_t;
+    typedef struct { ...; } page_header_t;
 ''')
 
 lib = ffi.verify(r'''
@@ -266,7 +266,7 @@
 
 HDR = ffi.sizeof("struct stm_object_s")
 WORD = lib.WORD
-#PAGE_ROOM = lib.GC_PAGE_SIZE - ffi.sizeof("page_header_t")
+PAGE_ROOM = lib.GC_PAGE_SIZE - ffi.sizeof("page_header_t")
 for name in lib.__dict__:
     if name.startswith('GCFLAG_') or name.startswith('PREBUILT_'):
         globals()[name] = getattr(lib, name)

From noreply at buildbot.pypy.org  Mon Jun 17 13:54:12 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 13:54:12 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617115412.8B8141C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r167:34b9dddf5a8c
Date: 2013-06-17 13:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/34b9dddf5a8c/

Log:	Next test

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1345,6 +1345,14 @@
 struct tx_public_descriptor *stm_descriptor_array[MAX_THREADS] = {0};
 static revision_t descriptor_array_free_list = 0;
 
+void _stm_test_forget_previous_state(void)
+{
+  fprintf(stderr, "=======================================================\n");
+  assert(thread_descriptor == NULL);
+  memset(stm_descriptor_array, 0, sizeof(stm_descriptor_array));
+  descriptor_array_free_list = 0;
+}
+
 int DescriptorInit(void)
 {
   if (GCFLAG_PREBUILT != PREBUILT_FLAGS)
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -181,6 +181,7 @@
 gcptr stm_get_private_from_protected(long);  /* debugging */
 gcptr stm_get_read_obj(long);  /* debugging */
 void stm_clear_read_cache(void);  /* debugging */
+void _stm_test_forget_previous_state(void);  /* debugging */
 
 int DescriptorInit(void);
 void DescriptorDone(void);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -90,6 +90,7 @@
     void *my_stub_thread(void);
 
     int _stm_can_access_memory(char *);
+    void stm_initialize_and_set_max_abort(int max_aborts);
     void stm_initialize_tests(int max_aborts);
 
     /* some constants normally private that are useful in the tests */
@@ -219,12 +220,18 @@
         return (void *)thread_descriptor->public_descriptor;
     }
 
-    void stm_initialize_tests(int max_aborts)
+    void stm_initialize_and_set_max_abort(int max_aborts)
     {
         stm_initialize();
         stm_set_max_aborts(max_aborts);
     }
 
+    void stm_initialize_tests(int max_aborts)
+    {
+        _stm_test_forget_previous_state();
+        stm_initialize_and_set_max_abort(max_aborts);
+    }
+
     size_t stmcb_size(gcptr obj)
     {
         if (gettid(obj) < 421) {
@@ -320,7 +327,7 @@
     def run(self, fn, lck):
         try:
             try:
-                lib.stm_initialize_tests(self.max_aborts)
+                lib.stm_initialize_and_set_max_abort(self.max_aborts)
                 try:
                     fn(self)
                 finally:
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -32,3 +32,14 @@
     assert distance(p2, p3) == HDR
     assert abs(distance(p3, p4)) > PAGE_ROOM / 2
     assert distance(p4, p5) == HDR + WORD
+
+def test_malloc_page_full():
+    plist = []
+    for i in range(PAGE_ROOM // HDR):
+        plist.append(lib.stmgcpage_malloc(HDR))
+    for p1, p2 in zip(plist[:-1], plist[1:]):
+        assert distance(p1, p2) == HDR
+    assert count_pages() == 1
+    p = lib.stmgcpage_malloc(HDR)
+    assert distance(plist[-1], p) != HDR
+    assert count_pages() == 2
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -362,7 +362,7 @@
                 p = self.pop_roots(extra=p)
 
     def run_single_thread(self):
-        lib.stm_initialize_tests(0)
+        lib.stm_initialize_and_set_max_abort(0)
         self.interruptible_transaction = False
         self.startrev()
         #

From noreply at buildbot.pypy.org  Mon Jun 17 14:22:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 14:22:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617122252.4C5881C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r168:37c581d1aab8
Date: 2013-06-17 14:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/37c581d1aab8/

Log:	Next test

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1347,10 +1347,12 @@
 
 void _stm_test_forget_previous_state(void)
 {
+  /* debug: reset all global states, between tests */
   fprintf(stderr, "=======================================================\n");
   assert(thread_descriptor == NULL);
   memset(stm_descriptor_array, 0, sizeof(stm_descriptor_array));
   descriptor_array_free_list = 0;
+  stmgcpage_count(2);
 }
 
 int DescriptorInit(void)
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -12,7 +12,7 @@
 static revision_t countdown_next_major_coll = GC_MIN;
 
 /* For statistics */
-static uintptr_t count_global_pages;
+static revision_t count_global_pages;
 
 /* For tests */
 long stmgcpage_count(int quantity)
@@ -20,6 +20,7 @@
     switch (quantity) {
     case 0: return count_global_pages;
     case 1: return LOCAL_GCPAGES()->count_pages;
+    case 2: count_global_pages = 0; return 0;
     default: return -1;
     }
 }
@@ -62,13 +63,8 @@
     /* Send to the shared area all my pages.  For now we don't extract
        the information about which locations are free or not; we just
        leave it to the next major GC to figure them out. */
-#if 0
     struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
-
-    gcp->gcp_next = finished_thread_gcpages;
-    finished_thread_gcpages = gcp;
-    count_global_pages += gcp->count_pages;*/
-#endif
+    count_global_pages += gcp->count_pages;
 }
 
 
diff --git a/c4/gcpage.h b/c4/gcpage.h
--- a/c4/gcpage.h
+++ b/c4/gcpage.h
@@ -60,7 +60,7 @@
     gcptr free_loc_for_size[GC_SMALL_REQUESTS];                         \
                                                                         \
     /* For statistics */                                                \
-    uintptr_t count_pages;
+    revision_t count_pages;
 
 
 #define LOCAL_GCPAGES()  (thread_descriptor->public_descriptor)
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -115,8 +115,6 @@
     #include "stmgc.h"
     #include "stmimpl.h"
 
-    //extern gcptr stmgcpage_malloc(size_t size);
-    //extern void stmgcpage_free(gcptr obj);
     //extern void stmgcpage_possibly_major_collect(int);
     extern revision_t stm_global_cur_time(void);
     //extern void stmgcpage_add_prebuilt_root(gcptr);
@@ -408,7 +406,7 @@
 
 def oalloc(size):
     "Allocate an 'old' protected object, outside any nursery"
-    p = ffi.cast("gcptr", lib.stm_malloc(size))
+    p = lib.stmgcpage_malloc(size)
     p.h_tid = GCFLAG_OLD | GCFLAG_WRITE_BARRIER
     p.h_revision = -sys.maxint
     lib.settid(p, 42 + size)
@@ -417,7 +415,7 @@
 def oalloc_refs(nrefs):
     """Allocate an 'old' protected object, outside any nursery,
     with nrefs pointers"""
-    p = ffi.cast("gcptr", lib.stm_malloc(HDR + WORD * nrefs))
+    p = lib.stmgcpage_malloc(HDR + WORD * nrefs)
     p.h_tid = GCFLAG_OLD | GCFLAG_WRITE_BARRIER
     p.h_revision = -sys.maxint
     lib.settid(p, 421 + nrefs)
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -43,3 +43,16 @@
     p = lib.stmgcpage_malloc(HDR)
     assert distance(plist[-1], p) != HDR
     assert count_pages() == 2
+
+def test_thread_local_malloc():
+    assert count_global_pages() == 0
+    where = []
+    def f1(r):
+        where.append(oalloc(HDR))
+    def f2(r):
+        where.append(oalloc(HDR))
+    run_parallel(f1, f2)
+    assert count_pages() == 0
+    assert count_global_pages() == 2
+    p3, p4 = where
+    assert abs(distance(p3, p4)) > PAGE_ROOM / 2

From noreply at buildbot.pypy.org  Mon Jun 17 14:22:53 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 14:22:53 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Move to the test runner a general fix
 for thread-order problems.
Message-ID: <20130617122253.93B851C300E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r169:88201fa26d98
Date: 2013-06-17 14:17 +0200
http://bitbucket.org/pypy/stmgc/changeset/88201fa26d98/

Log:	Move to the test runner a general fix for thread-order problems.

diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -308,11 +308,15 @@
         self.parallel_locks = (thread.allocate_lock(), thread.allocate_lock())
         self.parallel_locks[0].acquire()
         self.resulting_exception = None
+        self.start_locks = []
+        for fn in fns:
+            self.start_locks.append(thread.allocate_lock())
+            self.start_locks[-1].acquire()
         locks = []
-        for fn in fns:
+        for i, fn in enumerate(fns):
             lck = thread.allocate_lock()
             lck.acquire()
-            thread.start_new_thread(self.run, (fn, lck))
+            thread.start_new_thread(self.run, (fn, lck, i))
             locks.append(lck)
         for lck in locks:
             lck.acquire()
@@ -322,11 +326,17 @@
             exc, val, tb = self.resulting_exception
             raise exc, val, tb
 
-    def run(self, fn, lck):
+    def run(self, fn, lck, i):
         try:
             try:
                 lib.stm_initialize_and_set_max_abort(self.max_aborts)
                 try:
+                    # wait here until all threads reach this point
+                    self.start_locks[i].release()
+                    for _lck1 in self.start_locks:
+                        acquire_lock(_lck1)
+                        _lck1.release()
+                    #
                     fn(self)
                 finally:
                     lib.stm_finalize()
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -407,10 +407,8 @@
         assert classify(follow_revision(p)) == "stub"
         assert p1.h_revision & 1
         r.set(2)
-        r.wait(3)   # wait until the other thread really starts
     def f2(r):
         r.wait(2)
-        r.set(3)
         p2 = lib.stm_read_barrier(p)    # steals
         assert classify(p2) == "public"
         q2 = lib.getptr(p2, 0)

From noreply at buildbot.pypy.org  Mon Jun 17 14:22:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 14:22:54 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617122254.D4A8F1C300E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r170:e22d77eb0b4f
Date: 2013-06-17 14:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/e22d77eb0b4f/

Log:	Next test

diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -433,6 +433,8 @@
         rawsetptr(p, i, ffi.NULL)
     return p
 
+ofree = lib.stmgcpage_free
+
 def nalloc(size):
     "Allocate a fresh object from the nursery"
     p = lib.stm_allocate(size, 42 + size)
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -56,3 +56,9 @@
     assert count_global_pages() == 2
     p3, p4 = where
     assert abs(distance(p3, p4)) > PAGE_ROOM / 2
+
+def test_malloc_reuse():
+    p1 = oalloc(HDR)
+    ofree(p1)
+    p2 = oalloc(HDR)
+    assert p2 == p1

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:12 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: introduce newlist_unicode for wrapping
	lists of unicode
Message-ID: <20130617123512.27F9B1C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64900:de52390077e2
Date: 2013-06-13 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/de52390077e2/

Log:	introduce newlist_unicode for wrapping lists of unicode

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -904,6 +904,9 @@
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
+    def newlist_unicode(self, list_u):
+        return self.newlist([self.wrap(u) for u in list_u])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -166,6 +166,12 @@
         storage = strategy.erase(list_s)
         return W_ListObject.from_storage_and_strategy(space, storage, strategy)
 
+    @staticmethod
+    def newlist_unicode(space, list_u):
+        strategy = space.fromcache(UnicodeListStrategy)
+        storage = strategy.erase(list_u)
+        return W_ListObject.from_storage_and_strategy(space, storage, strategy)
+
     def __repr__(self):
         """ representation for debugging purposes """
         return "%s(%s, %s)" % (self.__class__.__name__, self.strategy,
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -294,6 +294,9 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
+    def newlist_unicode(self, list_u):
+        return W_ListObject.newlist_unicode(self, list_u)
+
     def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -560,6 +560,18 @@
         assert space.listview_str(w_l) == ["a", "b", "c"]
         assert space.listview_str(w_l2) == ["a", "b", "c"]
 
+    def test_unicode_uses_newlist_unicode(self):
+        space = self.space
+        w_u = space.wrap(u"a b c")
+        space.newlist = None
+        try:
+            w_l = space.call_method(w_u, "split")
+            w_l2 = space.call_method(w_u, "split", space.wrap(" "))
+        finally:
+            del space.newlist
+        assert space.listview_unicode(w_l) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l2) == [u"a", u"b", u"c"]
+
     def test_pop_without_argument_is_fast(self):
         space = self.space
         w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -650,7 +650,7 @@
 
 def unicode_split__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     length = len(value)
     i = 0
@@ -673,12 +673,12 @@
             maxsplit -= 1   # NB. if it's already < 0, it stays < 0
 
         # the word is value[i:j]
-        res_w.append(W_UnicodeObject(value[i:j]))
+        res.append(value[i:j])
 
         # continue to look from the character following the space after the word
         i = j + 1
 
-    return space.newlist(res_w)
+    return space.newlist_unicode(res)
 
 def unicode_split__Unicode_Unicode_ANY(space, w_self, w_delim, w_maxsplit):
     self = w_self._value
@@ -689,12 +689,12 @@
         raise OperationError(space.w_ValueError,
                              space.wrap('empty separator'))
     parts = _split_with(self, delim, maxsplit)
-    return space.newlist([W_UnicodeObject(part) for part in parts])
+    return space.newlist_unicode(parts)
 
 
 def unicode_rsplit__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -719,13 +719,13 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(W_UnicodeObject(value[j1:i+1]))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_unicode(res)
 
 def sliced(space, s, start, stop, orig_obj):
     assert start >= 0

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:15 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <20130617123515.575911C3334@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64902:d0cfac47ecad
Date: 2013-06-13 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/d0cfac47ecad/

Log:	merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,43 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+from .tklib import tklib, tkffi
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {			/* The internal representation: */
+	long longValue;		/*   - an long integer value. */
+	double doubleValue;	/*   - a double-precision floating value. */
+	struct {		/*   - internal rep as two pointers. */
+	    void *ptr1;
+	    void *ptr2;
+	} twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:14 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130617123514.095B41C332F@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3k
Changeset: r64901:df0e09e9a5a7
Date: 2013-06-13 18:24 +0200
http://bitbucket.org/pypy/pypy/changeset/df0e09e9a5a7/

Log:	merge default

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,9 @@
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
+    def newlist_unicode(self, list_u):
+        return self.newlist([self.wrap(u) for u in list_u])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py
--- a/pypy/module/thread/test/test_fork.py
+++ b/pypy/module/thread/test/test_fork.py
@@ -30,7 +30,7 @@
                 if pid == 0:
                     os._exit(0)
                 else:
-                    self.timeout_killer(pid, 5)
+                    self.timeout_killer(pid, 10)
                     exitcode = os.waitpid(pid, 0)[1]
                     assert exitcode == 0 # if 9, process was killed by timer!
             finally:
@@ -56,7 +56,7 @@
                 _thread.start_new_thread(lambda: None, ())
                 os._exit(0)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 assert exitcode == 0 # if 9, process was killed by timer!
 
@@ -75,7 +75,7 @@
                 signal.signal(signal.SIGUSR1, signal.SIG_IGN)
                 os._exit(42)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 feedback.append(exitcode)
 
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -171,6 +171,12 @@
         storage = strategy.erase(list_s)
         return W_ListObject.from_storage_and_strategy(space, storage, strategy)
 
+    @staticmethod
+    def newlist_unicode(space, list_u):
+        strategy = space.fromcache(UnicodeListStrategy)
+        storage = strategy.erase(list_u)
+        return W_ListObject.from_storage_and_strategy(space, storage, strategy)
+
     def __repr__(self):
         """ representation for debugging purposes """
         return "%s(%s, %s)" % (self.__class__.__name__, self.strategy,
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -317,6 +317,9 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
+    def newlist_unicode(self, list_u):
+        return W_ListObject.newlist_unicode(self, list_u)
+
     def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -565,6 +565,18 @@
         assert space.listview_str(w_l) == ["a", "b", "c"]
         assert space.listview_str(w_l2) == ["a", "b", "c"]
 
+    def test_unicode_uses_newlist_unicode(self):
+        space = self.space
+        w_u = space.wrap(u"a b c")
+        space.newlist = None
+        try:
+            w_l = space.call_method(w_u, "split")
+            w_l2 = space.call_method(w_u, "split", space.wrap(" "))
+        finally:
+            del space.newlist
+        assert space.listview_unicode(w_l) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l2) == [u"a", u"b", u"c"]
+
     def test_pop_without_argument_is_fast(self):
         space = self.space
         w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -646,7 +646,7 @@
         # continue to look from the character following the space after the word
         i = j + 1
 
-    return space.newlist_str(res)
+    return space.newlist_unicode(res)
 
 def unicode_split__Unicode_Unicode_ANY(space, w_self, w_delim, w_maxsplit):
     self = w_self._value
@@ -657,12 +657,12 @@
         raise OperationError(space.w_ValueError,
                              space.wrap('empty separator'))
     parts = _split_with(self, delim, maxsplit)
-    return space.newlist_str(parts)
+    return space.newlist_unicode(parts)
 
 
 def unicode_rsplit__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -687,13 +687,13 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(W_UnicodeObject(value[j1:i+1]))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_unicode(res)
 
 def sliced(space, s, start, stop, orig_obj):
     assert start >= 0
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -113,7 +113,7 @@
         tmpreg = r.lr
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None: # store
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.load_reg(mc, excvalloc, r.ip)
         if on_frame:
             # store exc_value in JITFRAME
@@ -125,7 +125,7 @@
             self.store_reg(mc, r.ip, r.fp, ofs, helper=tmpreg)
         if exctploc is not None:
             # store pos_exception in exctploc
-            assert exctploc.is_reg()
+            assert exctploc.is_core_reg()
             mc.gen_load_int(r.ip.value, self.cpu.pos_exception())
             self.load_reg(mc, exctploc, r.ip, helper=tmpreg)
 
@@ -146,7 +146,7 @@
         tmpreg = r.lr # use lr as a second temporary reg
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None:
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.store_reg(mc, excvalloc, r.ip)
         else:
             assert exctploc is not r.fp
@@ -947,7 +947,7 @@
     # regalloc support
     def load(self, loc, value):
         """load an immediate value into a register"""
-        assert (loc.is_reg() and value.is_imm()
+        assert (loc.is_core_reg() and value.is_imm()
                     or loc.is_vfp_reg() and value.is_imm_float())
         if value.is_imm():
             self.mc.gen_load_int(loc.value, value.getint())
@@ -958,7 +958,7 @@
     def load_reg(self, mc, target, base, ofs=0, cond=c.AL, helper=r.ip):
         if target.is_vfp_reg():
             return self._load_vfp_reg(mc, target, base, ofs, cond, helper)
-        elif target.is_reg():
+        elif target.is_core_reg():
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
@@ -1012,7 +1012,7 @@
     def _mov_imm_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.type == FLOAT:
             raise AssertionError("invalid target for move from imm value")
-        if loc.is_reg():
+        if loc.is_core_reg():
             new_loc = loc
         elif loc.is_stack() or loc.is_raw_sp():
             new_loc = r.lr
@@ -1027,7 +1027,7 @@
     def _mov_reg_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.is_imm():
             raise AssertionError("mov reg to imm doesn't make sense")
-        if loc.is_reg():
+        if loc.is_core_reg():
             self.mc.MOV_rr(loc.value, prev_loc.value, cond=cond)
         elif loc.is_stack() and loc.type != FLOAT:
             # spill a core register
@@ -1050,7 +1050,7 @@
         helper = None
         offset = prev_loc.value
         tmp = None
-        if loc.is_reg():
+        if loc.is_core_reg():
             assert prev_loc.type != FLOAT, 'trying to load from an \
                 incompatible location into a core register'
             # unspill a core register
@@ -1126,7 +1126,7 @@
         """Moves a value from a previous location to some other location"""
         if prev_loc.is_imm():
             return self._mov_imm_to_loc(prev_loc, loc, cond)
-        elif prev_loc.is_reg():
+        elif prev_loc.is_core_reg():
             self._mov_reg_to_loc(prev_loc, loc, cond)
         elif prev_loc.is_stack():
             self._mov_stack_to_loc(prev_loc, loc, cond)
@@ -1215,7 +1215,7 @@
                 scratch_reg = r.vfp_ip
             self.regalloc_mov(loc, scratch_reg, cond)
             self.regalloc_push(scratch_reg, cond)
-        elif loc.is_reg():
+        elif loc.is_core_reg():
             self.mc.PUSH([loc.value], cond=cond)
         elif loc.is_vfp_reg():
             self.mc.VPUSH([loc.value], cond=cond)
@@ -1238,7 +1238,7 @@
                 scratch_reg = r.vfp_ip
             self.regalloc_pop(scratch_reg)
             self.regalloc_mov(scratch_reg, loc)
-        elif loc.is_reg():
+        elif loc.is_core_reg():
             self.mc.POP([loc.value], cond=cond)
         elif loc.is_vfp_reg():
             self.mc.VPOP([loc.value], cond=cond)
@@ -1306,7 +1306,7 @@
 
         # lengthloc is the length of the array, which we must not modify!
         assert lengthloc is not r.r0 and lengthloc is not r.r1
-        if lengthloc.is_reg():
+        if lengthloc.is_core_reg():
             varsizeloc = lengthloc
         else:
             assert lengthloc.is_stack()
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -40,7 +40,7 @@
         if self.fnloc.is_stack():
             self.asm.mov_loc_loc(self.fnloc, r.ip)
             self.fnloc = r.ip
-        assert self.fnloc.is_reg()
+        assert self.fnloc.is_core_reg()
         self.mc.BLX(self.fnloc.value)
 
     def restore_stack_pointer(self):
@@ -135,7 +135,7 @@
             return [], []
         if self.resloc.is_vfp_reg():
             return [r.r0, r.r1], []
-        assert self.resloc.is_reg()
+        assert self.resloc.is_core_reg()
         return [r.r0], []
 
     def load_result(self):
@@ -146,7 +146,7 @@
         if resloc.is_vfp_reg():
             # move result to the allocated register
             self.asm.mov_to_vfp_loc(r.r0, r.r1, resloc)
-        elif resloc.is_reg():
+        elif resloc.is_core_reg():
             # move result to the allocated register
             if resloc is not r.r0:
                 self.asm.mov_loc_loc(r.r0, resloc)
@@ -283,7 +283,7 @@
     def load_result(self):
         resloc = self.resloc
         # ensure the result is wellformed and stored in the correct location
-        if resloc is not None and resloc.is_reg():
+        if resloc is not None and resloc.is_core_reg():
             self._ensure_result_bit_extension(resloc,
                                                   self.ressize, self.ressign)
 
@@ -292,7 +292,7 @@
             return [], []
         if self.resloc.is_vfp_reg():
             return [], [r.d0]
-        assert self.resloc.is_reg()
+        assert self.resloc.is_core_reg()
         return [r.r0], []
 
 
diff --git a/rpython/jit/backend/arm/helper/assembler.py b/rpython/jit/backend/arm/helper/assembler.py
--- a/rpython/jit/backend/arm/helper/assembler.py
+++ b/rpython/jit/backend/arm/helper/assembler.py
@@ -82,7 +82,7 @@
         assert guard is not None
         l0 = arglocs[0]
         l1 = arglocs[1]
-        assert l0.is_reg()
+        assert l0.is_core_reg()
 
         if l1.is_imm():
             self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond)
diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py
--- a/rpython/jit/backend/arm/locations.py
+++ b/rpython/jit/backend/arm/locations.py
@@ -15,7 +15,7 @@
     def is_raw_sp(self):
         return False
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def is_vfp_reg(self):
@@ -43,7 +43,7 @@
     def __repr__(self):
         return 'r%d' % self.value
 
-    def is_reg(self):
+    def is_core_reg(self):
         return True
 
     def as_key(self):
@@ -62,7 +62,7 @@
     def __repr__(self):
         return 'vfp%d' % self.value
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def is_vfp_reg(self):
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -248,7 +248,7 @@
         l1 = arglocs[1]
         failargs = arglocs[2:]
 
-        if l0.is_reg():
+        if l0.is_core_reg():
             if l1.is_imm():
                 self.mc.CMP_ri(l0.value, l1.getint())
             else:
@@ -488,7 +488,7 @@
             # case GCFLAG_CARDS_SET: emit a few instructions to do
             # directly the card flag setting
             loc_index = arglocs[1]
-            assert loc_index.is_reg()
+            assert loc_index.is_core_reg()
             # must save the register loc_index before it is mutated
             mc.PUSH([loc_index.value])
             tmp1 = loc_index
@@ -588,7 +588,7 @@
 
     def emit_op_setarrayitem_gc(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         if scale.value > 0:
             self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale.value)
             ofs_loc = r.ip
@@ -606,7 +606,7 @@
             # vstr only supports imm offsets
             # so if the ofset is too large we add it to the base and use an
             # offset of 0
-            if ofs_loc.is_reg():
+            if ofs_loc.is_core_reg():
                 tmploc, save = self.get_tmp_reg([value_loc, base_loc, ofs_loc])
                 assert not save
                 self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
@@ -644,13 +644,13 @@
 
     def emit_op_raw_store(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         self._write_to_mem(value_loc, base_loc, ofs_loc, scale, fcond)
         return fcond
 
     def emit_op_getarrayitem_gc(self, op, arglocs, regalloc, fcond):
         res_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         signed = op.getdescr().is_item_signed()
 
         # scale the offset as required
@@ -672,7 +672,7 @@
             # vldr only supports imm offsets
             # if the offset is in a register we add it to the base and use a
             # tmp reg
-            if ofs_loc.is_reg():
+            if ofs_loc.is_core_reg():
                 tmploc, save = self.get_tmp_reg([base_loc, ofs_loc])
                 assert not save
                 self.mc.ADD_rr(tmploc.value, base_loc.value, ofs_loc.value)
@@ -727,7 +727,7 @@
 
     def emit_op_raw_load(self, op, arglocs, regalloc, fcond):
         res_loc, base_loc, ofs_loc, scale, ofs = arglocs
-        assert ofs_loc.is_reg()
+        assert ofs_loc.is_core_reg()
         # no base offset
         assert ofs.value == 0
         signed = op.getdescr().is_item_signed()
@@ -805,10 +805,10 @@
             bytes_box = TempBox()
             bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, forbidden_vars)
             scale = self._get_unicode_item_scale()
-            if not length_loc.is_reg():
+            if not length_loc.is_core_reg():
                 self.regalloc_mov(length_loc, bytes_loc)
                 length_loc = bytes_loc
-            assert length_loc.is_reg()
+            assert length_loc.is_core_reg()
             self.mc.MOV_ri(r.ip.value, 1 << scale)
             self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value)
             length_box = bytes_box
@@ -835,8 +835,8 @@
 
    # result = base_loc  + (scaled_loc << scale) + static_offset
     def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0):
-        assert scaled_loc.is_reg()
-        assert base_loc.is_reg()
+        assert scaled_loc.is_core_reg()
+        assert base_loc.is_core_reg()
         assert check_imm_arg(scale)
         assert check_imm_arg(static_offset)
         if scale > 0:
@@ -1063,7 +1063,7 @@
     def emit_op_cast_float_to_int(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert arg.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VCVT_float_to_int(r.vfp_ip.value, arg.value)
         self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
@@ -1071,7 +1071,7 @@
     def emit_op_cast_int_to_float(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert res.is_vfp_reg()
-        assert arg.is_reg()
+        assert arg.is_core_reg()
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
         self.mc.VCVT_int_to_float(res.value, res.value)
@@ -1087,7 +1087,7 @@
         loc = arglocs[0]
         res = arglocs[1]
         assert loc.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VMOV_rc(res.value, r.ip.value, loc.value)
         return fcond
 
@@ -1108,7 +1108,7 @@
     def emit_op_cast_float_to_singlefloat(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert arg.is_vfp_reg()
-        assert res.is_reg()
+        assert res.is_core_reg()
         self.mc.VCVT_f64_f32(r.vfp_ip.value, arg.value)
         self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
@@ -1116,7 +1116,7 @@
     def emit_op_cast_singlefloat_to_float(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert res.is_vfp_reg()
-        assert arg.is_reg()
+        assert arg.is_core_reg()
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
         self.mc.VCVT_f32_f64(res.value, res.value)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -324,7 +324,7 @@
                 loc = r.fp
             arg = inputargs[i]
             i += 1
-            if loc.is_reg():
+            if loc.is_core_reg():
                 self.rm.reg_bindings[arg] = loc
                 used[loc] = None
             elif loc.is_vfp_reg():
@@ -346,6 +346,8 @@
         # note: we need to make a copy of inputargs because possibly_free_vars
         # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
+        self.fm.finish_binding()
+        self._check_invariants()
 
     def get_gcmap(self, forbidden_regs=[], noregs=False):
         frame_depth = self.fm.get_frame_depth()
@@ -356,7 +358,7 @@
                 continue
             if box.type == REF and self.rm.is_still_alive(box):
                 assert not noregs
-                assert loc.is_reg()
+                assert loc.is_core_reg()
                 val = loc.value
                 gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8))
         for box, loc in self.fm.bindings.iteritems():
@@ -1152,7 +1154,7 @@
             assert isinstance(arg, Box)
             loc = self.loc(arg)
             arglocs[i] = loc
-            if loc.is_reg():
+            if loc.is_core_reg() or loc.is_vfp_reg():
                 self.frame_manager.mark_as_free(arg)
         #
         descr._arm_arglocs = arglocs
diff --git a/rpython/jit/backend/arm/test/test_jump.py b/rpython/jit/backend/arm/test/test_jump.py
--- a/rpython/jit/backend/arm/test/test_jump.py
+++ b/rpython/jit/backend/arm/test/test_jump.py
@@ -255,7 +255,7 @@
                 else:
                     newvalue = 'value-vfp-%d' % i
                 regs2[loc.value] = newvalue
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 regs1[loc.value] = 'value-int-%d' % i
             elif loc.is_stack():
                 stack[loc.position] = 'value-width%d-%d' % (loc.width, i)
@@ -284,7 +284,7 @@
                 assert loc.width == expected_width*WORD
             if loc.is_vfp_reg():
                 return regs2[loc.value]
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 return regs1[loc.value]
             elif loc.is_stack():
                 got = stack[loc.position]
@@ -298,7 +298,7 @@
         def write(loc, newvalue):
             if loc.is_vfp_reg():
                 regs2[loc.value] = newvalue
-            elif loc.is_reg():
+            elif loc.is_core_reg():
                 regs1[loc.value] = newvalue
             elif loc.is_stack():
                 if loc.width > WORD:
@@ -317,17 +317,17 @@
         for op in assembler.ops:
             if op[0] == 'mov':
                 src, dst = op[1:]
-                assert src.is_reg() or src.is_vfp_reg() or src.is_stack() or src.is_imm_float() or src.is_imm()
-                assert dst.is_reg() or dst.is_vfp_reg() or dst.is_stack()
+                assert src.is_core_reg() or src.is_vfp_reg() or src.is_stack() or src.is_imm_float() or src.is_imm()
+                assert dst.is_core_reg() or dst.is_vfp_reg() or dst.is_stack()
                 assert not (src.is_stack() and dst.is_stack())
                 write(dst, read(src))
             elif op[0] == 'push':
                 src, = op[1:]
-                assert src.is_reg() or src.is_vfp_reg() or src.is_stack()
+                assert src.is_core_reg() or src.is_vfp_reg() or src.is_stack()
                 extrapushes.append(read(src))
             elif op[0] == 'pop':
                 dst, = op[1:]
-                assert dst.is_reg() or dst.is_vfp_reg() or dst.is_stack()
+                assert dst.is_core_reg() or dst.is_vfp_reg() or dst.is_stack()
                 write(dst, extrapushes.pop())
             else:
                 assert 0, "unknown op: %r" % (op,)
diff --git a/rpython/jit/backend/arm/test/test_runner.py b/rpython/jit/backend/arm/test/test_runner.py
--- a/rpython/jit/backend/arm/test/test_runner.py
+++ b/rpython/jit/backend/arm/test/test_runner.py
@@ -11,6 +11,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import JitCellToken, TargetToken
 from rpython.jit.backend.arm.detect import detect_arch_version
+from rpython.jit.codewriter import longlong
 
 
 CPU = getcpuclass()
@@ -261,3 +262,43 @@
         l1 = ('debug_print', preambletoken.repr_of_descr() + ':1')
         l2 = ('debug_print', targettoken.repr_of_descr() + ':9')
         assert ('jit-backend-counts', [l0, l1, l2]) in dlog
+
+
+    def test_label_float_in_reg_and_on_stack(self):
+        targettoken = TargetToken()
+        ops = """
+        [i0, f3]
+        i2 = same_as(i0)    # but forced to be in a register
+        force_spill(i2)
+        force_spill(f3)
+        f4 = float_add(f3, 5.0)
+        label(f3, f4, descr=targettoken)
+        force_spill(f3)
+        f5 = same_as(f3)    # but forced to be in a register
+        finish(f5)
+        """
+        faildescr = BasicFailDescr(2)
+        loop = parse(ops, self.cpu, namespace=locals())
+        looptoken = JitCellToken()
+        info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        ops2 = """
+        [i0, f1]
+        i1 = same_as(i0)
+        f2 = same_as(f1)
+        f3 = float_add(f1, 10.0)
+        force_spill(f3)
+        force_spill(i1)
+        f4 = float_add(f3, f1)
+        jump(f3, f4, descr=targettoken)
+        """
+        loop2 = parse(ops2, self.cpu, namespace=locals())
+        looptoken2 = JitCellToken()
+        info = self.cpu.compile_loop(loop2.inputargs, loop2.operations, looptoken2)
+
+        deadframe = self.cpu.execute_token(looptoken, -9, longlong.getfloatstorage(-13.5))
+        res = longlong.getrealfloat(self.cpu.get_float_value(deadframe, 0))
+        assert res == -13.5
+        #
+        deadframe = self.cpu.execute_token(looptoken2, -9, longlong.getfloatstorage(-13.5))
+        res = longlong.getrealfloat(self.cpu.get_float_value(deadframe, 0))
+        assert res == -3.5
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -47,7 +47,7 @@
             input_i += 1
             if arg.type == REF:
                 loc = fail_locs[i]
-                if loc.is_reg():
+                if loc.is_core_reg():
                     val = self.cpu.all_reg_indexes[loc.value]
                 else:
                     val = loc.get_position() + self.cpu.JITFRAME_FIXED_SIZE
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1551,14 +1551,14 @@
         frame in jf_guard_exc
         """
         if excvalloc is not None:
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             mc.MOV(excvalloc, heap(self.cpu.pos_exc_value()))
         elif tmploc is not None: # if both are None, just ignore
             ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc')
             mc.MOV(tmploc, heap(self.cpu.pos_exc_value()))
             mc.MOV(RawEbpLoc(ofs), tmploc)
         if exctploc is not None:
-            assert exctploc.is_reg()
+            assert exctploc.is_core_reg()
             mc.MOV(exctploc, heap(self.cpu.pos_exception()))
 
         mc.MOV(heap(self.cpu.pos_exception()), imm0)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -227,18 +227,6 @@
         else:
             return self.xrm.make_sure_var_in_reg(var, forbidden_vars)
 
-    def _frame_bindings(self, locs, inputargs):
-        bindings = {}
-        i = 0
-        for loc in locs:
-            if loc is None:
-                continue
-            arg = inputargs[i]
-            i += 1
-            if not isinstance(loc, RegLoc):
-                bindings[arg] = loc
-        return bindings
-
     def _update_bindings(self, locs, inputargs):
         # XXX this should probably go to llsupport/regalloc.py
         used = {}
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -45,7 +45,7 @@
     def is_stack(self):
         return False
 
-    def is_reg(self):
+    def is_core_reg(self):
         return False
 
     def get_position(self):
@@ -169,7 +169,7 @@
     def is_float(self):
         return self.is_xmm
 
-    def is_reg(self):
+    def is_core_reg(self):
         return True
 
 class ImmediateAssemblerLocation(AssemblerLocation):

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:16 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: make rstring.split and rsplit work for
 unicode strings too
Message-ID: <20130617123516.86B231C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64903:005d786754eb
Date: 2013-06-13 22:32 +0200
http://bitbucket.org/pypy/pypy/changeset/005d786754eb/

Log:	make rstring.split and rsplit work for unicode strings too

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -3,14 +3,17 @@
 
 from rpython.annotator.model import (SomeObject, SomeString, s_None, SomeChar,
     SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePtr, SomePBC)
-from rpython.rlib.objectmodel import newlist_hint
+from rpython.rlib.objectmodel import newlist_hint, specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pairtype
 
 
 # -------------- public API for string functions -----------------------
+
+ at specialize.argtype(0)
 def split(value, by, maxsplit=-1):
+    assert type(value) == type(by)
     bylen = len(by)
     if bylen == 0:
         raise ValueError("empty separator")
@@ -32,7 +35,9 @@
     return res
 
 
+ at specialize.argtype(0)
 def rsplit(value, by, maxsplit=-1):
+    assert type(value) == type(by)
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -13,6 +13,17 @@
     assert split('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, split, 'abc', '')
 
+def test_split_unicode():
+    assert split(u"", u'x') == [u'']
+    assert split(u"a", u"a", 1) == [u'', u'']
+    assert split(u" ", u" ", 1) == [u'', u'']
+    assert split(u"aa", u"a", 2) == [u'', u'', u'']
+    assert split(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert split(u'a|b|c|d', u'|', 2) == [u'a', u'b', u'c|d']
+    assert split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert split(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, split, u'abc', u'')
+
 def test_rsplit():
     assert rsplit("a", "a", 1) == ['', '']
     assert rsplit(" ", " ", 1) == ['', '']
@@ -23,6 +34,16 @@
     assert rsplit('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, rsplit, "abc", '')
 
+def test_rsplit_unicode():
+    assert rsplit(u"a", u"a", 1) == [u'', u'']
+    assert rsplit(u" ", u" ", 1) == [u'', u'']
+    assert rsplit(u"aa", u"a", 2) == [u'', u'', u'']
+    assert rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+    assert rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, rsplit, u"abc", u'')
+
 def test_string_builder():
     s = StringBuilder()
     s.append("a")

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:18 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: move an optimization from
	stringobject.py to rstring.py
Message-ID: <20130617123518.7D8921C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64904:70f3823a6ba8
Date: 2013-06-13 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/70f3823a6ba8/

Log:	move an optimization from stringobject.py to rstring.py

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -16,7 +16,7 @@
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import StringBuilder, split
+from rpython.rlib.rstring import StringBuilder, split, rsplit
 
 
 class W_AbstractStringObject(W_Object):
@@ -287,31 +287,12 @@
     bylen = len(by)
     if bylen == 0:
         raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-    if bylen == 1 and maxsplit < 0:
-        res = []
-        start = 0
-        # fast path: uses str.rfind(character) and str.count(character)
-        by = by[0]    # annotator hack: string -> char
-        count = value.count(by)
-        res = [None] * (count + 1)
-        end = len(value)
-        while count >= 0:
-            assert end >= 0
-            prev = value.rfind(by, 0, end)
-            start = prev + 1
-            assert start >= 0
-            res[count] = value[start:end]
-            count -= 1
-            end = prev
-    else:
-        res = split(value, by, maxsplit)
-
+    res = split(value, by, maxsplit)
     return space.newlist_str(res)
 
 def str_rsplit__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -336,20 +317,19 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(sliced(space, value, j1, i+1, w_self))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_str(res)
 
 def make_rsplit_with_delim(funcname, sliced):
     from rpython.tool.sourcetools import func_with_new_name
 
     def fn(space, w_self, w_by, w_maxsplit=-1):
         maxsplit = space.int_w(w_maxsplit)
-        res_w = []
         value = w_self._value
         end = len(value)
         by = w_by._value
@@ -357,6 +337,7 @@
         if bylen == 0:
             raise OperationError(space.w_ValueError, space.wrap("empty separator"))
 
+        res_w = []
         while maxsplit != 0:
             next = value.rfind(by, 0, end)
             if next < 0:
@@ -371,8 +352,13 @@
 
     return func_with_new_name(fn, funcname)
 
-str_rsplit__String_String_ANY = make_rsplit_with_delim('str_rsplit__String_String_ANY',
-                                                       sliced)
+def str_rsplit__String_String_ANY(space, w_self, w_by, w_maxsplit=-1):
+    maxsplit = space.int_w(w_maxsplit)
+    value = w_self._value
+    by = w_by._value
+    if not by:
+        raise OperationError(space.w_ValueError, space.wrap("empty separator"))
+    return space.newlist_str(rsplit(value, by, maxsplit))
 
 def str_join__String_ANY(space, w_self, w_list):
     l = space.listview_str(w_list)
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -555,10 +555,14 @@
         try:
             w_l = space.call_method(w_s, "split")
             w_l2 = space.call_method(w_s, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_s, "rsplit")
+            w_l4 = space.call_method(w_s, "rsplit", space.wrap(" "))
         finally:
             del space.newlist
         assert space.listview_str(w_l) == ["a", "b", "c"]
         assert space.listview_str(w_l2) == ["a", "b", "c"]
+        assert space.listview_str(w_l3) == ["a", "b", "c"]
+        assert space.listview_str(w_l4) == ["a", "b", "c"]
 
     def test_unicode_uses_newlist_unicode(self):
         space = self.space
@@ -567,10 +571,14 @@
         try:
             w_l = space.call_method(w_u, "split")
             w_l2 = space.call_method(w_u, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_u, "rsplit")
+            w_l4 = space.call_method(w_u, "rsplit", space.wrap(" "))
         finally:
             del space.newlist
         assert space.listview_unicode(w_l) == [u"a", u"b", u"c"]
         assert space.listview_unicode(w_l2) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l3) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l4) == [u"a", u"b", u"c"]
 
     def test_pop_without_argument_is_fast(self):
         space = self.space
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -18,11 +18,27 @@
     if bylen == 0:
         raise ValueError("empty separator")
 
+    start = 0
+    if bylen == 1:
+        # fast path: uses str.rfind(character) and str.count(character)
+        by = by[0]    # annotator hack: string -> char
+        count = value.count(by)
+        if 0 <= maxsplit < count:
+            count = maxsplit
+        res = newlist_hint(count + 1)
+        while count > 0:
+            next = value.find(by, start)
+            res.append(value[start:next])
+            start = next + bylen
+            count -= 1
+        res.append(value[start:len(value)])
+        return res
+
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
         res = []
-    start = 0
+
     while maxsplit != 0:
         next = value.find(by, start)
         if next < 0:
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -10,6 +10,7 @@
     assert split('a|b|c|d', '|') == ['a', 'b', 'c', 'd']
     assert split('a|b|c|d', '|', 2) == ['a', 'b', 'c|d']
     assert split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+    assert split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
     assert split('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, split, 'abc', '')
 

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:19 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: use rstring.split
Message-ID: <20130617123519.C2FEB1C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64905:b89e2ccc4560
Date: 2013-06-13 23:47 +0200
http://bitbucket.org/pypy/pypy/changeset/b89e2ccc4560/

Log:	use rstring.split

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,7 +16,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rstring import UnicodeBuilder, split, rsplit
 from rpython.rlib.runicode import make_unicode_escape_function
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -688,7 +688,7 @@
     if delim_len == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap('empty separator'))
-    parts = _split_with(self, delim, maxsplit)
+    parts = split(self, delim, maxsplit)
     return space.newlist_unicode(parts)
 
 
@@ -727,15 +727,13 @@
     res.reverse()
     return space.newlist_unicode(res)
 
-def sliced(space, s, start, stop, orig_obj):
-    assert start >= 0
-    assert stop >= 0
-    if start == 0 and stop == len(s) and space.is_w(space.type(orig_obj), space.w_unicode):
-        return orig_obj
-    return space.wrap( s[start:stop])
-
-unicode_rsplit__Unicode_Unicode_ANY = make_rsplit_with_delim('unicode_rsplit__Unicode_Unicode_ANY',
-                                                             sliced)
+def unicode_rsplit__Unicode_Unicode_ANY(space, w_self, w_by, w_maxsplit=-1):
+    maxsplit = space.int_w(w_maxsplit)
+    value = w_self._value
+    by = w_by._value
+    if not by:
+        raise OperationError(space.w_ValueError, space.wrap("empty separator"))
+    return space.newlist_unicode(rsplit(value, by, maxsplit))
 
 def _split_into_chars(self, maxsplit):
     if maxsplit == 0:
@@ -753,21 +751,6 @@
     parts.append(self[index:])
     return parts
 
-def _split_with(self, with_, maxsplit=-1):
-    parts = []
-    start = 0
-    end = len(self)
-    length = len(with_)
-    while maxsplit != 0:
-        index = self.find(with_, start, end)
-        if index < 0:
-            break
-        parts.append(self[start:index])
-        start = index + length
-        maxsplit -= 1
-    parts.append(self[start:])
-    return parts
-
 def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old,
                                                  w_new, w_maxsplit):
     return _unicode_replace(space, w_self, w_old._value, w_new._value,
@@ -787,7 +770,7 @@
 
 def _unicode_replace(space, w_self, old, new, w_maxsplit):
     if len(old):
-        parts = _split_with(w_self._value, old, space.int_w(w_maxsplit))
+        parts = split(w_self._value, old, space.int_w(w_maxsplit))
     else:
         self = w_self._value
         maxsplit = space.int_w(w_maxsplit)
@@ -848,7 +831,7 @@
 def unicode_expandtabs__Unicode_ANY(space, w_self, w_tabsize):
     self = w_self._value
     tabsize  = space.int_w(w_tabsize)
-    parts = _split_with(self, u'\t')
+    parts = self.split(self, u'\t')
     result = [parts[0]]
     prevsize = 0
     for ch in parts[0]:

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:20 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: use more newlist_str/unicode
Message-ID: <20130617123520.EDF5D1C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64906:8a1612cebb3c
Date: 2013-06-13 23:53 +0200
http://bitbucket.org/pypy/pypy/changeset/8a1612cebb3c/

Log:	use more newlist_str/unicode

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -325,33 +325,6 @@
     res.reverse()
     return space.newlist_str(res)
 
-def make_rsplit_with_delim(funcname, sliced):
-    from rpython.tool.sourcetools import func_with_new_name
-
-    def fn(space, w_self, w_by, w_maxsplit=-1):
-        maxsplit = space.int_w(w_maxsplit)
-        value = w_self._value
-        end = len(value)
-        by = w_by._value
-        bylen = len(by)
-        if bylen == 0:
-            raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-        res_w = []
-        while maxsplit != 0:
-            next = value.rfind(by, 0, end)
-            if next < 0:
-                break
-            res_w.append(sliced(space, value, next+bylen, end, w_self))
-            end = next
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        res_w.append(sliced(space, value, 0, end, w_self))
-        res_w.reverse()
-        return space.newlist(res_w)
-
-    return func_with_new_name(fn, funcname)
-
 def str_rsplit__String_String_ANY(space, w_self, w_by, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
     value = w_self._value
@@ -755,7 +728,7 @@
     u_keepends = space.int_w(w_keepends)  # truth value, but type checked
     data = w_self._value
     selflen = len(data)
-    strs_w = []
+    strs = []
     i = j = 0
     while i < selflen:
         # Find a line and append it
@@ -768,12 +741,12 @@
             i += 1
         if u_keepends:
             eol = i
-        strs_w.append(sliced(space, data, j, eol, w_self))
+        strs.append(data[j:eol])
         j = i
 
     if j < selflen:
-        strs_w.append(sliced(space, data, j, len(data), w_self))
-    return space.newlist(strs_w)
+        strs.append(data[j:])
+    return space.newlist_str(strs)
 
 def str_zfill__String_ANY(space, w_self, w_width):
     input = w_self._value
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -8,8 +8,7 @@
 from pypy.objspace.std.multimethod import FailedToImplement
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
-from pypy.objspace.std.stringobject import (
-    W_StringObject, make_rsplit_with_delim)
+from pypy.objspace.std.stringobject import W_StringObject
 from pypy.objspace.std.stringtype import stringendswith, stringstartswith
 from pypy.objspace.std.register_all import register_all
 from rpython.rlib import jit
@@ -608,17 +607,17 @@
             if (self[pos] == u'\r' and pos + 1 < end and
                 self[pos + 1] == u'\n'):
                 # Count CRLF as one linebreak
-                lines.append(W_UnicodeObject(self[start:pos + keepends * 2]))
+                lines.append(self[start:pos + keepends * 2])
                 pos += 1
             else:
-                lines.append(W_UnicodeObject(self[start:pos + keepends]))
+                lines.append(self[start:pos + keepends])
             pos += 1
             start = pos
         else:
             pos += 1
     if not unicodedb.islinebreak(ord(self[end - 1])):
-        lines.append(W_UnicodeObject(self[start:]))
-    return space.newlist(lines)
+        lines.append(self[start:])
+    return space.newlist_unicode(lines)
 
 def unicode_find__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self, w_start, w_end)
@@ -831,7 +830,7 @@
 def unicode_expandtabs__Unicode_ANY(space, w_self, w_tabsize):
     self = w_self._value
     tabsize  = space.int_w(w_tabsize)
-    parts = self.split(self, u'\t')
+    parts = self.split(u'\t')
     result = [parts[0]]
     prevsize = 0
     for ch in parts[0]:

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:22 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: what isn't tested is broken
Message-ID: <20130617123522.265A91C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64907:33f5229fbc8a
Date: 2013-06-14 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/33f5229fbc8a/

Log:	what isn't tested is broken

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -13,7 +13,10 @@
 
 @specialize.argtype(0)
 def split(value, by, maxsplit=-1):
-    assert type(value) == type(by)
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     bylen = len(by)
     if bylen == 0:
         raise ValueError("empty separator")
@@ -28,6 +31,7 @@
         res = newlist_hint(count + 1)
         while count > 0:
             next = value.find(by, start)
+            assert next >= 0 # cannot fail due to the value.count above
             res.append(value[start:next])
             start = next + bylen
             count -= 1
@@ -53,7 +57,10 @@
 
 @specialize.argtype(0)
 def rsplit(value, by, maxsplit=-1):
-    assert type(value) == type(by)
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -1,6 +1,7 @@
 import sys, py
 
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit
+from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin
 
 def test_split():
     assert split("", 'x') == ['']
@@ -64,4 +65,22 @@
     s.append_multiple_char(u'd', 4)
     assert s.build() == 'aabcbdddd'
     assert isinstance(s.build(), unicode)
-        
+
+
+class TestTranslates(LLRtypeMixin, BaseRtypingTest):
+    def test_split_rsplit_translate(self):
+        def fn():
+            res = True
+            res = res and split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
+            res = res and split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            res = res and split(u'endcase test', u'test') == [u'endcase ', u'']
+            res = res and rsplit('a|b|c|d', '|', 2) == ['a|b', 'c', 'd']
+            res = res and rsplit('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+            res = res and rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+            res = res and rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            return res
+        res = self.interpret(fn, [])
+        assert res
+

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:23 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: use the rpython splitlines
Message-ID: <20130617123523.6AAC61C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64908:6aefb00ce08d
Date: 2013-06-14 17:53 +0200
http://bitbucket.org/pypy/pypy/changeset/6aefb00ce08d/

Log:	use the rpython splitlines

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -727,26 +727,7 @@
 def str_splitlines__String_ANY(space, w_self, w_keepends):
     u_keepends = space.int_w(w_keepends)  # truth value, but type checked
     data = w_self._value
-    selflen = len(data)
-    strs = []
-    i = j = 0
-    while i < selflen:
-        # Find a line and append it
-        while i < selflen and data[i] != '\n' and data[i] != '\r':
-            i += 1
-        # Skip the line break reading CRLF as one line break
-        eol = i
-        i += 1
-        if i < selflen and data[i-1] == '\r' and data[i] == '\n':
-            i += 1
-        if u_keepends:
-            eol = i
-        strs.append(data[j:eol])
-        j = i
-
-    if j < selflen:
-        strs.append(data[j:])
-    return space.newlist_str(strs)
+    return space.newlist_str(data.splitlines(u_keepends))
 
 def str_zfill__String_ANY(space, w_self, w_width):
     input = w_self._value

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:24 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: an implementation of string_replace in
	rstring
Message-ID: <20130617123524.9EAE01C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64909:34888d691e22
Date: 2013-06-14 18:14 +0200
http://bitbucket.org/pypy/pypy/changeset/34888d691e22/

Log:	an implementation of string_replace in rstring

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -82,6 +82,59 @@
     res.reverse()
     return res
 
+def string_replace(input, sub, by, maxsplit=-1):
+    if maxsplit == 0:
+        return input
+
+    if not sub:
+        upper = len(input)
+        if maxsplit > 0 and maxsplit < upper + 2:
+            upper = maxsplit - 1
+            assert upper >= 0
+
+        try:
+            result_size = ovfcheck(upper * len(by))
+            result_size = ovfcheck(result_size + upper)
+            result_size = ovfcheck(result_size + len(by))
+            remaining_size = len(input) - upper
+            result_size = ovfcheck(result_size + remaining_size)
+        except OverflowError:
+            raise
+        builder = StringBuilder(result_size)
+        for i in range(upper):
+            builder.append(by)
+            builder.append(input[i])
+        builder.append(by)
+        builder.append_slice(input, upper, len(input))
+    else:
+        # First compute the exact result size
+        count = input.count(sub)
+        if count > maxsplit and maxsplit > 0:
+            count = maxsplit
+        diff_len = len(by) - len(sub)
+        try:
+            result_size = ovfcheck(diff_len * count)
+            result_size = ovfcheck(result_size + len(input))
+        except OverflowError:
+            raise
+
+        builder = StringBuilder(result_size)
+        start = 0
+        sublen = len(sub)
+
+        while maxsplit != 0:
+            next = input.find(sub, start)
+            if next < 0:
+                break
+            builder.append_slice(input, start, next)
+            builder.append(by)
+            start = next + sublen
+            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+        builder.append_slice(input, start, len(input))
+
+    return builder.build()
+
 # -------------- public API ---------------------------------
 
 INIT_SIZE = 100 # XXX tweak
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -1,6 +1,7 @@
 import sys, py
 
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit
+from rpython.rlib.rstring import string_replace
 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin
 
 def test_split():
@@ -46,6 +47,39 @@
     assert rsplit(u'endcase test', u'test') == [u'endcase ', u'']
     py.test.raises(ValueError, rsplit, u"abc", u'')
 
+def test_string_replace():
+    assert string_replace('one!two!three!', '!', '@', 1) == 'one at two!three!'
+    assert string_replace('one!two!three!', '!', '') == 'onetwothree'
+    assert string_replace('one!two!three!', '!', '@', 2) == 'one at two@three!'
+    assert string_replace('one!two!three!', '!', '@', 3) == 'one at two@three@'
+    assert string_replace('one!two!three!', '!', '@', 4) == 'one at two@three@'
+    assert string_replace('one!two!three!', '!', '@', 0) == 'one!two!three!'
+    assert string_replace('one!two!three!', '!', '@') == 'one at two@three@'
+    assert string_replace('one!two!three!', 'x', '@') == 'one!two!three!'
+    assert string_replace('one!two!three!', 'x', '@', 2) == 'one!two!three!'
+    assert string_replace('abc', '', '-') == '-a-b-c-'
+    assert string_replace('abc', '', '-', 3) == '-a-b-c'
+    assert string_replace('abc', '', '-', 0) == 'abc'
+    assert string_replace('', '', '') == ''
+    assert string_replace('', '', 'a') == 'a'
+    assert string_replace('abc', 'ab', '--', 0) == 'abc'
+    assert string_replace('abc', 'xy', '--') == 'abc'
+    assert string_replace('123', '123', '') == ''
+    assert string_replace('123123', '123', '') == ''
+    assert string_replace('123x123', '123', '') == 'x'
+
+def test_string_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = "a" * (2**16)
+    with py.test.raises(OverflowError):
+        string_replace(s, "", s)
+    with py.test.raises(OverflowError):
+        string_replace(s, "a", s)
+    with py.test.raises(OverflowError):
+        string_replace(s, "a", s, len(s) - 10)
+
+
 def test_string_builder():
     s = StringBuilder()
     s.append("a")

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:25 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: make a unicode version of replace as
 well, and expose them both under a
Message-ID: <20130617123525.C5BF91C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64910:16bde61a597c
Date: 2013-06-14 18:41 +0200
http://bitbucket.org/pypy/pypy/changeset/16bde61a597c/

Log:	make a unicode version of replace as well, and expose them both
	under a specialized interface, "replace"

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -82,58 +82,8 @@
     res.reverse()
     return res
 
-def string_replace(input, sub, by, maxsplit=-1):
-    if maxsplit == 0:
-        return input
 
-    if not sub:
-        upper = len(input)
-        if maxsplit > 0 and maxsplit < upper + 2:
-            upper = maxsplit - 1
-            assert upper >= 0
 
-        try:
-            result_size = ovfcheck(upper * len(by))
-            result_size = ovfcheck(result_size + upper)
-            result_size = ovfcheck(result_size + len(by))
-            remaining_size = len(input) - upper
-            result_size = ovfcheck(result_size + remaining_size)
-        except OverflowError:
-            raise
-        builder = StringBuilder(result_size)
-        for i in range(upper):
-            builder.append(by)
-            builder.append(input[i])
-        builder.append(by)
-        builder.append_slice(input, upper, len(input))
-    else:
-        # First compute the exact result size
-        count = input.count(sub)
-        if count > maxsplit and maxsplit > 0:
-            count = maxsplit
-        diff_len = len(by) - len(sub)
-        try:
-            result_size = ovfcheck(diff_len * count)
-            result_size = ovfcheck(result_size + len(input))
-        except OverflowError:
-            raise
-
-        builder = StringBuilder(result_size)
-        start = 0
-        sublen = len(sub)
-
-        while maxsplit != 0:
-            next = input.find(sub, start)
-            if next < 0:
-                break
-            builder.append_slice(input, start, next)
-            builder.append(by)
-            start = next + sublen
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        builder.append_slice(input, start, len(input))
-
-    return builder.build()
 
 # -------------- public API ---------------------------------
 
@@ -352,3 +302,75 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+
+
+
+def make_replace(func_name, Builder):
+    def replace(input, sub, by, maxsplit=-1):
+        if maxsplit == 0:
+            return input
+
+        if not sub:
+            upper = len(input)
+            if maxsplit > 0 and maxsplit < upper + 2:
+                upper = maxsplit - 1
+                assert upper >= 0
+
+            try:
+                result_size = ovfcheck(upper * len(by))
+                result_size = ovfcheck(result_size + upper)
+                result_size = ovfcheck(result_size + len(by))
+                remaining_size = len(input) - upper
+                result_size = ovfcheck(result_size + remaining_size)
+            except OverflowError:
+                raise
+            builder = Builder(result_size)
+            for i in range(upper):
+                builder.append(by)
+                builder.append(input[i])
+            builder.append(by)
+            builder.append_slice(input, upper, len(input))
+        else:
+            # First compute the exact result size
+            count = input.count(sub)
+            if count > maxsplit and maxsplit > 0:
+                count = maxsplit
+            diff_len = len(by) - len(sub)
+            try:
+                result_size = ovfcheck(diff_len * count)
+                result_size = ovfcheck(result_size + len(input))
+            except OverflowError:
+                raise
+
+            builder = Builder(result_size)
+            start = 0
+            sublen = len(sub)
+
+            while maxsplit != 0:
+                next = input.find(sub, start)
+                if next < 0:
+                    break
+                builder.append_slice(input, start, next)
+                builder.append(by)
+                start = next + sublen
+                maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+            builder.append_slice(input, start, len(input))
+
+        return builder.build()
+    replace.func_name = func_name
+    return replace
+
+_string_replace = make_replace("_string_replace", StringBuilder)
+_unicode_replace = make_replace("_unicode_replace", UnicodeBuilder)
+
+ at specialize.argtype(0)
+def replace(input, sub, by, maxsplit=-1):
+    if isinstance(input, str):
+        assert isinstance(sub, str)
+        assert isinstance(by, str)
+        return _string_replace(input, sub, by, maxsplit)
+    else:
+        assert isinstance(sub, unicode)
+        assert isinstance(by, unicode)
+        return _unicode_replace(input, sub, by, maxsplit)
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -1,7 +1,7 @@
 import sys, py
 
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit
-from rpython.rlib.rstring import string_replace
+from rpython.rlib.rstring import replace
 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin
 
 def test_split():
@@ -48,37 +48,68 @@
     py.test.raises(ValueError, rsplit, u"abc", u'')
 
 def test_string_replace():
-    assert string_replace('one!two!three!', '!', '@', 1) == 'one at two!three!'
-    assert string_replace('one!two!three!', '!', '') == 'onetwothree'
-    assert string_replace('one!two!three!', '!', '@', 2) == 'one at two@three!'
-    assert string_replace('one!two!three!', '!', '@', 3) == 'one at two@three@'
-    assert string_replace('one!two!three!', '!', '@', 4) == 'one at two@three@'
-    assert string_replace('one!two!three!', '!', '@', 0) == 'one!two!three!'
-    assert string_replace('one!two!three!', '!', '@') == 'one at two@three@'
-    assert string_replace('one!two!three!', 'x', '@') == 'one!two!three!'
-    assert string_replace('one!two!three!', 'x', '@', 2) == 'one!two!three!'
-    assert string_replace('abc', '', '-') == '-a-b-c-'
-    assert string_replace('abc', '', '-', 3) == '-a-b-c'
-    assert string_replace('abc', '', '-', 0) == 'abc'
-    assert string_replace('', '', '') == ''
-    assert string_replace('', '', 'a') == 'a'
-    assert string_replace('abc', 'ab', '--', 0) == 'abc'
-    assert string_replace('abc', 'xy', '--') == 'abc'
-    assert string_replace('123', '123', '') == ''
-    assert string_replace('123123', '123', '') == ''
-    assert string_replace('123x123', '123', '') == 'x'
+    assert replace('one!two!three!', '!', '@', 1) == 'one at two!three!'
+    assert replace('one!two!three!', '!', '') == 'onetwothree'
+    assert replace('one!two!three!', '!', '@', 2) == 'one at two@three!'
+    assert replace('one!two!three!', '!', '@', 3) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 4) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 0) == 'one!two!three!'
+    assert replace('one!two!three!', '!', '@') == 'one at two@three@'
+    assert replace('one!two!three!', 'x', '@') == 'one!two!three!'
+    assert replace('one!two!three!', 'x', '@', 2) == 'one!two!three!'
+    assert replace('abc', '', '-') == '-a-b-c-'
+    assert replace('abc', '', '-', 3) == '-a-b-c'
+    assert replace('abc', '', '-', 0) == 'abc'
+    assert replace('', '', '') == ''
+    assert replace('', '', 'a') == 'a'
+    assert replace('abc', 'ab', '--', 0) == 'abc'
+    assert replace('abc', 'xy', '--') == 'abc'
+    assert replace('123', '123', '') == ''
+    assert replace('123123', '123', '') == ''
+    assert replace('123x123', '123', '') == 'x'
 
 def test_string_replace_overflow():
     if sys.maxint > 2**31-1:
         py.test.skip("Wrong platform")
     s = "a" * (2**16)
     with py.test.raises(OverflowError):
-        string_replace(s, "", s)
+        replace(s, "", s)
     with py.test.raises(OverflowError):
-        string_replace(s, "a", s)
+        replace(s, "a", s)
     with py.test.raises(OverflowError):
-        string_replace(s, "a", s, len(s) - 10)
+        replace(s, "a", s, len(s) - 10)
 
+def test_unicode_replace():
+    assert replace(u'one!two!three!', u'!', u'@', 1) == u'one at two!three!'
+    assert replace(u'one!two!three!', u'!', u'') == u'onetwothree'
+    assert replace(u'one!two!three!', u'!', u'@', 2) == u'one at two@three!'
+    assert replace(u'one!two!three!', u'!', u'@', 3) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 4) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 0) == u'one!two!three!'
+    assert replace(u'one!two!three!', u'!', u'@') == u'one at two@three@'
+    assert replace(u'one!two!three!', u'x', u'@') == u'one!two!three!'
+    assert replace(u'one!two!three!', u'x', u'@', 2) == u'one!two!three!'
+    assert replace(u'abc', u'', u'-') == u'-a-b-c-'
+    assert replace(u'abc', u'', u'-', 3) == u'-a-b-c'
+    assert replace(u'abc', u'', u'-', 0) == u'abc'
+    assert replace(u'', u'', u'') == u''
+    assert replace(u'', u'', u'a') == u'a'
+    assert replace(u'abc', u'ab', u'--', 0) == u'abc'
+    assert replace(u'abc', u'xy', u'--') == u'abc'
+    assert replace(u'123', u'123', u'') == u''
+    assert replace(u'123123', u'123', u'') == u''
+    assert replace(u'123x123', u'123', u'') == u'x'
+
+def test_unicode_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = u"a" * (2**16)
+    with py.test.raises(OverflowError):
+        replace(s, u"", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s, len(s) - 10)
 
 def test_string_builder():
     s = StringBuilder()
@@ -102,7 +133,7 @@
 
 
 class TestTranslates(LLRtypeMixin, BaseRtypingTest):
-    def test_split_rsplit_translate(self):
+    def test_split_rsplit(self):
         def fn():
             res = True
             res = res and split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
@@ -118,3 +149,13 @@
         res = self.interpret(fn, [])
         assert res
 
+    def test_replace(self):
+        def fn():
+            res = True
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            return res
+        res = self.interpret(fn, [])
+        assert res

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:27 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: use the new replace functions and kill
	the old ones
Message-ID: <20130617123527.039F31C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64911:e53c9a99343c
Date: 2013-06-14 18:50 +0200
http://bitbucket.org/pypy/pypy/changeset/e53c9a99343c/

Log:	use the new replace functions and kill the old ones

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -16,7 +16,7 @@
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import StringBuilder, split, rsplit
+from rpython.rlib.rstring import StringBuilder, split, rsplit, replace
 
 
 class W_AbstractStringObject(W_Object):
@@ -483,75 +483,30 @@
 
     return space.wrap(res)
 
-def _string_replace(space, input, sub, by, maxsplit):
-    if maxsplit == 0:
-        return space.wrap(input)
-
-    if not sub:
-        upper = len(input)
-        if maxsplit > 0 and maxsplit < upper + 2:
-            upper = maxsplit - 1
-            assert upper >= 0
-
-        try:
-            result_size = ovfcheck(upper * len(by))
-            result_size = ovfcheck(result_size + upper)
-            result_size = ovfcheck(result_size + len(by))
-            remaining_size = len(input) - upper
-            result_size = ovfcheck(result_size + remaining_size)
-        except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                space.wrap("replace string is too long")
-            )
-        builder = StringBuilder(result_size)
-        for i in range(upper):
-            builder.append(by)
-            builder.append(input[i])
-        builder.append(by)
-        builder.append_slice(input, upper, len(input))
-    else:
-        # First compute the exact result size
-        count = input.count(sub)
-        if count > maxsplit and maxsplit > 0:
-            count = maxsplit
-        diff_len = len(by) - len(sub)
-        try:
-            result_size = ovfcheck(diff_len * count)
-            result_size = ovfcheck(result_size + len(input))
-        except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                space.wrap("replace string is too long")
-            )
-
-        builder = StringBuilder(result_size)
-        start = 0
-        sublen = len(sub)
-
-        while maxsplit != 0:
-            next = input.find(sub, start)
-            if next < 0:
-                break
-            builder.append_slice(input, start, next)
-            builder.append(by)
-            start = next + sublen
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        builder.append_slice(input, start, len(input))
-
-    return space.wrap(builder.build())
-
 
 def str_replace__String_ANY_ANY_ANY(space, w_self, w_sub, w_by, w_maxsplit):
-    return _string_replace(space, w_self._value, space.buffer_w(w_sub).as_str(),
-                           space.buffer_w(w_by).as_str(),
-                           space.int_w(w_maxsplit))
+    sub = space.buffer_w(w_sub).as_str()
+    by = space.buffer_w(w_by).as_str()
+    maxsplit = space.int_w(w_maxsplit)
+    try:
+        res = replace(w_self._value, sub, by, maxsplit)
+    except OverflowError:
+        raise OperationError(space.w_OverflowError,
+            space.wrap("replace string is too long")
+        )
+    return space.wrap(res)
 
 def str_replace__String_String_String_ANY(space, w_self, w_sub, w_by, w_maxsplit=-1):
-    input = w_self._value
     sub = w_sub._value
     by = w_by._value
     maxsplit = space.int_w(w_maxsplit)
-    return _string_replace(space, input, sub, by, maxsplit)
+    try:
+        res = replace(w_self._value, sub, by, maxsplit)
+    except OverflowError:
+        raise OperationError(space.w_OverflowError,
+            space.wrap("replace string is too long")
+        )
+    return space.wrap(res)
 
 def _strip(space, w_self, w_chars, left, right):
     "internal function called by str_xstrip methods"
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -15,7 +15,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
-from rpython.rlib.rstring import UnicodeBuilder, split, rsplit
+from rpython.rlib.rstring import UnicodeBuilder, split, rsplit, replace
 from rpython.rlib.runicode import make_unicode_escape_function
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -734,26 +734,16 @@
         raise OperationError(space.w_ValueError, space.wrap("empty separator"))
     return space.newlist_unicode(rsplit(value, by, maxsplit))
 
-def _split_into_chars(self, maxsplit):
-    if maxsplit == 0:
-        return [self]
-    index = 0
-    end = len(self)
-    parts = [u'']
-    maxsplit -= 1
-    while maxsplit != 0:
-        if index >= end:
-            break
-        parts.append(self[index])
-        index += 1
-        maxsplit -= 1
-    parts.append(self[index:])
-    return parts
-
 def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old,
                                                  w_new, w_maxsplit):
-    return _unicode_replace(space, w_self, w_old._value, w_new._value,
-                            w_maxsplit)
+    maxsplit = space.int_w(w_maxsplit)
+    try:
+        return W_UnicodeObject(
+                replace(w_self._value, w_old._value, w_new._value, maxsplit))
+    except OverflowError:
+        raise OperationError(
+            space.w_OverflowError,
+            space.wrap("replace string is too long"))
 
 def unicode_replace__Unicode_ANY_ANY_ANY(space, w_self, w_old, w_new,
                                          w_maxsplit):
@@ -765,27 +755,14 @@
         new = unicode(space.bufferstr_w(w_new))
     else:
         new = space.unicode_w(w_new)
-    return _unicode_replace(space, w_self, old, new, w_maxsplit)
-
-def _unicode_replace(space, w_self, old, new, w_maxsplit):
-    if len(old):
-        parts = split(w_self._value, old, space.int_w(w_maxsplit))
-    else:
-        self = w_self._value
-        maxsplit = space.int_w(w_maxsplit)
-        parts = _split_into_chars(self, maxsplit)
-
+    maxsplit = space.int_w(w_maxsplit)
     try:
-        one = ovfcheck(len(parts) * len(new))
-        ovfcheck(one + len(w_self._value))
+        return W_UnicodeObject(replace(w_self._value, old, new, maxsplit))
     except OverflowError:
         raise OperationError(
             space.w_OverflowError,
             space.wrap("replace string is too long"))
 
-    return W_UnicodeObject(new.join(parts))
-
-
 def unicode_encode__Unicode_ANY_ANY(space, w_unistr,
                                     w_encoding=None,
                                     w_errors=None):

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:28 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: don't really need to use such heavy
	machinery
Message-ID: <20130617123528.3437D1C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64912:cc52d477c5f3
Date: 2013-06-14 18:51 +0200
http://bitbucket.org/pypy/pypy/changeset/cc52d477c5f3/

Log:	don't really need to use such heavy machinery

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -83,6 +83,67 @@
     return res
 
 
+ at specialize.argtype(0)
+def replace(input, sub, by, maxsplit=-1):
+    if isinstance(input, str):
+        assert isinstance(sub, str)
+        assert isinstance(by, str)
+        Builder = StringBuilder
+    else:
+        assert isinstance(sub, unicode)
+        assert isinstance(by, unicode)
+        Builder = UnicodeBuilder
+    if maxsplit == 0:
+        return input
+
+    if not sub:
+        upper = len(input)
+        if maxsplit > 0 and maxsplit < upper + 2:
+            upper = maxsplit - 1
+            assert upper >= 0
+
+        try:
+            result_size = ovfcheck(upper * len(by))
+            result_size = ovfcheck(result_size + upper)
+            result_size = ovfcheck(result_size + len(by))
+            remaining_size = len(input) - upper
+            result_size = ovfcheck(result_size + remaining_size)
+        except OverflowError:
+            raise
+        builder = Builder(result_size)
+        for i in range(upper):
+            builder.append(by)
+            builder.append(input[i])
+        builder.append(by)
+        builder.append_slice(input, upper, len(input))
+    else:
+        # First compute the exact result size
+        count = input.count(sub)
+        if count > maxsplit and maxsplit > 0:
+            count = maxsplit
+        diff_len = len(by) - len(sub)
+        try:
+            result_size = ovfcheck(diff_len * count)
+            result_size = ovfcheck(result_size + len(input))
+        except OverflowError:
+            raise
+
+        builder = Builder(result_size)
+        start = 0
+        sublen = len(sub)
+
+        while maxsplit != 0:
+            next = input.find(sub, start)
+            if next < 0:
+                break
+            builder.append_slice(input, start, next)
+            builder.append(by)
+            start = next + sublen
+            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+        builder.append_slice(input, start, len(input))
+
+    return builder.build()
 
 
 # -------------- public API ---------------------------------
@@ -304,73 +365,3 @@
         hop.exception_cannot_occur()
 
 
-
-def make_replace(func_name, Builder):
-    def replace(input, sub, by, maxsplit=-1):
-        if maxsplit == 0:
-            return input
-
-        if not sub:
-            upper = len(input)
-            if maxsplit > 0 and maxsplit < upper + 2:
-                upper = maxsplit - 1
-                assert upper >= 0
-
-            try:
-                result_size = ovfcheck(upper * len(by))
-                result_size = ovfcheck(result_size + upper)
-                result_size = ovfcheck(result_size + len(by))
-                remaining_size = len(input) - upper
-                result_size = ovfcheck(result_size + remaining_size)
-            except OverflowError:
-                raise
-            builder = Builder(result_size)
-            for i in range(upper):
-                builder.append(by)
-                builder.append(input[i])
-            builder.append(by)
-            builder.append_slice(input, upper, len(input))
-        else:
-            # First compute the exact result size
-            count = input.count(sub)
-            if count > maxsplit and maxsplit > 0:
-                count = maxsplit
-            diff_len = len(by) - len(sub)
-            try:
-                result_size = ovfcheck(diff_len * count)
-                result_size = ovfcheck(result_size + len(input))
-            except OverflowError:
-                raise
-
-            builder = Builder(result_size)
-            start = 0
-            sublen = len(sub)
-
-            while maxsplit != 0:
-                next = input.find(sub, start)
-                if next < 0:
-                    break
-                builder.append_slice(input, start, next)
-                builder.append(by)
-                start = next + sublen
-                maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-            builder.append_slice(input, start, len(input))
-
-        return builder.build()
-    replace.func_name = func_name
-    return replace
-
-_string_replace = make_replace("_string_replace", StringBuilder)
-_unicode_replace = make_replace("_unicode_replace", UnicodeBuilder)
-
- at specialize.argtype(0)
-def replace(input, sub, by, maxsplit=-1):
-    if isinstance(input, str):
-        assert isinstance(sub, str)
-        assert isinstance(by, str)
-        return _string_replace(input, sub, by, maxsplit)
-    else:
-        assert isinstance(sub, unicode)
-        assert isinstance(by, unicode)
-        return _unicode_replace(input, sub, by, maxsplit)

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:29 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: startswith and endswith implementations
Message-ID: <20130617123529.6CC391C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64913:3e0a1d3ed9e0
Date: 2013-06-16 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/3e0a1d3ed9e0/

Log:	startswith and endswith implementations

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -1,5 +1,6 @@
 """ String builder interface and string functions
 """
+import sys
 
 from rpython.annotator.model import (SomeObject, SomeString, s_None, SomeChar,
     SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePtr, SomePBC)
@@ -7,6 +8,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pairtype
+from rpython.rlib import jit
 
 
 # -------------- public API for string functions -----------------------
@@ -145,6 +147,45 @@
 
     return builder.build()
 
+def _normalize_start_end(length, start, end):
+    if start < 0:
+        start += length
+        if start < 0:
+            start = 0
+    if end < 0:
+        end += length
+        if end < 0:
+            end = 0
+    elif end > length:
+        end = length
+    return start, end
+
+ at specialize.argtype(0)
+ at jit.elidable
+def startswith(u_self, prefix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    stop = start + len(prefix)
+    if stop > end:
+        return False
+    for i in range(len(prefix)):
+        if u_self[start+i] != prefix[i]:
+            return False
+    return True
+
+ at specialize.argtype(0)
+ at jit.elidable
+def endswith(u_self, suffix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    begin = end - len(suffix)
+    if begin < start:
+        return False
+    for i in range(len(suffix)):
+        if u_self[begin+i] != suffix[i]:
+            return False
+    return True
+
 
 # -------------- public API ---------------------------------
 
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -1,7 +1,7 @@
 import sys, py
 
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit
-from rpython.rlib.rstring import replace
+from rpython.rlib.rstring import replace, startswith, endswith
 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin
 
 def test_split():
@@ -111,6 +111,37 @@
     with py.test.raises(OverflowError):
         replace(s, u"a", s, len(s) - 10)
 
+def test_startswith():
+    assert startswith('ab', 'ab') is True
+    assert startswith('ab', 'a') is True
+    assert startswith('ab', '') is True
+    assert startswith('x', 'a') is False
+    assert startswith('x', 'x') is True
+    assert startswith('', '') is True
+    assert startswith('', 'a') is False
+    assert startswith('x', 'xx') is False
+    assert startswith('y', 'xx') is False
+    assert startswith('ab', 'a', 0) is True
+    assert startswith('ab', 'a', 1) is False
+    assert startswith('ab', 'b', 1) is True
+    assert startswith('abc', 'bc', 1, 2) is False
+    assert startswith('abc', 'c', -1, 4) is True
+
+def test_endswith():
+    assert endswith('ab', 'ab') is True
+    assert endswith('ab', 'b') is True
+    assert endswith('ab', '') is True
+    assert endswith('x', 'a') is False
+    assert endswith('x', 'x') is True
+    assert endswith('', '') is True
+    assert endswith('', 'a') is False
+    assert endswith('x', 'xx') is False
+    assert endswith('y', 'xx') is False
+    assert endswith('abc', 'ab', 0, 2) is True
+    assert endswith('abc', 'bc', 1) is True
+    assert endswith('abc', 'bc', 2) is False
+    assert endswith('abc', 'b', -3, -1) is True
+
 def test_string_builder():
     s = StringBuilder()
     s.append("a")

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:30 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: use the new methods
Message-ID: <20130617123530.9C0CD1C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64914:682f12897cc8
Date: 2013-06-16 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/682f12897cc8/

Log:	use the new methods

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -11,12 +11,13 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
 from pypy.objspace.std.stringtype import (
-    joined2, sliced, stringendswith, stringstartswith, wrapchar, wrapstr)
+    joined2, sliced, wrapchar, wrapstr)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import StringBuilder, split, rsplit, replace
+from rpython.rlib.rstring import (StringBuilder, split, rsplit, replace,
+    endswith, startswith)
 
 
 class W_AbstractStringObject(W_Object):
@@ -593,7 +594,7 @@
 def str_endswith__String_String_ANY_ANY(space, w_self, w_suffix, w_start, w_end):
     (u_self, start, end) = _convert_idx_params(space, w_self, w_start,
                                                w_end, True)
-    return space.newbool(stringendswith(u_self, w_suffix._value, start, end))
+    return space.newbool(endswith(u_self, w_suffix._value, start, end))
 
 def str_endswith__String_ANY_ANY_ANY(space, w_self, w_suffixes, w_start, w_end):
     if not space.isinstance_w(w_suffixes, space.w_tuple):
@@ -606,14 +607,14 @@
             return space.call_method(w_u, "endswith", w_suffixes, w_start,
                                      w_end)
         suffix = space.str_w(w_suffix)
-        if stringendswith(u_self, suffix, start, end):
+        if endswith(u_self, suffix, start, end):
             return space.w_True
     return space.w_False
 
 def str_startswith__String_String_ANY_ANY(space, w_self, w_prefix, w_start, w_end):
     (u_self, start, end) = _convert_idx_params(space, w_self, w_start,
                                                w_end, True)
-    return space.newbool(stringstartswith(u_self, w_prefix._value, start, end))
+    return space.newbool(startswith(u_self, w_prefix._value, start, end))
 
 def str_startswith__String_ANY_ANY_ANY(space, w_self, w_prefixes, w_start, w_end):
     if not space.isinstance_w(w_prefixes, space.w_tuple):
@@ -626,7 +627,7 @@
             return space.call_method(w_u, "startswith", w_prefixes, w_start,
                                      w_end)
         prefix = space.str_w(w_prefix)
-        if stringstartswith(u_self, prefix, start, end):
+        if startswith(u_self, prefix, start, end):
             return space.w_True
     return space.w_False
 
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -295,28 +295,3 @@
 str_typedef.registermethods(globals())
 
 
-# ____________________________________________________________
-
-# Helpers for several string implementations
-
- at specialize.argtype(0)
- at jit.elidable
-def stringendswith(u_self, suffix, start, end):
-    begin = end - len(suffix)
-    if begin < start:
-        return False
-    for i in range(len(suffix)):
-        if u_self[begin+i] != suffix[i]:
-            return False
-    return True
-
- at specialize.argtype(0)
- at jit.elidable
-def stringstartswith(u_self, prefix, start, end):
-    stop = start + len(prefix)
-    if stop > end:
-        return False
-    for i in range(len(prefix)):
-        if u_self[start+i] != prefix[i]:
-            return False
-    return True
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -9,13 +9,13 @@
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
 from pypy.objspace.std.stringobject import W_StringObject
-from pypy.objspace.std.stringtype import stringendswith, stringstartswith
 from pypy.objspace.std.register_all import register_all
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
-from rpython.rlib.rstring import UnicodeBuilder, split, rsplit, replace
+from rpython.rlib.rstring import (UnicodeBuilder, split, rsplit, replace,
+    startswith, endswith)
 from rpython.rlib.runicode import make_unicode_escape_function
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -489,14 +489,14 @@
 def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self,
                                                    w_start, w_end, True)
-    return space.newbool(stringendswith(self, w_substr._value, start, end))
+    return space.newbool(endswith(self, w_substr._value, start, end))
 
 def unicode_startswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self, w_start, w_end, True)
     # XXX this stuff can be waaay better for ootypebased backends if
     #     we re-use more of our rpython machinery (ie implement startswith
     #     with additional parameters as rpython)
-    return space.newbool(stringstartswith(self, w_substr._value, start, end))
+    return space.newbool(startswith(self, w_substr._value, start, end))
 
 def unicode_startswith__Unicode_ANY_ANY_ANY(space, w_unistr, w_prefixes,
                                               w_start, w_end):
@@ -506,7 +506,7 @@
                                              w_start, w_end, True)
     for w_prefix in space.fixedview(w_prefixes):
         prefix = space.unicode_w(w_prefix)
-        if stringstartswith(unistr, prefix, start, end):
+        if startswith(unistr, prefix, start, end):
             return space.w_True
     return space.w_False
 
@@ -518,7 +518,7 @@
                                              w_start, w_end, True)
     for w_suffix in space.fixedview(w_suffixes):
         suffix = space.unicode_w(w_suffix)
-        if stringendswith(unistr, suffix, start, end):
+        if endswith(unistr, suffix, start, end):
             return space.w_True
     return space.w_False
 

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:32 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130617123532.0E4011C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3k
Changeset: r64915:0aee2842b7ef
Date: 2013-06-17 13:59 +0200
http://bitbucket.org/pypy/pypy/changeset/0aee2842b7ef/

Log:	merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,43 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+from .tklib import tklib, tkffi
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {			/* The internal representation: */
+	long longValue;		/*   - an long integer value. */
+	double doubleValue;	/*   - a double-precision floating value. */
+	struct {		/*   - internal rep as two pointers. */
+	    void *ptr1;
+	    void *ptr2;
+	} twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -11,12 +11,13 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.sliceobject import W_SliceObject
 from pypy.objspace.std.stringtype import (
-    joined2, sliced, stringendswith, stringstartswith, wrapstr)
+    joined2, sliced, wrapstr)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.rstring import (StringBuilder, split, rsplit, replace,
+    endswith, startswith)
 
 
 class W_AbstractStringObject(W_Object):
@@ -276,38 +277,12 @@
     bylen = len(by)
     if bylen == 0:
         raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-    start = 0
-    if bylen == 1 and maxsplit < 0:
-        # fast path: uses str.rfind(character) and str.count(character)
-        by = by[0]    # annotator hack: string -> char
-        count = value.count(by)
-        res_w = [None] * (count + 1)
-        end = len(value)
-        while count >= 0:
-            assert end >= 0
-            prev = value.rfind(by, 0, end)
-            start = prev + 1
-            assert start >= 0
-            res_w[count] = sliced(space, value, start, end, w_self)
-            count -= 1
-            end = prev
-    else:
-        res_w = []
-        while maxsplit != 0:
-            next = value.find(by, start)
-            if next < 0:
-                break
-            res_w.append(sliced(space, value, start, next, w_self))
-            start = next + bylen
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-        res_w.append(sliced(space, value, start, len(value), w_self))
-
-    return space.newlist(res_w)
+    res = split(value, by, maxsplit)
+    return space.newlist_str(res)
 
 def str_rsplit__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -332,50 +307,21 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(sliced(space, value, j1, i+1, w_self))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_str(res)
 
-def make_rsplit_with_delim(funcname, sliced):
-    from rpython.tool.sourcetools import func_with_new_name
-
-    if 'Unicode' in funcname:
-        def unwrap_sep(space, w_by):
-            return w_by._value
-    else:
-        def unwrap_sep(space, w_by):
-            return space.bufferstr_w(w_by)
-
-    def fn(space, w_self, w_by, w_maxsplit=-1):
-        maxsplit = space.int_w(w_maxsplit)
-        res_w = []
-        value = w_self._value
-        end = len(value)
-        by = unwrap_sep(space, w_by)
-        bylen = len(by)
-        if bylen == 0:
-            raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-        while maxsplit != 0:
-            next = value.rfind(by, 0, end)
-            if next < 0:
-                break
-            res_w.append(sliced(space, value, next+bylen, end, w_self))
-            end = next
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        res_w.append(sliced(space, value, 0, end, w_self))
-        res_w.reverse()
-        return space.newlist(res_w)
-
-    return func_with_new_name(fn, funcname)
-
-str_rsplit__String_ANY_ANY = make_rsplit_with_delim(
-    'str_rsplit__String_ANY_ANY', sliced)
+def str_rsplit__String_ANY_ANY(space, w_self, w_by, w_maxsplit=-1):
+    maxsplit = space.int_w(w_maxsplit)
+    value = w_self._value
+    by = space.bufferstr_w(w_by)
+    if not by:
+        raise OperationError(space.w_ValueError, space.wrap("empty separator"))
+    return space.newlist_str(rsplit(value, by, maxsplit))
 
 def str_join__String_ANY(space, w_self, w_list):
     list_w = space.listview(w_list)
@@ -550,75 +496,30 @@
 
     return space.wrap(res)
 
-def _string_replace(space, input, sub, by, maxsplit):
-    if maxsplit == 0:
-        return space.wrapbytes(input)
-
-    if not sub:
-        upper = len(input)
-        if maxsplit > 0 and maxsplit < upper + 2:
-            upper = maxsplit - 1
-            assert upper >= 0
-
-        try:
-            result_size = ovfcheck(upper * len(by))
-            result_size = ovfcheck(result_size + upper)
-            result_size = ovfcheck(result_size + len(by))
-            remaining_size = len(input) - upper
-            result_size = ovfcheck(result_size + remaining_size)
-        except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                space.wrap("replace string is too long")
-            )
-        builder = StringBuilder(result_size)
-        for i in range(upper):
-            builder.append(by)
-            builder.append(input[i])
-        builder.append(by)
-        builder.append_slice(input, upper, len(input))
-    else:
-        # First compute the exact result size
-        count = input.count(sub)
-        if count > maxsplit and maxsplit > 0:
-            count = maxsplit
-        diff_len = len(by) - len(sub)
-        try:
-            result_size = ovfcheck(diff_len * count)
-            result_size = ovfcheck(result_size + len(input))
-        except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                space.wrap("replace string is too long")
-            )
-
-        builder = StringBuilder(result_size)
-        start = 0
-        sublen = len(sub)
-
-        while maxsplit != 0:
-            next = input.find(sub, start)
-            if next < 0:
-                break
-            builder.append_slice(input, start, next)
-            builder.append(by)
-            start = next + sublen
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        builder.append_slice(input, start, len(input))
-
-    return space.wrapbytes(builder.build())
-
 
 def str_replace__String_ANY_ANY_ANY(space, w_self, w_sub, w_by, w_maxsplit):
-    return _string_replace(space, w_self._value, space.buffer_w(w_sub).as_str(),
-                           space.buffer_w(w_by).as_str(),
-                           space.int_w(w_maxsplit))
+    sub = space.buffer_w(w_sub).as_str()
+    by = space.buffer_w(w_by).as_str()
+    maxsplit = space.int_w(w_maxsplit)
+    try:
+        res = replace(w_self._value, sub, by, maxsplit)
+    except OverflowError:
+        raise OperationError(space.w_OverflowError,
+            space.wrap("replace string is too long")
+        )
+    return space.wrapbytes(res)
 
 def str_replace__String_String_String_ANY(space, w_self, w_sub, w_by, w_maxsplit=-1):
-    input = w_self._value
     sub = w_sub._value
     by = w_by._value
     maxsplit = space.int_w(w_maxsplit)
-    return _string_replace(space, input, sub, by, maxsplit)
+    try:
+        res = replace(w_self._value, sub, by, maxsplit)
+    except OverflowError:
+        raise OperationError(space.w_OverflowError,
+            space.wrap("replace string is too long")
+        )
+    return space.wrapbytes(res)
 
 def _strip(space, w_self, w_chars, left, right):
     "internal function called by str_xstrip methods"
@@ -714,42 +615,42 @@
 def str_endswith__String_ANY_ANY_ANY(space, w_self, w_suffix, w_start, w_end):
     (u_self, start, end) = _convert_idx_params(space, w_self, w_start,
                                                w_end, True)
-    return space.newbool(stringendswith(
-            u_self, _suffix_to_str(space, w_suffix, 'endswith'), start, end))
+    suffix = _suffix_to_str(space, w_suffix, 'endswith')
+    return space.newbool(endswith(u_self, suffix, start, end))
 
 def str_endswith__String_String_ANY_ANY(space, w_self, w_suffix, w_start, w_end):
     (u_self, start, end) = _convert_idx_params(space, w_self, w_start,
                                                w_end, True)
-    return space.newbool(stringendswith(u_self, w_suffix._value, start, end))
+    return space.newbool(endswith(u_self, w_suffix._value, start, end))
 
 def str_endswith__String_ANY_ANY_ANY(space, w_self, w_suffix, w_start, w_end):
     u_self, start, end = _convert_idx_params(space, w_self, w_start, w_end,
                                              True)
     if not space.isinstance_w(w_suffix, space.w_tuple):
         suffix = _suffix_to_str(space, w_suffix, 'endswith')
-        return space.newbool(stringendswith(u_self, suffix, start, end))
+        return space.newbool(endswith(u_self, suffix, start, end))
 
     for w_item in space.fixedview(w_suffix):
         suffix = space.bufferstr_w(w_item)
-        if stringendswith(u_self, suffix, start, end):
+        if endswith(u_self, suffix, start, end):
             return space.w_True
     return space.w_False
 
 def str_startswith__String_String_ANY_ANY(space, w_self, w_prefix, w_start, w_end):
     (u_self, start, end) = _convert_idx_params(space, w_self, w_start,
                                                w_end, True)
-    return space.newbool(stringstartswith(u_self, w_prefix._value, start, end))
+    return space.newbool(startswith(u_self, w_prefix._value, start, end))
 
 def str_startswith__String_ANY_ANY_ANY(space, w_self, w_prefix, w_start, w_end):
     u_self, start, end = _convert_idx_params(space, w_self, w_start, w_end,
                                              True)
     if not space.isinstance_w(w_prefix, space.w_tuple):
         prefix = _suffix_to_str(space, w_prefix, 'startswith')
-        return space.newbool(stringstartswith(u_self, prefix, start, end))
+        return space.newbool(startswith(u_self, prefix, start, end))
 
     for w_item in space.fixedview(w_prefix):
         prefix = space.bufferstr_w(w_item)
-        if stringstartswith(u_self, prefix, start, end):
+        if startswith(u_self, prefix, start, end):
             return space.w_True
     return space.w_False
 
@@ -808,26 +709,7 @@
 def str_splitlines__String_ANY(space, w_self, w_keepends):
     u_keepends = space.int_w(w_keepends)  # truth value, but type checked
     data = w_self._value
-    selflen = len(data)
-    strs_w = []
-    i = j = 0
-    while i < selflen:
-        # Find a line and append it
-        while i < selflen and data[i] != '\n' and data[i] != '\r':
-            i += 1
-        # Skip the line break reading CRLF as one line break
-        eol = i
-        i += 1
-        if i < selflen and data[i-1] == '\r' and data[i] == '\n':
-            i += 1
-        if u_keepends:
-            eol = i
-        strs_w.append(sliced(space, data, j, eol, w_self))
-        j = i
-
-    if j < selflen:
-        strs_w.append(sliced(space, data, j, len(data), w_self))
-    return space.newlist(strs_w)
+    return space.newlist_str(data.splitlines(u_keepends))
 
 def str_zfill__String_ANY(space, w_self, w_width):
     input = w_self._value
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -395,28 +395,3 @@
 str_typedef.registermethods(globals())
 
 
-# ____________________________________________________________
-
-# Helpers for several string implementations
-
- at specialize.argtype(0)
- at jit.elidable
-def stringendswith(u_self, suffix, start, end):
-    begin = end - len(suffix)
-    if begin < start:
-        return False
-    for i in range(len(suffix)):
-        if u_self[begin+i] != suffix[i]:
-            return False
-    return True
-
- at specialize.argtype(0)
- at jit.elidable
-def stringstartswith(u_self, prefix, start, end):
-    stop = start + len(prefix)
-    if stop > end:
-        return False
-    for i in range(len(prefix)):
-        if u_self[start+i] != prefix[i]:
-            return False
-    return True
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -560,10 +560,14 @@
         try:
             w_l = space.call_method(w_s, "split")
             w_l2 = space.call_method(w_s, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_s, "rsplit")
+            w_l4 = space.call_method(w_s, "rsplit", space.wrap(" "))
         finally:
             del space.newlist
         assert space.listview_str(w_l) == ["a", "b", "c"]
         assert space.listview_str(w_l2) == ["a", "b", "c"]
+        assert space.listview_str(w_l3) == ["a", "b", "c"]
+        assert space.listview_str(w_l4) == ["a", "b", "c"]
 
     def test_unicode_uses_newlist_unicode(self):
         space = self.space
@@ -572,10 +576,14 @@
         try:
             w_l = space.call_method(w_u, "split")
             w_l2 = space.call_method(w_u, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_u, "rsplit")
+            w_l4 = space.call_method(w_u, "rsplit", space.wrap(" "))
         finally:
             del space.newlist
         assert space.listview_unicode(w_l) == [u"a", u"b", u"c"]
         assert space.listview_unicode(w_l2) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l3) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l4) == [u"a", u"b", u"c"]
 
     def test_pop_without_argument_is_fast(self):
         space = self.space
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -8,14 +8,13 @@
 from pypy.objspace.std.multimethod import FailedToImplement
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.sliceobject import W_SliceObject
-from pypy.objspace.std.stringobject import make_rsplit_with_delim
-from pypy.objspace.std.stringtype import stringendswith, stringstartswith
 from pypy.objspace.std.register_all import register_all
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rstring import (UnicodeBuilder, split, rsplit, replace,
+    startswith, endswith)
 from rpython.rlib.runicode import make_unicode_escape_function
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -454,14 +453,14 @@
 def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self,
                                                    w_start, w_end, True)
-    return space.newbool(stringendswith(self, w_substr._value, start, end))
+    return space.newbool(endswith(self, w_substr._value, start, end))
 
 def unicode_startswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self, w_start, w_end, True)
     # XXX this stuff can be waaay better for ootypebased backends if
     #     we re-use more of our rpython machinery (ie implement startswith
     #     with additional parameters as rpython)
-    return space.newbool(stringstartswith(self, w_substr._value, start, end))
+    return space.newbool(startswith(self, w_substr._value, start, end))
 
 def unicode_startswith__Unicode_ANY_ANY_ANY(space, w_unistr, w_prefixes,
                                               w_start, w_end):
@@ -473,7 +472,7 @@
                                              w_start, w_end, True)
     for w_prefix in space.fixedview(w_prefixes):
         prefix = space.unicode_w(w_prefix)
-        if stringstartswith(unistr, prefix, start, end):
+        if startswith(unistr, prefix, start, end):
             return space.w_True
     return space.w_False
 
@@ -487,7 +486,7 @@
                                              w_start, w_end, True)
     for w_suffix in space.fixedview(w_suffixes):
         suffix = space.unicode_w(w_suffix)
-        if stringendswith(unistr, suffix, start, end):
+        if endswith(unistr, suffix, start, end):
             return space.w_True
     return space.w_False
 
@@ -576,17 +575,17 @@
             if (self[pos] == u'\r' and pos + 1 < end and
                 self[pos + 1] == u'\n'):
                 # Count CRLF as one linebreak
-                lines.append(W_UnicodeObject(self[start:pos + keepends * 2]))
+                lines.append(self[start:pos + keepends * 2])
                 pos += 1
             else:
-                lines.append(W_UnicodeObject(self[start:pos + keepends]))
+                lines.append(self[start:pos + keepends])
             pos += 1
             start = pos
         else:
             pos += 1
     if not unicodedb.islinebreak(ord(self[end - 1])):
-        lines.append(W_UnicodeObject(self[start:]))
-    return space.newlist(lines)
+        lines.append(self[start:])
+    return space.newlist_unicode(lines)
 
 def unicode_find__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self, w_start, w_end)
@@ -656,7 +655,7 @@
     if delim_len == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap('empty separator'))
-    parts = _split_with(self, delim, maxsplit)
+    parts = split(self, delim, maxsplit)
     return space.newlist_unicode(parts)
 
 
@@ -695,70 +694,25 @@
     res.reverse()
     return space.newlist_unicode(res)
 
-def sliced(space, s, start, stop, orig_obj):
-    assert start >= 0
-    assert stop >= 0
-    if start == 0 and stop == len(s) and space.is_w(space.type(orig_obj), space.w_unicode):
-        return orig_obj
-    return space.wrap( s[start:stop])
-
-unicode_rsplit__Unicode_Unicode_ANY = make_rsplit_with_delim('unicode_rsplit__Unicode_Unicode_ANY',
-                                                             sliced)
-
-def _split_into_chars(self, maxsplit):
-    if maxsplit == 0:
-        return [self]
-    index = 0
-    end = len(self)
-    parts = [u'']
-    maxsplit -= 1
-    while maxsplit != 0:
-        if index >= end:
-            break
-        parts.append(self[index])
-        index += 1
-        maxsplit -= 1
-    parts.append(self[index:])
-    return parts
-
-def _split_with(self, with_, maxsplit=-1):
-    parts = []
-    start = 0
-    end = len(self)
-    length = len(with_)
-    while maxsplit != 0:
-        index = self.find(with_, start, end)
-        if index < 0:
-            break
-        parts.append(self[start:index])
-        start = index + length
-        maxsplit -= 1
-    parts.append(self[start:])
-    return parts
+def unicode_rsplit__Unicode_Unicode_ANY(space, w_self, w_by, w_maxsplit=-1):
+    maxsplit = space.int_w(w_maxsplit)
+    value = w_self._value
+    by = w_by._value
+    if not by:
+        raise OperationError(space.w_ValueError, space.wrap("empty separator"))
+    return space.newlist_unicode(rsplit(value, by, maxsplit))
 
 def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old,
                                                  w_new, w_maxsplit):
-    return _unicode_replace(space, w_self, w_old._value, w_new._value,
-                            w_maxsplit)
-
-def _unicode_replace(space, w_self, old, new, w_maxsplit):
-    if len(old):
-        parts = _split_with(w_self._value, old, space.int_w(w_maxsplit))
-    else:
-        self = w_self._value
-        maxsplit = space.int_w(w_maxsplit)
-        parts = _split_into_chars(self, maxsplit)
-
+    maxsplit = space.int_w(w_maxsplit)
     try:
-        one = ovfcheck(len(parts) * len(new))
-        ovfcheck(one + len(w_self._value))
+        return W_UnicodeObject(
+                replace(w_self._value, w_old._value, w_new._value, maxsplit))
     except OverflowError:
         raise OperationError(
             space.w_OverflowError,
             space.wrap("replace string is too long"))
 
-    return W_UnicodeObject(new.join(parts))
-
 
 def unicode_encode__Unicode_ANY_ANY(space, w_unistr,
                                     w_encoding=None,
@@ -804,7 +758,7 @@
 def unicode_expandtabs__Unicode_ANY(space, w_self, w_tabsize):
     self = w_self._value
     tabsize  = space.int_w(w_tabsize)
-    parts = _split_with(self, u'\t')
+    parts = self.split(u'\t')
     result = [parts[0]]
     prevsize = 0
     for ch in parts[0]:
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -1,25 +1,50 @@
 """ String builder interface and string functions
 """
+import sys
 
 from rpython.annotator.model import (SomeObject, SomeString, s_None, SomeChar,
     SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePtr, SomePBC)
-from rpython.rlib.objectmodel import newlist_hint
+from rpython.rlib.objectmodel import newlist_hint, specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pairtype
+from rpython.rlib import jit
 
 
 # -------------- public API for string functions -----------------------
+
+ at specialize.argtype(0)
 def split(value, by, maxsplit=-1):
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     bylen = len(by)
     if bylen == 0:
         raise ValueError("empty separator")
 
+    start = 0
+    if bylen == 1:
+        # fast path: uses str.rfind(character) and str.count(character)
+        by = by[0]    # annotator hack: string -> char
+        count = value.count(by)
+        if 0 <= maxsplit < count:
+            count = maxsplit
+        res = newlist_hint(count + 1)
+        while count > 0:
+            next = value.find(by, start)
+            assert next >= 0 # cannot fail due to the value.count above
+            res.append(value[start:next])
+            start = next + bylen
+            count -= 1
+        res.append(value[start:len(value)])
+        return res
+
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
         res = []
-    start = 0
+
     while maxsplit != 0:
         next = value.find(by, start)
         if next < 0:
@@ -32,7 +57,12 @@
     return res
 
 
+ at specialize.argtype(0)
 def rsplit(value, by, maxsplit=-1):
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
@@ -54,6 +84,109 @@
     res.reverse()
     return res
 
+
+ at specialize.argtype(0)
+def replace(input, sub, by, maxsplit=-1):
+    if isinstance(input, str):
+        assert isinstance(sub, str)
+        assert isinstance(by, str)
+        Builder = StringBuilder
+    else:
+        assert isinstance(sub, unicode)
+        assert isinstance(by, unicode)
+        Builder = UnicodeBuilder
+    if maxsplit == 0:
+        return input
+
+    if not sub:
+        upper = len(input)
+        if maxsplit > 0 and maxsplit < upper + 2:
+            upper = maxsplit - 1
+            assert upper >= 0
+
+        try:
+            result_size = ovfcheck(upper * len(by))
+            result_size = ovfcheck(result_size + upper)
+            result_size = ovfcheck(result_size + len(by))
+            remaining_size = len(input) - upper
+            result_size = ovfcheck(result_size + remaining_size)
+        except OverflowError:
+            raise
+        builder = Builder(result_size)
+        for i in range(upper):
+            builder.append(by)
+            builder.append(input[i])
+        builder.append(by)
+        builder.append_slice(input, upper, len(input))
+    else:
+        # First compute the exact result size
+        count = input.count(sub)
+        if count > maxsplit and maxsplit > 0:
+            count = maxsplit
+        diff_len = len(by) - len(sub)
+        try:
+            result_size = ovfcheck(diff_len * count)
+            result_size = ovfcheck(result_size + len(input))
+        except OverflowError:
+            raise
+
+        builder = Builder(result_size)
+        start = 0
+        sublen = len(sub)
+
+        while maxsplit != 0:
+            next = input.find(sub, start)
+            if next < 0:
+                break
+            builder.append_slice(input, start, next)
+            builder.append(by)
+            start = next + sublen
+            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+        builder.append_slice(input, start, len(input))
+
+    return builder.build()
+
+def _normalize_start_end(length, start, end):
+    if start < 0:
+        start += length
+        if start < 0:
+            start = 0
+    if end < 0:
+        end += length
+        if end < 0:
+            end = 0
+    elif end > length:
+        end = length
+    return start, end
+
+ at specialize.argtype(0)
+ at jit.elidable
+def startswith(u_self, prefix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    stop = start + len(prefix)
+    if stop > end:
+        return False
+    for i in range(len(prefix)):
+        if u_self[start+i] != prefix[i]:
+            return False
+    return True
+
+ at specialize.argtype(0)
+ at jit.elidable
+def endswith(u_self, suffix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    begin = end - len(suffix)
+    if begin < start:
+        return False
+    for i in range(len(suffix)):
+        if u_self[begin+i] != suffix[i]:
+            return False
+    return True
+
+
 # -------------- public API ---------------------------------
 
 INIT_SIZE = 100 # XXX tweak
@@ -271,3 +404,5 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+
+
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -1,6 +1,8 @@
 import sys, py
 
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit
+from rpython.rlib.rstring import replace, startswith, endswith
+from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin
 
 def test_split():
     assert split("", 'x') == ['']
@@ -10,9 +12,21 @@
     assert split('a|b|c|d', '|') == ['a', 'b', 'c', 'd']
     assert split('a|b|c|d', '|', 2) == ['a', 'b', 'c|d']
     assert split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+    assert split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
     assert split('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, split, 'abc', '')
 
+def test_split_unicode():
+    assert split(u"", u'x') == [u'']
+    assert split(u"a", u"a", 1) == [u'', u'']
+    assert split(u" ", u" ", 1) == [u'', u'']
+    assert split(u"aa", u"a", 2) == [u'', u'', u'']
+    assert split(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert split(u'a|b|c|d', u'|', 2) == [u'a', u'b', u'c|d']
+    assert split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert split(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, split, u'abc', u'')
+
 def test_rsplit():
     assert rsplit("a", "a", 1) == ['', '']
     assert rsplit(" ", " ", 1) == ['', '']
@@ -23,6 +37,111 @@
     assert rsplit('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, rsplit, "abc", '')
 
+def test_rsplit_unicode():
+    assert rsplit(u"a", u"a", 1) == [u'', u'']
+    assert rsplit(u" ", u" ", 1) == [u'', u'']
+    assert rsplit(u"aa", u"a", 2) == [u'', u'', u'']
+    assert rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+    assert rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, rsplit, u"abc", u'')
+
+def test_string_replace():
+    assert replace('one!two!three!', '!', '@', 1) == 'one at two!three!'
+    assert replace('one!two!three!', '!', '') == 'onetwothree'
+    assert replace('one!two!three!', '!', '@', 2) == 'one at two@three!'
+    assert replace('one!two!three!', '!', '@', 3) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 4) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 0) == 'one!two!three!'
+    assert replace('one!two!three!', '!', '@') == 'one at two@three@'
+    assert replace('one!two!three!', 'x', '@') == 'one!two!three!'
+    assert replace('one!two!three!', 'x', '@', 2) == 'one!two!three!'
+    assert replace('abc', '', '-') == '-a-b-c-'
+    assert replace('abc', '', '-', 3) == '-a-b-c'
+    assert replace('abc', '', '-', 0) == 'abc'
+    assert replace('', '', '') == ''
+    assert replace('', '', 'a') == 'a'
+    assert replace('abc', 'ab', '--', 0) == 'abc'
+    assert replace('abc', 'xy', '--') == 'abc'
+    assert replace('123', '123', '') == ''
+    assert replace('123123', '123', '') == ''
+    assert replace('123x123', '123', '') == 'x'
+
+def test_string_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = "a" * (2**16)
+    with py.test.raises(OverflowError):
+        replace(s, "", s)
+    with py.test.raises(OverflowError):
+        replace(s, "a", s)
+    with py.test.raises(OverflowError):
+        replace(s, "a", s, len(s) - 10)
+
+def test_unicode_replace():
+    assert replace(u'one!two!three!', u'!', u'@', 1) == u'one at two!three!'
+    assert replace(u'one!two!three!', u'!', u'') == u'onetwothree'
+    assert replace(u'one!two!three!', u'!', u'@', 2) == u'one at two@three!'
+    assert replace(u'one!two!three!', u'!', u'@', 3) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 4) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 0) == u'one!two!three!'
+    assert replace(u'one!two!three!', u'!', u'@') == u'one at two@three@'
+    assert replace(u'one!two!three!', u'x', u'@') == u'one!two!three!'
+    assert replace(u'one!two!three!', u'x', u'@', 2) == u'one!two!three!'
+    assert replace(u'abc', u'', u'-') == u'-a-b-c-'
+    assert replace(u'abc', u'', u'-', 3) == u'-a-b-c'
+    assert replace(u'abc', u'', u'-', 0) == u'abc'
+    assert replace(u'', u'', u'') == u''
+    assert replace(u'', u'', u'a') == u'a'
+    assert replace(u'abc', u'ab', u'--', 0) == u'abc'
+    assert replace(u'abc', u'xy', u'--') == u'abc'
+    assert replace(u'123', u'123', u'') == u''
+    assert replace(u'123123', u'123', u'') == u''
+    assert replace(u'123x123', u'123', u'') == u'x'
+
+def test_unicode_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = u"a" * (2**16)
+    with py.test.raises(OverflowError):
+        replace(s, u"", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s, len(s) - 10)
+
+def test_startswith():
+    assert startswith('ab', 'ab') is True
+    assert startswith('ab', 'a') is True
+    assert startswith('ab', '') is True
+    assert startswith('x', 'a') is False
+    assert startswith('x', 'x') is True
+    assert startswith('', '') is True
+    assert startswith('', 'a') is False
+    assert startswith('x', 'xx') is False
+    assert startswith('y', 'xx') is False
+    assert startswith('ab', 'a', 0) is True
+    assert startswith('ab', 'a', 1) is False
+    assert startswith('ab', 'b', 1) is True
+    assert startswith('abc', 'bc', 1, 2) is False
+    assert startswith('abc', 'c', -1, 4) is True
+
+def test_endswith():
+    assert endswith('ab', 'ab') is True
+    assert endswith('ab', 'b') is True
+    assert endswith('ab', '') is True
+    assert endswith('x', 'a') is False
+    assert endswith('x', 'x') is True
+    assert endswith('', '') is True
+    assert endswith('', 'a') is False
+    assert endswith('x', 'xx') is False
+    assert endswith('y', 'xx') is False
+    assert endswith('abc', 'ab', 0, 2) is True
+    assert endswith('abc', 'bc', 1) is True
+    assert endswith('abc', 'bc', 2) is False
+    assert endswith('abc', 'b', -3, -1) is True
+
 def test_string_builder():
     s = StringBuilder()
     s.append("a")
@@ -42,4 +161,32 @@
     s.append_multiple_char(u'd', 4)
     assert s.build() == 'aabcbdddd'
     assert isinstance(s.build(), unicode)
-        
+
+
+class TestTranslates(LLRtypeMixin, BaseRtypingTest):
+    def test_split_rsplit(self):
+        def fn():
+            res = True
+            res = res and split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
+            res = res and split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            res = res and split(u'endcase test', u'test') == [u'endcase ', u'']
+            res = res and rsplit('a|b|c|d', '|', 2) == ['a|b', 'c', 'd']
+            res = res and rsplit('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+            res = res and rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+            res = res and rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            return res
+        res = self.interpret(fn, [])
+        assert res
+
+    def test_replace(self):
+        def fn():
+            res = True
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            return res
+        res = self.interpret(fn, [])
+        assert res

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:33 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:33 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: now that newlist_unicode exists,
 newlist_str needs to be for bytestrings.
Message-ID: <20130617123533.4BC701C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3k
Changeset: r64916:011a13df3ba7
Date: 2013-06-17 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/011a13df3ba7/

Log:	now that newlist_unicode exists, newlist_str needs to be for
	bytestrings. StringStrategy is still broken, so just fall back to
	the default of immediately wrapping things.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -912,7 +912,7 @@
         return (None, None)
 
     def newlist_str(self, list_s):
-        return self.newlist([self.wrap(s) for s in list_s])
+        return self.newlist([self.wrapbytes(s) for s in list_s])
 
     def newlist_unicode(self, list_u):
         return self.newlist([self.wrap(u) for u in list_u])
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -163,13 +163,12 @@
             self.switch_to_object_strategy()
         return self
 
-    @staticmethod
-    def newlist_str(space, list_s):
-        # XXX: StringListStrategy is currently broken
-        #strategy = space.fromcache(StringListStrategy)
-        strategy = space.fromcache(UnicodeListStrategy)
-        storage = strategy.erase(list_s)
-        return W_ListObject.from_storage_and_strategy(space, storage, strategy)
+    # XXX: StringListStrategy is currently broken
+    #@staticmethod
+    #def newlist_str(space, list_s):
+    #    strategy = space.fromcache(StringListStrategy)
+    #    storage = strategy.erase(list_s)
+    #    return W_ListObject.from_storage_and_strategy(space, storage, strategy)
 
     @staticmethod
     def newlist_unicode(space, list_u):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -314,8 +314,10 @@
         assert not list_w or sizehint == -1
         return W_ListObject(self, list_w, sizehint)
 
-    def newlist_str(self, list_s):
-        return W_ListObject.newlist_str(self, list_s)
+    # XXX: StringListStrategy is currently broken
+    # use the default implementation, which simply wraps
+    #def newlist_str(self, list_s):
+    #    return W_ListObject.newlist_str(self, list_s)
 
     def newlist_unicode(self, list_u):
         return W_ListObject.newlist_unicode(self, list_u)

From noreply at buildbot.pypy.org  Mon Jun 17 14:35:34 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 17 Jun 2013 14:35:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20130617123534.8CB751C31D2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64917:53b7cc76daf1
Date: 2013-06-17 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/53b7cc76daf1/

Log:	merge

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -904,6 +904,9 @@
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
+    def newlist_unicode(self, list_u):
+        return self.newlist([self.wrap(u) for u in list_u])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -166,6 +166,12 @@
         storage = strategy.erase(list_s)
         return W_ListObject.from_storage_and_strategy(space, storage, strategy)
 
+    @staticmethod
+    def newlist_unicode(space, list_u):
+        strategy = space.fromcache(UnicodeListStrategy)
+        storage = strategy.erase(list_u)
+        return W_ListObject.from_storage_and_strategy(space, storage, strategy)
+
     def __repr__(self):
         """ representation for debugging purposes """
         return "%s(%s, %s)" % (self.__class__.__name__, self.strategy,
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -294,6 +294,9 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
+    def newlist_unicode(self, list_u):
+        return W_ListObject.newlist_unicode(self, list_u)
+
     def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -11,12 +11,13 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
 from pypy.objspace.std.stringtype import (
-    joined2, sliced, stringendswith, stringstartswith, wrapchar, wrapstr)
+    joined2, sliced, wrapchar, wrapstr)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import StringBuilder, split
+from rpython.rlib.rstring import (StringBuilder, split, rsplit, replace,
+    endswith, startswith)
 
 
 class W_AbstractStringObject(W_Object):
@@ -287,31 +288,12 @@
     bylen = len(by)
     if bylen == 0:
         raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-    if bylen == 1 and maxsplit < 0:
-        res = []
-        start = 0
-        # fast path: uses str.rfind(character) and str.count(character)
-        by = by[0]    # annotator hack: string -> char
-        count = value.count(by)
-        res = [None] * (count + 1)
-        end = len(value)
-        while count >= 0:
-            assert end >= 0
-            prev = value.rfind(by, 0, end)
-            start = prev + 1
-            assert start >= 0
-            res[count] = value[start:end]
-            count -= 1
-            end = prev
-    else:
-        res = split(value, by, maxsplit)
-
+    res = split(value, by, maxsplit)
     return space.newlist_str(res)
 
 def str_rsplit__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -336,43 +318,21 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(sliced(space, value, j1, i+1, w_self))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_str(res)
 
-def make_rsplit_with_delim(funcname, sliced):
-    from rpython.tool.sourcetools import func_with_new_name
-
-    def fn(space, w_self, w_by, w_maxsplit=-1):
-        maxsplit = space.int_w(w_maxsplit)
-        res_w = []
-        value = w_self._value
-        end = len(value)
-        by = w_by._value
-        bylen = len(by)
-        if bylen == 0:
-            raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-        while maxsplit != 0:
-            next = value.rfind(by, 0, end)
-            if next < 0:
-                break
-            res_w.append(sliced(space, value, next+bylen, end, w_self))
-            end = next
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        res_w.append(sliced(space, value, 0, end, w_self))
-        res_w.reverse()
-        return space.newlist(res_w)
-
-    return func_with_new_name(fn, funcname)
-
-str_rsplit__String_String_ANY = make_rsplit_with_delim('str_rsplit__String_String_ANY',
-                                                       sliced)
+def str_rsplit__String_String_ANY(space, w_self, w_by, w_maxsplit=-1):
+    maxsplit = space.int_w(w_maxsplit)
+    value = w_self._value
+    by = w_by._value
+    if not by:
+        raise OperationError(space.w_ValueError, space.wrap("empty separator"))
+    return space.newlist_str(rsplit(value, by, maxsplit))
 
 def str_join__String_ANY(space, w_self, w_list):
     l = space.listview_str(w_list)
@@ -524,75 +484,30 @@
 
     return space.wrap(res)
 
-def _string_replace(space, input, sub, by, maxsplit):
-    if maxsplit == 0:
-        return space.wrap(input)
-
-    if not sub:
-        upper = len(input)
-        if maxsplit > 0 and maxsplit < upper + 2:
-            upper = maxsplit - 1
-            assert upper >= 0
-
-        try:
-            result_size = ovfcheck(upper * len(by))
-            result_size = ovfcheck(result_size + upper)
-            result_size = ovfcheck(result_size + len(by))
-            remaining_size = len(input) - upper
-            result_size = ovfcheck(result_size + remaining_size)
-        except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                space.wrap("replace string is too long")
-            )
-        builder = StringBuilder(result_size)
-        for i in range(upper):
-            builder.append(by)
-            builder.append(input[i])
-        builder.append(by)
-        builder.append_slice(input, upper, len(input))
-    else:
-        # First compute the exact result size
-        count = input.count(sub)
-        if count > maxsplit and maxsplit > 0:
-            count = maxsplit
-        diff_len = len(by) - len(sub)
-        try:
-            result_size = ovfcheck(diff_len * count)
-            result_size = ovfcheck(result_size + len(input))
-        except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                space.wrap("replace string is too long")
-            )
-
-        builder = StringBuilder(result_size)
-        start = 0
-        sublen = len(sub)
-
-        while maxsplit != 0:
-            next = input.find(sub, start)
-            if next < 0:
-                break
-            builder.append_slice(input, start, next)
-            builder.append(by)
-            start = next + sublen
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        builder.append_slice(input, start, len(input))
-
-    return space.wrap(builder.build())
-
 
 def str_replace__String_ANY_ANY_ANY(space, w_self, w_sub, w_by, w_maxsplit):
-    return _string_replace(space, w_self._value, space.buffer_w(w_sub).as_str(),
-                           space.buffer_w(w_by).as_str(),
-                           space.int_w(w_maxsplit))
+    sub = space.buffer_w(w_sub).as_str()
+    by = space.buffer_w(w_by).as_str()
+    maxsplit = space.int_w(w_maxsplit)
+    try:
+        res = replace(w_self._value, sub, by, maxsplit)
+    except OverflowError:
+        raise OperationError(space.w_OverflowError,
+            space.wrap("replace string is too long")
+        )
+    return space.wrap(res)
 
 def str_replace__String_String_String_ANY(space, w_self, w_sub, w_by, w_maxsplit=-1):
-    input = w_self._value
     sub = w_sub._value
     by = w_by._value
     maxsplit = space.int_w(w_maxsplit)
-    return _string_replace(space, input, sub, by, maxsplit)
+    try:
+        res = replace(w_self._value, sub, by, maxsplit)
+    except OverflowError:
+        raise OperationError(space.w_OverflowError,
+            space.wrap("replace string is too long")
+        )
+    return space.wrap(res)
 
 def _strip(space, w_self, w_chars, left, right):
     "internal function called by str_xstrip methods"
@@ -679,7 +594,7 @@
 def str_endswith__String_String_ANY_ANY(space, w_self, w_suffix, w_start, w_end):
     (u_self, start, end) = _convert_idx_params(space, w_self, w_start,
                                                w_end, True)
-    return space.newbool(stringendswith(u_self, w_suffix._value, start, end))
+    return space.newbool(endswith(u_self, w_suffix._value, start, end))
 
 def str_endswith__String_ANY_ANY_ANY(space, w_self, w_suffixes, w_start, w_end):
     if not space.isinstance_w(w_suffixes, space.w_tuple):
@@ -692,14 +607,14 @@
             return space.call_method(w_u, "endswith", w_suffixes, w_start,
                                      w_end)
         suffix = space.str_w(w_suffix)
-        if stringendswith(u_self, suffix, start, end):
+        if endswith(u_self, suffix, start, end):
             return space.w_True
     return space.w_False
 
 def str_startswith__String_String_ANY_ANY(space, w_self, w_prefix, w_start, w_end):
     (u_self, start, end) = _convert_idx_params(space, w_self, w_start,
                                                w_end, True)
-    return space.newbool(stringstartswith(u_self, w_prefix._value, start, end))
+    return space.newbool(startswith(u_self, w_prefix._value, start, end))
 
 def str_startswith__String_ANY_ANY_ANY(space, w_self, w_prefixes, w_start, w_end):
     if not space.isinstance_w(w_prefixes, space.w_tuple):
@@ -712,7 +627,7 @@
             return space.call_method(w_u, "startswith", w_prefixes, w_start,
                                      w_end)
         prefix = space.str_w(w_prefix)
-        if stringstartswith(u_self, prefix, start, end):
+        if startswith(u_self, prefix, start, end):
             return space.w_True
     return space.w_False
 
@@ -768,26 +683,7 @@
 def str_splitlines__String_ANY(space, w_self, w_keepends):
     u_keepends = space.int_w(w_keepends)  # truth value, but type checked
     data = w_self._value
-    selflen = len(data)
-    strs_w = []
-    i = j = 0
-    while i < selflen:
-        # Find a line and append it
-        while i < selflen and data[i] != '\n' and data[i] != '\r':
-            i += 1
-        # Skip the line break reading CRLF as one line break
-        eol = i
-        i += 1
-        if i < selflen and data[i-1] == '\r' and data[i] == '\n':
-            i += 1
-        if u_keepends:
-            eol = i
-        strs_w.append(sliced(space, data, j, eol, w_self))
-        j = i
-
-    if j < selflen:
-        strs_w.append(sliced(space, data, j, len(data), w_self))
-    return space.newlist(strs_w)
+    return space.newlist_str(data.splitlines(u_keepends))
 
 def str_zfill__String_ANY(space, w_self, w_width):
     input = w_self._value
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -295,28 +295,3 @@
 str_typedef.registermethods(globals())
 
 
-# ____________________________________________________________
-
-# Helpers for several string implementations
-
- at specialize.argtype(0)
- at jit.elidable
-def stringendswith(u_self, suffix, start, end):
-    begin = end - len(suffix)
-    if begin < start:
-        return False
-    for i in range(len(suffix)):
-        if u_self[begin+i] != suffix[i]:
-            return False
-    return True
-
- at specialize.argtype(0)
- at jit.elidable
-def stringstartswith(u_self, prefix, start, end):
-    stop = start + len(prefix)
-    if stop > end:
-        return False
-    for i in range(len(prefix)):
-        if u_self[start+i] != prefix[i]:
-            return False
-    return True
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -555,10 +555,30 @@
         try:
             w_l = space.call_method(w_s, "split")
             w_l2 = space.call_method(w_s, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_s, "rsplit")
+            w_l4 = space.call_method(w_s, "rsplit", space.wrap(" "))
         finally:
             del space.newlist
         assert space.listview_str(w_l) == ["a", "b", "c"]
         assert space.listview_str(w_l2) == ["a", "b", "c"]
+        assert space.listview_str(w_l3) == ["a", "b", "c"]
+        assert space.listview_str(w_l4) == ["a", "b", "c"]
+
+    def test_unicode_uses_newlist_unicode(self):
+        space = self.space
+        w_u = space.wrap(u"a b c")
+        space.newlist = None
+        try:
+            w_l = space.call_method(w_u, "split")
+            w_l2 = space.call_method(w_u, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_u, "rsplit")
+            w_l4 = space.call_method(w_u, "rsplit", space.wrap(" "))
+        finally:
+            del space.newlist
+        assert space.listview_unicode(w_l) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l2) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l3) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l4) == [u"a", u"b", u"c"]
 
     def test_pop_without_argument_is_fast(self):
         space = self.space
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -8,15 +8,14 @@
 from pypy.objspace.std.multimethod import FailedToImplement
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
-from pypy.objspace.std.stringobject import (
-    W_StringObject, make_rsplit_with_delim)
-from pypy.objspace.std.stringtype import stringendswith, stringstartswith
+from pypy.objspace.std.stringobject import W_StringObject
 from pypy.objspace.std.register_all import register_all
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rstring import (UnicodeBuilder, split, rsplit, replace,
+    startswith, endswith)
 from rpython.rlib.runicode import make_unicode_escape_function
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -490,14 +489,14 @@
 def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self,
                                                    w_start, w_end, True)
-    return space.newbool(stringendswith(self, w_substr._value, start, end))
+    return space.newbool(endswith(self, w_substr._value, start, end))
 
 def unicode_startswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self, w_start, w_end, True)
     # XXX this stuff can be waaay better for ootypebased backends if
     #     we re-use more of our rpython machinery (ie implement startswith
     #     with additional parameters as rpython)
-    return space.newbool(stringstartswith(self, w_substr._value, start, end))
+    return space.newbool(startswith(self, w_substr._value, start, end))
 
 def unicode_startswith__Unicode_ANY_ANY_ANY(space, w_unistr, w_prefixes,
                                               w_start, w_end):
@@ -507,7 +506,7 @@
                                              w_start, w_end, True)
     for w_prefix in space.fixedview(w_prefixes):
         prefix = space.unicode_w(w_prefix)
-        if stringstartswith(unistr, prefix, start, end):
+        if startswith(unistr, prefix, start, end):
             return space.w_True
     return space.w_False
 
@@ -519,7 +518,7 @@
                                              w_start, w_end, True)
     for w_suffix in space.fixedview(w_suffixes):
         suffix = space.unicode_w(w_suffix)
-        if stringendswith(unistr, suffix, start, end):
+        if endswith(unistr, suffix, start, end):
             return space.w_True
     return space.w_False
 
@@ -608,17 +607,17 @@
             if (self[pos] == u'\r' and pos + 1 < end and
                 self[pos + 1] == u'\n'):
                 # Count CRLF as one linebreak
-                lines.append(W_UnicodeObject(self[start:pos + keepends * 2]))
+                lines.append(self[start:pos + keepends * 2])
                 pos += 1
             else:
-                lines.append(W_UnicodeObject(self[start:pos + keepends]))
+                lines.append(self[start:pos + keepends])
             pos += 1
             start = pos
         else:
             pos += 1
     if not unicodedb.islinebreak(ord(self[end - 1])):
-        lines.append(W_UnicodeObject(self[start:]))
-    return space.newlist(lines)
+        lines.append(self[start:])
+    return space.newlist_unicode(lines)
 
 def unicode_find__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
     self, start, end = _convert_idx_params(space, w_self, w_start, w_end)
@@ -650,7 +649,7 @@
 
 def unicode_split__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     length = len(value)
     i = 0
@@ -673,12 +672,12 @@
             maxsplit -= 1   # NB. if it's already < 0, it stays < 0
 
         # the word is value[i:j]
-        res_w.append(W_UnicodeObject(value[i:j]))
+        res.append(value[i:j])
 
         # continue to look from the character following the space after the word
         i = j + 1
 
-    return space.newlist(res_w)
+    return space.newlist_unicode(res)
 
 def unicode_split__Unicode_Unicode_ANY(space, w_self, w_delim, w_maxsplit):
     self = w_self._value
@@ -688,13 +687,13 @@
     if delim_len == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap('empty separator'))
-    parts = _split_with(self, delim, maxsplit)
-    return space.newlist([W_UnicodeObject(part) for part in parts])
+    parts = split(self, delim, maxsplit)
+    return space.newlist_unicode(parts)
 
 
 def unicode_rsplit__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -719,59 +718,32 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(W_UnicodeObject(value[j1:i+1]))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_unicode(res)
 
-def sliced(space, s, start, stop, orig_obj):
-    assert start >= 0
-    assert stop >= 0
-    if start == 0 and stop == len(s) and space.is_w(space.type(orig_obj), space.w_unicode):
-        return orig_obj
-    return space.wrap( s[start:stop])
-
-unicode_rsplit__Unicode_Unicode_ANY = make_rsplit_with_delim('unicode_rsplit__Unicode_Unicode_ANY',
-                                                             sliced)
-
-def _split_into_chars(self, maxsplit):
-    if maxsplit == 0:
-        return [self]
-    index = 0
-    end = len(self)
-    parts = [u'']
-    maxsplit -= 1
-    while maxsplit != 0:
-        if index >= end:
-            break
-        parts.append(self[index])
-        index += 1
-        maxsplit -= 1
-    parts.append(self[index:])
-    return parts
-
-def _split_with(self, with_, maxsplit=-1):
-    parts = []
-    start = 0
-    end = len(self)
-    length = len(with_)
-    while maxsplit != 0:
-        index = self.find(with_, start, end)
-        if index < 0:
-            break
-        parts.append(self[start:index])
-        start = index + length
-        maxsplit -= 1
-    parts.append(self[start:])
-    return parts
+def unicode_rsplit__Unicode_Unicode_ANY(space, w_self, w_by, w_maxsplit=-1):
+    maxsplit = space.int_w(w_maxsplit)
+    value = w_self._value
+    by = w_by._value
+    if not by:
+        raise OperationError(space.w_ValueError, space.wrap("empty separator"))
+    return space.newlist_unicode(rsplit(value, by, maxsplit))
 
 def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old,
                                                  w_new, w_maxsplit):
-    return _unicode_replace(space, w_self, w_old._value, w_new._value,
-                            w_maxsplit)
+    maxsplit = space.int_w(w_maxsplit)
+    try:
+        return W_UnicodeObject(
+                replace(w_self._value, w_old._value, w_new._value, maxsplit))
+    except OverflowError:
+        raise OperationError(
+            space.w_OverflowError,
+            space.wrap("replace string is too long"))
 
 def unicode_replace__Unicode_ANY_ANY_ANY(space, w_self, w_old, w_new,
                                          w_maxsplit):
@@ -783,27 +755,14 @@
         new = unicode(space.bufferstr_w(w_new))
     else:
         new = space.unicode_w(w_new)
-    return _unicode_replace(space, w_self, old, new, w_maxsplit)
-
-def _unicode_replace(space, w_self, old, new, w_maxsplit):
-    if len(old):
-        parts = _split_with(w_self._value, old, space.int_w(w_maxsplit))
-    else:
-        self = w_self._value
-        maxsplit = space.int_w(w_maxsplit)
-        parts = _split_into_chars(self, maxsplit)
-
+    maxsplit = space.int_w(w_maxsplit)
     try:
-        one = ovfcheck(len(parts) * len(new))
-        ovfcheck(one + len(w_self._value))
+        return W_UnicodeObject(replace(w_self._value, old, new, maxsplit))
     except OverflowError:
         raise OperationError(
             space.w_OverflowError,
             space.wrap("replace string is too long"))
 
-    return W_UnicodeObject(new.join(parts))
-
-
 def unicode_encode__Unicode_ANY_ANY(space, w_unistr,
                                     w_encoding=None,
                                     w_errors=None):
@@ -848,7 +807,7 @@
 def unicode_expandtabs__Unicode_ANY(space, w_self, w_tabsize):
     self = w_self._value
     tabsize  = space.int_w(w_tabsize)
-    parts = _split_with(self, u'\t')
+    parts = self.split(u'\t')
     result = [parts[0]]
     prevsize = 0
     for ch in parts[0]:
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -1,25 +1,50 @@
 """ String builder interface and string functions
 """
+import sys
 
 from rpython.annotator.model import (SomeObject, SomeString, s_None, SomeChar,
     SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePtr, SomePBC)
-from rpython.rlib.objectmodel import newlist_hint
+from rpython.rlib.objectmodel import newlist_hint, specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pairtype
+from rpython.rlib import jit
 
 
 # -------------- public API for string functions -----------------------
+
+ at specialize.argtype(0)
 def split(value, by, maxsplit=-1):
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     bylen = len(by)
     if bylen == 0:
         raise ValueError("empty separator")
 
+    start = 0
+    if bylen == 1:
+        # fast path: uses str.rfind(character) and str.count(character)
+        by = by[0]    # annotator hack: string -> char
+        count = value.count(by)
+        if 0 <= maxsplit < count:
+            count = maxsplit
+        res = newlist_hint(count + 1)
+        while count > 0:
+            next = value.find(by, start)
+            assert next >= 0 # cannot fail due to the value.count above
+            res.append(value[start:next])
+            start = next + bylen
+            count -= 1
+        res.append(value[start:len(value)])
+        return res
+
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
         res = []
-    start = 0
+
     while maxsplit != 0:
         next = value.find(by, start)
         if next < 0:
@@ -32,7 +57,12 @@
     return res
 
 
+ at specialize.argtype(0)
 def rsplit(value, by, maxsplit=-1):
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
@@ -54,6 +84,109 @@
     res.reverse()
     return res
 
+
+ at specialize.argtype(0)
+def replace(input, sub, by, maxsplit=-1):
+    if isinstance(input, str):
+        assert isinstance(sub, str)
+        assert isinstance(by, str)
+        Builder = StringBuilder
+    else:
+        assert isinstance(sub, unicode)
+        assert isinstance(by, unicode)
+        Builder = UnicodeBuilder
+    if maxsplit == 0:
+        return input
+
+    if not sub:
+        upper = len(input)
+        if maxsplit > 0 and maxsplit < upper + 2:
+            upper = maxsplit - 1
+            assert upper >= 0
+
+        try:
+            result_size = ovfcheck(upper * len(by))
+            result_size = ovfcheck(result_size + upper)
+            result_size = ovfcheck(result_size + len(by))
+            remaining_size = len(input) - upper
+            result_size = ovfcheck(result_size + remaining_size)
+        except OverflowError:
+            raise
+        builder = Builder(result_size)
+        for i in range(upper):
+            builder.append(by)
+            builder.append(input[i])
+        builder.append(by)
+        builder.append_slice(input, upper, len(input))
+    else:
+        # First compute the exact result size
+        count = input.count(sub)
+        if count > maxsplit and maxsplit > 0:
+            count = maxsplit
+        diff_len = len(by) - len(sub)
+        try:
+            result_size = ovfcheck(diff_len * count)
+            result_size = ovfcheck(result_size + len(input))
+        except OverflowError:
+            raise
+
+        builder = Builder(result_size)
+        start = 0
+        sublen = len(sub)
+
+        while maxsplit != 0:
+            next = input.find(sub, start)
+            if next < 0:
+                break
+            builder.append_slice(input, start, next)
+            builder.append(by)
+            start = next + sublen
+            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+        builder.append_slice(input, start, len(input))
+
+    return builder.build()
+
+def _normalize_start_end(length, start, end):
+    if start < 0:
+        start += length
+        if start < 0:
+            start = 0
+    if end < 0:
+        end += length
+        if end < 0:
+            end = 0
+    elif end > length:
+        end = length
+    return start, end
+
+ at specialize.argtype(0)
+ at jit.elidable
+def startswith(u_self, prefix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    stop = start + len(prefix)
+    if stop > end:
+        return False
+    for i in range(len(prefix)):
+        if u_self[start+i] != prefix[i]:
+            return False
+    return True
+
+ at specialize.argtype(0)
+ at jit.elidable
+def endswith(u_self, suffix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    begin = end - len(suffix)
+    if begin < start:
+        return False
+    for i in range(len(suffix)):
+        if u_self[begin+i] != suffix[i]:
+            return False
+    return True
+
+
 # -------------- public API ---------------------------------
 
 INIT_SIZE = 100 # XXX tweak
@@ -271,3 +404,5 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+
+
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -1,6 +1,8 @@
 import sys, py
 
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit
+from rpython.rlib.rstring import replace, startswith, endswith
+from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin
 
 def test_split():
     assert split("", 'x') == ['']
@@ -10,9 +12,21 @@
     assert split('a|b|c|d', '|') == ['a', 'b', 'c', 'd']
     assert split('a|b|c|d', '|', 2) == ['a', 'b', 'c|d']
     assert split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+    assert split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
     assert split('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, split, 'abc', '')
 
+def test_split_unicode():
+    assert split(u"", u'x') == [u'']
+    assert split(u"a", u"a", 1) == [u'', u'']
+    assert split(u" ", u" ", 1) == [u'', u'']
+    assert split(u"aa", u"a", 2) == [u'', u'', u'']
+    assert split(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert split(u'a|b|c|d', u'|', 2) == [u'a', u'b', u'c|d']
+    assert split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert split(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, split, u'abc', u'')
+
 def test_rsplit():
     assert rsplit("a", "a", 1) == ['', '']
     assert rsplit(" ", " ", 1) == ['', '']
@@ -23,6 +37,111 @@
     assert rsplit('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, rsplit, "abc", '')
 
+def test_rsplit_unicode():
+    assert rsplit(u"a", u"a", 1) == [u'', u'']
+    assert rsplit(u" ", u" ", 1) == [u'', u'']
+    assert rsplit(u"aa", u"a", 2) == [u'', u'', u'']
+    assert rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+    assert rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, rsplit, u"abc", u'')
+
+def test_string_replace():
+    assert replace('one!two!three!', '!', '@', 1) == 'one at two!three!'
+    assert replace('one!two!three!', '!', '') == 'onetwothree'
+    assert replace('one!two!three!', '!', '@', 2) == 'one at two@three!'
+    assert replace('one!two!three!', '!', '@', 3) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 4) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 0) == 'one!two!three!'
+    assert replace('one!two!three!', '!', '@') == 'one at two@three@'
+    assert replace('one!two!three!', 'x', '@') == 'one!two!three!'
+    assert replace('one!two!three!', 'x', '@', 2) == 'one!two!three!'
+    assert replace('abc', '', '-') == '-a-b-c-'
+    assert replace('abc', '', '-', 3) == '-a-b-c'
+    assert replace('abc', '', '-', 0) == 'abc'
+    assert replace('', '', '') == ''
+    assert replace('', '', 'a') == 'a'
+    assert replace('abc', 'ab', '--', 0) == 'abc'
+    assert replace('abc', 'xy', '--') == 'abc'
+    assert replace('123', '123', '') == ''
+    assert replace('123123', '123', '') == ''
+    assert replace('123x123', '123', '') == 'x'
+
+def test_string_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = "a" * (2**16)
+    with py.test.raises(OverflowError):
+        replace(s, "", s)
+    with py.test.raises(OverflowError):
+        replace(s, "a", s)
+    with py.test.raises(OverflowError):
+        replace(s, "a", s, len(s) - 10)
+
+def test_unicode_replace():
+    assert replace(u'one!two!three!', u'!', u'@', 1) == u'one at two!three!'
+    assert replace(u'one!two!three!', u'!', u'') == u'onetwothree'
+    assert replace(u'one!two!three!', u'!', u'@', 2) == u'one at two@three!'
+    assert replace(u'one!two!three!', u'!', u'@', 3) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 4) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 0) == u'one!two!three!'
+    assert replace(u'one!two!three!', u'!', u'@') == u'one at two@three@'
+    assert replace(u'one!two!three!', u'x', u'@') == u'one!two!three!'
+    assert replace(u'one!two!three!', u'x', u'@', 2) == u'one!two!three!'
+    assert replace(u'abc', u'', u'-') == u'-a-b-c-'
+    assert replace(u'abc', u'', u'-', 3) == u'-a-b-c'
+    assert replace(u'abc', u'', u'-', 0) == u'abc'
+    assert replace(u'', u'', u'') == u''
+    assert replace(u'', u'', u'a') == u'a'
+    assert replace(u'abc', u'ab', u'--', 0) == u'abc'
+    assert replace(u'abc', u'xy', u'--') == u'abc'
+    assert replace(u'123', u'123', u'') == u''
+    assert replace(u'123123', u'123', u'') == u''
+    assert replace(u'123x123', u'123', u'') == u'x'
+
+def test_unicode_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = u"a" * (2**16)
+    with py.test.raises(OverflowError):
+        replace(s, u"", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s, len(s) - 10)
+
+def test_startswith():
+    assert startswith('ab', 'ab') is True
+    assert startswith('ab', 'a') is True
+    assert startswith('ab', '') is True
+    assert startswith('x', 'a') is False
+    assert startswith('x', 'x') is True
+    assert startswith('', '') is True
+    assert startswith('', 'a') is False
+    assert startswith('x', 'xx') is False
+    assert startswith('y', 'xx') is False
+    assert startswith('ab', 'a', 0) is True
+    assert startswith('ab', 'a', 1) is False
+    assert startswith('ab', 'b', 1) is True
+    assert startswith('abc', 'bc', 1, 2) is False
+    assert startswith('abc', 'c', -1, 4) is True
+
+def test_endswith():
+    assert endswith('ab', 'ab') is True
+    assert endswith('ab', 'b') is True
+    assert endswith('ab', '') is True
+    assert endswith('x', 'a') is False
+    assert endswith('x', 'x') is True
+    assert endswith('', '') is True
+    assert endswith('', 'a') is False
+    assert endswith('x', 'xx') is False
+    assert endswith('y', 'xx') is False
+    assert endswith('abc', 'ab', 0, 2) is True
+    assert endswith('abc', 'bc', 1) is True
+    assert endswith('abc', 'bc', 2) is False
+    assert endswith('abc', 'b', -3, -1) is True
+
 def test_string_builder():
     s = StringBuilder()
     s.append("a")
@@ -42,4 +161,32 @@
     s.append_multiple_char(u'd', 4)
     assert s.build() == 'aabcbdddd'
     assert isinstance(s.build(), unicode)
-        
+
+
+class TestTranslates(LLRtypeMixin, BaseRtypingTest):
+    def test_split_rsplit(self):
+        def fn():
+            res = True
+            res = res and split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
+            res = res and split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            res = res and split(u'endcase test', u'test') == [u'endcase ', u'']
+            res = res and rsplit('a|b|c|d', '|', 2) == ['a|b', 'c', 'd']
+            res = res and rsplit('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+            res = res and rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+            res = res and rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            return res
+        res = self.interpret(fn, [])
+        assert res
+
+    def test_replace(self):
+        def fn():
+            res = True
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            return res
+        res = self.interpret(fn, [])
+        assert res

From noreply at buildbot.pypy.org  Mon Jun 17 15:44:12 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Mon, 17 Jun 2013 15:44:12 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Make sure nditer.index does not
 give an index when the iterator is consumed
Message-ID: <20130617134412.3986A1C0134@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64918:97a5ef769a6c
Date: 2013-06-17 15:43 +0200
http://bitbucket.org/pypy/pypy/changeset/97a5ef769a6c/

Log:	Make sure nditer.index does not give an index when the iterator is
	consumed

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -264,6 +264,7 @@
         self.reduce_ok = False
         self.zerosize_ok = False
         self.index_iter = None
+        self.done = False
         if space.isinstance_w(w_seq, space.w_tuple) or \
            space.isinstance_w(w_seq, space.w_list):
             w_seq_as_list = space.listview(w_seq)
@@ -308,6 +309,7 @@
             if not it.done():
                 break
         else:
+            self.done = True
             raise OperationError(space.w_StopIteration, space.w_None)
         res = []
         if self.index_iter:
@@ -370,6 +372,8 @@
     def descr_get_index(self, space):
         if self.tracked_index == "":
             raise OperationError(space.w_ValueError, space.wrap("Iterator does not have an index"))
+        if self.done:
+            raise OperationError(space.w_ValueError, space.wrap("Iterator is past the end"))
         return space.wrap(self.index_iter.getvalue())
 
     def descr_get_has_multi_index(self, space):
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -76,6 +76,12 @@
         for value in it:
             r.append((value, it.index))
         assert r == [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)]
+        exc = None
+        try:
+            it.index
+        except ValueError, e:
+            exc = e
+        assert exc
 
         r = []
         it = nditer(a, flags=['f_index'])

From noreply at buildbot.pypy.org  Mon Jun 17 17:02:52 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Mon, 17 Jun 2013 17:02:52 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Implement C style iteration
Message-ID: <20130617150252.2332B1C300E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64919:324066eec689
Date: 2013-06-17 17:01 +0200
http://bitbucket.org/pypy/pypy/changeset/324066eec689/

Log:	Implement C style iteration

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -221,13 +221,9 @@
         self.shape = shape
         self.index = [0] * len(shape)
         self.backward = backward
-        self.called = False
 
     def next(self):
         # TODO It's probably possible to refactor all the "next" method from each iterator
-        if not self.called:
-            self.called = True
-            return
         for i in range(len(self.shape) - 1, -1, -1):
             if self.index[i] < self.shape[i] - 1:
                 self.index[i] += 1
@@ -236,8 +232,6 @@
                 self.index[i] = 0
 
     def getvalue(self):
-        if not self.called:
-            return 0
         if not self.backward:
             ret = self.index[-1]
             for i in range(len(self.shape) - 2, -1, -1):
@@ -265,6 +259,7 @@
         self.zerosize_ok = False
         self.index_iter = None
         self.done = False
+        self.first_next = True
         if space.isinstance_w(w_seq, space.w_tuple) or \
            space.isinstance_w(w_seq, space.w_list):
             w_seq_as_list = space.listview(w_seq)
@@ -293,8 +288,12 @@
         return space.wrap(self)
 
     def descr_getitem(self, space, w_idx):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        idx = space.int_w(w_idx)
+        try:
+            ret = space.wrap(self.iters[idx].getitem(space, self.seq[idx]))
+        except IndexError:
+            raise OperationError(space.w_IndexError, space.wrap("Iterator operand index %d is out of bounds" % idx))
+        return ret
 
     def descr_setitem(self, space, w_idx, w_value):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -313,7 +312,10 @@
             raise OperationError(space.w_StopIteration, space.w_None)
         res = []
         if self.index_iter:
-            self.index_iter.next()
+            if not self.first_next:
+                self.index_iter.next()
+            else:
+                self.first_next = False
         for i in range(len(self.iters)):
             res.append(self.iters[i].getitem(space, self.seq[i]))
             self.iters[i].next()
@@ -321,9 +323,21 @@
             return res[0]
         return space.newtuple(res)
 
+    def iternext(self):
+        if self.index_iter:
+            self.index_iter.next()
+        for i in range(len(self.iters)):
+            self.iters[i].next()
+        for it in self.iters:
+            if not it.done():
+                break
+        else:
+            self.done = True
+            return self.done
+        return self.done
+
     def descr_iternext(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        return space.wrap(self.iternext())
 
     def descr_copy(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -359,8 +373,7 @@
             'not implemented yet'))
 
     def descr_get_finished(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        return space.wrap(self.done)
 
     def descr_get_has_delayed_bufalloc(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(

From noreply at buildbot.pypy.org  Mon Jun 17 17:29:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 17:29:35 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617152935.765A31C31D2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r171:1654bffcfe89
Date: 2013-06-17 14:31 +0200
http://bitbucket.org/pypy/stmgc/changeset/1654bffcfe89/

Log:	Next test

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -56,6 +56,11 @@
 {
     if (nblocks_for_size[1] == 0)
         init_global_data();
+
+    /* Take back ownership of the pages currently assigned to
+       LOCAL_GCPAGES that come from a previous thread. */
+    struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
+    count_global_pages -= gcp->count_pages;
 }
 
 void stmgcpage_done_tls(void)
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -62,3 +62,13 @@
     ofree(p1)
     p2 = oalloc(HDR)
     assert p2 == p1
+
+def test_move_away_as_full_pages():
+    assert count_global_pages() == 0
+    oalloc(HDR)
+    assert count_pages() == 1
+    lib.stm_finalize()
+    assert count_global_pages() == 1
+    lib.stm_initialize_and_set_max_abort(0)    # reuse the same
+    assert count_global_pages() == 0
+    assert count_pages() == 1

From noreply at buildbot.pypy.org  Mon Jun 17 17:29:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 17:29:36 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617152936.B90951C31D2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r172:0d47af9a8ec0
Date: 2013-06-17 14:35 +0200
http://bitbucket.org/pypy/stmgc/changeset/0d47af9a8ec0/

Log:	Next test

diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -337,9 +337,11 @@
                         acquire_lock(_lck1)
                         _lck1.release()
                     #
-                    fn(self)
+                    set_value = fn(self)
                 finally:
                     lib.stm_finalize()
+                if set_value is not None:
+                    self.set_afterwards(set_value)
             except Interrupted:
                 pass
             except:
@@ -378,6 +380,14 @@
             print 'set(%d)' % num
             self._get_lock(num).release()
 
+    def set_afterwards(self, num):
+        # Set the value of 'self.step' to 'num'.
+        with self.settinglock:
+            self._get_lock(self.step).acquire()
+            self.step = num
+            print 'set_afterwards(%d)' % num
+            self._get_lock(num).release()
+
     def set_interrupted(self):
         with self.settinglock:
             self.step = Interrupted
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -72,3 +72,18 @@
     lib.stm_initialize_and_set_max_abort(0)    # reuse the same
     assert count_global_pages() == 0
     assert count_pages() == 1
+
+def test_move_away_as_full_pages_2():
+    def f1(r):
+        assert count_global_pages() == 0
+        oalloc(HDR)
+        assert count_pages() == 1
+        return 2
+    def f2(r):
+        r.wait(2)
+        assert count_global_pages() == 1
+        assert count_pages() == 0
+        oalloc(HDR)
+        assert count_global_pages() == 1
+        assert count_pages() == 1
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Mon Jun 17 17:29:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 17:29:37 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Mention this.
Message-ID: <20130617152937.DB8911C31D2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r173:f3351e588de4
Date: 2013-06-17 16:46 +0200
http://bitbucket.org/pypy/stmgc/changeset/f3351e588de4/

Log:	Mention this.

diff --git a/README.txt b/README.txt
--- a/README.txt
+++ b/README.txt
@@ -10,6 +10,7 @@
 The library interface is in "c4/stmgc.h".
 
 Progress (these revisions are roughly stable versions, pick the latest):
+- 3aea86a96daf: last rev of "c3", the previous version
 - f1ccf5bbcb6f: first step, working but with no GC
 - 8da924453f29: minor collection seems to be working, no major GC
 

From noreply at buildbot.pypy.org  Mon Jun 17 17:29:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 17:29:39 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Copy and adapt more code and a test.
Message-ID: <20130617152939.24A961C31D2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r174:aef30a00a16d
Date: 2013-06-17 16:47 +0200
http://bitbucket.org/pypy/stmgc/changeset/aef30a00a16d/

Log:	Copy and adapt more code and a test.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1355,6 +1355,18 @@
   stmgcpage_count(2);
 }
 
+struct tx_public_descriptor *stm_remove_next_public_descriptor(void)
+{
+  revision_t i = descriptor_array_free_list;
+  struct tx_public_descriptor *pd = stm_descriptor_array[i];
+  if (pd != NULL)
+    {
+      descriptor_array_free_list = pd->free_list_next;
+      assert(descriptor_array_free_list >= 0);
+    }
+  return pd;
+}
+
 int DescriptorInit(void)
 {
   if (GCFLAG_PREBUILT != PREBUILT_FLAGS)
@@ -1401,7 +1413,6 @@
       stm_private_rev_num = -d->my_lock;
       d->private_revision_ref = &stm_private_rev_num;
       d->max_aborts = -1;
-      pd->descriptor = d;
       thread_descriptor = d;
 
       fprintf(stderr, "[%lx] pthread %lx starting\n",
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -112,7 +112,6 @@
  * thread shuts down.  It is reused the next time a thread starts. */
 struct tx_public_descriptor {
   revision_t collection_lock;
-  struct tx_descriptor *descriptor;
   struct stub_block_s *stub_blocks;
   gcptr stub_free_list;
   struct GcPtrList stolen_objects;
@@ -185,5 +184,6 @@
 
 int DescriptorInit(void);
 void DescriptorDone(void);
+struct tx_public_descriptor *stm_remove_next_public_descriptor(void);
 
 #endif  /* _ET_H */
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -1,6 +1,11 @@
 #include "stmimpl.h"
 
 
+/* In this file, we use size_t to measure various sizes that can poten-
+ * tially be large.  Be careful that it's an unsigned type --- but it
+ * is needed to represent more than 2GB on 32-bit machines (up to 4GB).
+ */
+
 /* This maps each small request size to the number of blocks of this size
    that fit in a page. */
 static int nblocks_for_size[GC_SMALL_REQUESTS];
@@ -12,7 +17,10 @@
 static revision_t countdown_next_major_coll = GC_MIN;
 
 /* For statistics */
-static revision_t count_global_pages;
+static size_t count_global_pages;
+
+/* Only computed during a major collection */
+static size_t mc_total_in_use, mc_total_reserved;
 
 /* For tests */
 long stmgcpage_count(int quantity)
@@ -59,8 +67,6 @@
 
     /* Take back ownership of the pages currently assigned to
        LOCAL_GCPAGES that come from a previous thread. */
-    struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
-    count_global_pages -= gcp->count_pages;
 }
 
 void stmgcpage_done_tls(void)
@@ -68,8 +74,6 @@
     /* Send to the shared area all my pages.  For now we don't extract
        the information about which locations are free or not; we just
        leave it to the next major GC to figure them out. */
-    struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
-    count_global_pages += gcp->count_pages;
 }
 
 
@@ -102,6 +106,7 @@
     }
     struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
     gcp->count_pages++;
+    count_global_pages++;
 
     /* Initialize the fields of the resulting page */
     page->next_page = gcp->pages_for_size[size_class];
@@ -176,3 +181,156 @@
         abort();
     }
 }
+
+
+/***** Major collections: sweeping *****/
+
+static void sweep_pages(struct tx_public_descriptor *gcp, int size_class,
+                        page_header_t *lpage)
+{
+    int objs_per_page = nblocks_for_size[size_class];
+    revision_t obj_size = size_class * WORD;
+    gcptr freelist = gcp->free_loc_for_size[size_class];
+    page_header_t *lpagenext;
+    int j;
+    gcptr p;
+
+    for (; lpage; lpage = lpagenext) {
+        lpagenext = lpage->next_page;
+        /* sweep 'page': any object with GCFLAG_VISITED stays alive
+           and the flag is removed; other locations are marked as free. */
+        p = (gcptr)(lpage + 1);
+        for (j = 0; j < objs_per_page; j++) {
+            if (p->h_tid & GCFLAG_VISITED)
+                break;  /* first object that stays alive */
+            p = (gcptr)(((char *)p) + obj_size);
+        }
+        if (j < objs_per_page) {
+            /* the page contains at least one object that stays alive */
+            lpage->next_page = gcp->pages_for_size[size_class];
+            gcp->pages_for_size[size_class] = lpage;
+            p = (gcptr)(lpage + 1);
+            for (j = 0; j < objs_per_page; j++) {
+                if (p->h_tid & GCFLAG_VISITED) {
+                    p->h_tid &= ~GCFLAG_VISITED;
+                    mc_total_in_use += obj_size;
+                }
+                else {
+#ifdef DUMP_EXTRA
+                    if (p->h_tid != DEBUG_WORD(0xDD)) {
+                        fprintf(stderr, "| freeing %p\n", p);
+                    }
+#endif
+                    /* skip the assignment if compiled without asserts */
+                    assert(!(GCFLAG_VISITED & DEBUG_WORD(0xDD)));
+                    assert(p->h_tid = DEBUG_WORD(0xDD));
+                    p->h_revision = (revision_t)freelist;
+                    //stm_dbgmem_not_used(p, size_class * WORD, 0);
+                    freelist = p;
+                }
+                p = (gcptr)(((char *)p) + obj_size);
+            }
+            mc_total_reserved += obj_size * objs_per_page;
+        }
+        else {
+            /* the page is fully free */
+#ifdef DUMP_EXTRA
+            p = (gcptr)(lpage + 1);
+            for (j = 0; j < objs_per_page; j++) {
+                assert(!(p->h_tid & GCFLAG_VISITED));
+                if (p->h_tid != DEBUG_WORD(0xDD)) {
+                    fprintf(stderr, "| freeing %p (with page %p)\n", p, lpage);
+                }
+                p = (gcptr)(((char *)p) + obj_size);
+            }
+#endif
+            stm_free(lpage, GC_PAGE_SIZE);
+            assert(gcp->count_pages > 0);
+            assert(count_global_pages > 0);
+            gcp->count_pages--;
+            count_global_pages--;
+        }
+    }
+    gcp->free_loc_for_size[size_class] = freelist;
+}
+
+
+/***** Major collections: main *****/
+
+static void free_closed_thread_descriptors(void)
+{
+    int i;
+    page_header_t *gpage;
+    struct tx_public_descriptor *gcp;
+
+    while ((gcp = stm_remove_next_public_descriptor()) != NULL) {
+        for (i = 1; i < GC_SMALL_REQUESTS; i++) {
+            gpage = gcp->pages_for_size[i];
+            sweep_pages(gcp, i, gpage);
+        }
+        assert(gcp->collection_lock == 0);
+        /* XXX ...stub_blocks... */
+        assert(gcp->stolen_objects.size == 0);
+        assert(gcp->stolen_young_stubs.size == 0);
+        gcptrlist_delete(&gcp->stolen_objects);
+        gcptrlist_delete(&gcp->stolen_young_stubs);
+        stm_free(gcp, sizeof(struct tx_public_descriptor));
+    }
+}
+
+void stm_major_collect(void)
+{
+    stmgcpage_acquire_global_lock();
+    fprintf(stderr, ",-----\n| running major collection...\n");
+
+#if 0
+    force_minor_collections();
+
+    assert(gcptrlist_size(&objects_to_trace) == 0);
+    mark_prebuilt_roots();
+    mark_all_stack_roots();
+    visit_all_objects();
+    gcptrlist_delete(&objects_to_trace);
+    clean_up_lists_of_read_objects_and_fix_outdated_flags();
+#endif
+
+    mc_total_in_use = mc_total_reserved = 0;
+#if 0
+    free_all_unused_local_pages();
+    free_unused_global_pages();
+#endif
+    free_closed_thread_descriptors();
+#if 0
+    update_next_threshold();
+#endif
+
+    fprintf(stderr, "| %lu bytes alive, %lu not used, countdown %lu\n`-----\n",
+            (unsigned long)mc_total_in_use,
+            (unsigned long)(mc_total_reserved - mc_total_in_use),
+            (unsigned long)countdown_next_major_coll);
+    stmgcpage_release_global_lock();
+}
+
+void stmgcpage_possibly_major_collect(int force)
+{
+    if (force)
+        stmgcpage_reduce_threshold((size_t)-1);
+
+    /* If 'countdown_next_major_coll' reached 0, then run a major coll now. */
+    if (ACCESS_ONCE(countdown_next_major_coll) > 0)
+        return;
+
+    stm_start_single_thread();
+
+    /* If several threads were blocked on the previous line, the first
+       one to proceed sees 0 in 'countdown_next_major_coll'.  It's the
+       thread that will do the major collection.  Afterwards the other
+       threads will also acquire the RW lock in exclusive mode, but won't
+       do anything. */
+    if (countdown_next_major_coll == 0)
+        stm_major_collect();
+
+    stm_stop_single_thread();
+
+    AbortNowIfDelayed();
+}
diff --git a/c4/gcpage.h b/c4/gcpage.h
--- a/c4/gcpage.h
+++ b/c4/gcpage.h
@@ -60,7 +60,7 @@
     gcptr free_loc_for_size[GC_SMALL_REQUESTS];                         \
                                                                         \
     /* For statistics */                                                \
-    revision_t count_pages;
+    size_t count_pages;
 
 
 #define LOCAL_GCPAGES()  (thread_descriptor->public_descriptor)
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -58,7 +58,7 @@
     gcptr stmgcpage_malloc(size_t size);
     void stmgcpage_free(gcptr obj);
     long stmgcpage_count(int quantity);
-    //void stmgcpage_possibly_major_collect(int);
+    void stmgcpage_possibly_major_collect(int);
     revision_t stm_global_cur_time(void);
     //void stmgcpage_add_prebuilt_root(gcptr);
     void stm_clear_between_tests(void);
@@ -115,7 +115,6 @@
     #include "stmgc.h"
     #include "stmimpl.h"
 
-    //extern void stmgcpage_possibly_major_collect(int);
     extern revision_t stm_global_cur_time(void);
     //extern void stmgcpage_add_prebuilt_root(gcptr);
     extern revision_t get_private_rev_num(void);
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -43,6 +43,7 @@
     p = lib.stmgcpage_malloc(HDR)
     assert distance(plist[-1], p) != HDR
     assert count_pages() == 2
+    assert count_global_pages() == 2
 
 def test_thread_local_malloc():
     assert count_global_pages() == 0
@@ -70,7 +71,7 @@
     lib.stm_finalize()
     assert count_global_pages() == 1
     lib.stm_initialize_and_set_max_abort(0)    # reuse the same
-    assert count_global_pages() == 0
+    assert count_global_pages() == 1
     assert count_pages() == 1
 
 def test_move_away_as_full_pages_2():
@@ -84,6 +85,17 @@
         assert count_global_pages() == 1
         assert count_pages() == 0
         oalloc(HDR)
-        assert count_global_pages() == 1
+        assert count_global_pages() == 2
         assert count_pages() == 1
     run_parallel(f1, f2)
+
+def test_free_unused_global_pages():
+    def f1(r):
+        oalloc(HDR)
+        return 2
+    def f2(r):
+        r.wait(2)
+        assert count_global_pages() == 1
+        major_collect()
+        assert count_global_pages() == 0
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Mon Jun 17 17:42:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 17:42:00 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test: link again all
 tx_descriptors in a chained list (duh, and I
Message-ID: <20130617154200.445AD1C0134@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r175:1a57a7dfd909
Date: 2013-06-17 17:41 +0200
http://bitbucket.org/pypy/stmgc/changeset/1a57a7dfd909/

Log:	Next test: link again all tx_descriptors in a chained list (duh, and
	I still cannot get chained list manipulations bug-free the first
	time)

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1049,7 +1049,7 @@
         }
       else
         {
-          stm_free(B, stmcb_size(B));
+          stmgcpage_free(B);
           fprintf(stderr, "commit: free backup at %p\n", B);
         }
     };
@@ -1096,7 +1096,7 @@
                  ((char *)B) + offsetof(struct stm_object_s, h_revision),
                  size - offsetof(struct stm_object_s, h_revision));
           assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
-          stm_free(B, size);
+          stmgcpage_free(B);
           fprintf(stderr, "abort: free backup at %p\n", B);
         }
     };
@@ -1342,6 +1342,7 @@
 
 /************************************************************/
 
+struct tx_descriptor *stm_tx_head = NULL;
 struct tx_public_descriptor *stm_descriptor_array[MAX_THREADS] = {0};
 static revision_t descriptor_array_free_list = 0;
 
@@ -1352,17 +1353,20 @@
   assert(thread_descriptor == NULL);
   memset(stm_descriptor_array, 0, sizeof(stm_descriptor_array));
   descriptor_array_free_list = 0;
-  stmgcpage_count(2);
+  stm_tx_head = NULL;
+  stmgcpage_count(2);  /* reset */
 }
 
-struct tx_public_descriptor *stm_remove_next_public_descriptor(void)
+struct tx_public_descriptor *stm_get_free_public_descriptor(revision_t *pindex)
 {
-  revision_t i = descriptor_array_free_list;
-  struct tx_public_descriptor *pd = stm_descriptor_array[i];
+  if (*pindex < 0)
+    *pindex = descriptor_array_free_list;
+
+  struct tx_public_descriptor *pd = stm_descriptor_array[*pindex];
   if (pd != NULL)
     {
-      descriptor_array_free_list = pd->free_list_next;
-      assert(descriptor_array_free_list >= 0);
+      *pindex = pd->free_list_next;
+      assert(*pindex >= 0);
     }
   return pd;
 }
@@ -1390,6 +1394,8 @@
           /* we are reusing 'pd' */
           descriptor_array_free_list = pd->free_list_next;
           assert(descriptor_array_free_list >= 0);
+          assert(pd->collection_lock == 0 || pd->collection_lock == -1);
+          pd->collection_lock = 0;
       }
       else {
           /* no item in the free list */
@@ -1413,6 +1419,10 @@
       stm_private_rev_num = -d->my_lock;
       d->private_revision_ref = &stm_private_rev_num;
       d->max_aborts = -1;
+      d->tx_prev = NULL;
+      d->tx_next = stm_tx_head;
+      if (d->tx_next != NULL) d->tx_next->tx_prev = d;
+      stm_tx_head = d;
       thread_descriptor = d;
 
       fprintf(stderr, "[%lx] pthread %lx starting\n",
@@ -1442,6 +1452,9 @@
     assert(stm_descriptor_array[i] == d->public_descriptor);
     d->public_descriptor->free_list_next = descriptor_array_free_list;
     descriptor_array_free_list = i;
+    if (d->tx_prev != NULL) d->tx_prev->tx_next = d->tx_next;
+    if (d->tx_next != NULL) d->tx_next->tx_prev = d->tx_prev;
+    if (d == stm_tx_head) stm_tx_head = d->tx_next;
     stmgcpage_release_global_lock();
 
     thread_descriptor = NULL;
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -153,11 +153,13 @@
   long long longest_abort_info_time;
   revision_t *private_revision_ref;
   struct FXCache recent_reads_cache;
+  struct tx_descriptor *tx_prev, *tx_next;
 };
 
 extern __thread struct tx_descriptor *thread_descriptor;
 extern __thread revision_t stm_private_rev_num;
 extern struct tx_public_descriptor *stm_descriptor_array[];
+extern struct tx_descriptor *stm_tx_head;
 
 /************************************************************/
 
@@ -182,8 +184,8 @@
 void stm_clear_read_cache(void);  /* debugging */
 void _stm_test_forget_previous_state(void);  /* debugging */
 
+struct tx_public_descriptor *stm_get_free_public_descriptor(revision_t *);
 int DescriptorInit(void);
 void DescriptorDone(void);
-struct tx_public_descriptor *stm_remove_next_public_descriptor(void);
 
 #endif  /* _ET_H */
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -66,7 +66,7 @@
         init_global_data();
 
     /* Take back ownership of the pages currently assigned to
-       LOCAL_GCPAGES that come from a previous thread. */
+       LOCAL_GCPAGES that might come from a previous thread. */
 }
 
 void stmgcpage_done_tls(void)
@@ -254,30 +254,55 @@
     gcp->free_loc_for_size[size_class] = freelist;
 }
 
+static void free_unused_local_pages(struct tx_public_descriptor *gcp)
+{
+    int i;
+    page_header_t *lpage;
 
-/***** Major collections: main *****/
+    for (i = 1; i < GC_SMALL_REQUESTS; i++) {
+        lpage = gcp->pages_for_size[i];
+        gcp->pages_for_size[i] = NULL;
+        gcp->free_loc_for_size[i] = NULL;
+        sweep_pages(gcp, i, lpage);
+    }
+}
+
+static void free_all_unused_local_pages(void)
+{
+    struct tx_descriptor *d;
+    for (d = stm_tx_head; d; d = d->tx_next) {
+        free_unused_local_pages(d->public_descriptor);
+    }
+}
 
 static void free_closed_thread_descriptors(void)
 {
     int i;
     page_header_t *gpage;
     struct tx_public_descriptor *gcp;
+    revision_t index = -1;
 
-    while ((gcp = stm_remove_next_public_descriptor()) != NULL) {
+    while ((gcp = stm_get_free_public_descriptor(&index)) != NULL) {
+        if (gcp->collection_lock == -1)
+            continue;
+
         for (i = 1; i < GC_SMALL_REQUESTS; i++) {
             gpage = gcp->pages_for_size[i];
             sweep_pages(gcp, i, gpage);
         }
         assert(gcp->collection_lock == 0);
+        gcp->collection_lock = -1;
         /* XXX ...stub_blocks... */
         assert(gcp->stolen_objects.size == 0);
         assert(gcp->stolen_young_stubs.size == 0);
         gcptrlist_delete(&gcp->stolen_objects);
         gcptrlist_delete(&gcp->stolen_young_stubs);
-        stm_free(gcp, sizeof(struct tx_public_descriptor));
     }
 }
 
+
+/***** Major collections: main *****/
+
 void stm_major_collect(void)
 {
     stmgcpage_acquire_global_lock();
@@ -295,8 +320,8 @@
 #endif
 
     mc_total_in_use = mc_total_reserved = 0;
+    free_all_unused_local_pages();
 #if 0
-    free_all_unused_local_pages();
     free_unused_global_pages();
 #endif
     free_closed_thread_descriptors();
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -77,7 +77,7 @@
 gcptr stmgc_duplicate_old(gcptr P)
 {
     size_t size = stmcb_size(P);
-    gcptr L = (gcptr)stm_malloc(size);
+    gcptr L = (gcptr)stmgcpage_malloc(size);
     memcpy(L, P, size);
     L->h_tid |= GCFLAG_OLD;
     return L;
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -19,6 +19,7 @@
     if (p == NULL) {
         assert(sizeof(struct stub_block_s) == STUB_BLOCK_SIZE);
 
+        /* XXX free! */
         char *page = stm_malloc(STUB_PAGE);
         char *page_end = page + STUB_PAGE;
         page += (-(revision_t)page) & (STUB_BLOCK_SIZE-1);  /* round up */
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -99,3 +99,9 @@
         major_collect()
         assert count_global_pages() == 0
     run_parallel(f1, f2)
+
+def test_free_unused_local_pages():
+    p1 = oalloc(HDR)
+    assert count_pages() == 1
+    major_collect()
+    assert count_pages() == 0

From noreply at buildbot.pypy.org  Mon Jun 17 17:59:21 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Mon, 17 Jun 2013 17:59:21 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: hg merge 81aa4bb
Message-ID: <20130617155921.AF76C1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64920:1d3cfb9cf858
Date: 2013-06-17 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/1d3cfb9cf858/

Log:	hg merge 81aa4bb

diff too long, truncating to 2000 out of 2996 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,6 +134,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -168,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,6 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
-jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Distance to target address
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -278,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,48 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:
 
 * Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -50,3 +50,10 @@
 .. branch: win32-fixes3
 Skip and fix some non-translated (own) tests for win32 builds
 
+.. branch: ctypes-byref
+Add the '_obj' attribute on ctypes pointer() and byref() objects
+
+.. branch: argsort-segfault
+Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
+
+.. branch: dtype-isnative
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -289,16 +289,6 @@
                "with verify() (see pypy/module/_cffi_backend/ctypefunc.py "
                "for details)"))
 
-        if USE_C_LIBFFI_MSVC and is_result_type:
-            # MSVC returns small structures in registers.  Pretend int32 or
-            # int64 return type.  This is needed as a workaround for what
-            # is really a bug of libffi_msvc seen as an independent library
-            # (ctypes has a similar workaround).
-            if ctype.size <= 4:
-                return clibffi.ffi_type_sint32
-            if ctype.size <= 8:
-                return clibffi.ffi_type_sint64
-
         # walk the fields, expanding arrays into repetitions; first,
         # only count how many flattened fields there are
         nflat = 0
@@ -318,6 +308,16 @@
                                "a struct with a zero-length array"))
             nflat += flat
 
+        if USE_C_LIBFFI_MSVC and is_result_type:
+            # MSVC returns small structures in registers.  Pretend int32 or
+            # int64 return type.  This is needed as a workaround for what
+            # is really a bug of libffi_msvc seen as an independent library
+            # (ctypes has a similar workaround).
+            if ctype.size <= 4:
+                return clibffi.ffi_type_sint32
+            if ctype.size <= 8:
+                return clibffi.ffi_type_sint64
+
         # allocate an array of (nflat + 1) ffi_types
         elements = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * (nflat + 1))
         elements = rffi.cast(FFI_TYPE_PP, elements)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 
 from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr,
     ctypearray, ctypestruct, ctypevoid, ctypeenum)
@@ -115,13 +116,15 @@
 # ____________________________________________________________
 
 SF_MSVC_BITFIELDS = 1
+SF_GCC_ARM_BITFIELDS = 2
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+elif rffi_platform.getdefined('__arm__', ''):
+    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
     DEFAULT_SFLAGS = 0
 
-
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -180,7 +183,7 @@
         # field is an anonymous bitfield
         falign = ftype.alignof()
         do_align = True
-        if fbitsize >= 0:
+        if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
                 # GCC: anonymous bitfields (of any size) don't cause alignment
                 do_align = (fname != '')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2767,10 +2767,10 @@
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
@@ -2783,7 +2783,10 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
@@ -2795,10 +2798,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2807,6 +2815,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -917,7 +917,8 @@
 
     try:
         optimize = space.sys.get_flag('optimize')
-    except Exception:
+    except RuntimeError:
+        # during bootstrapping
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)
@@ -1018,7 +1019,8 @@
     code_w = read_compiled_module(space, cpathname, source)
     try:
         optimize = space.sys.get_flag('optimize')
-    except Exception:
+    except RuntimeError:
+        # during bootstrapping
         optimize = 0
     if optimize >= 2:
         code_w.remove_docstrings(space)
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -200,3 +200,49 @@
                 except KeyError:
                     pass
             rmtree(dir_name, True)
+
+    def test_builtin_reimport(self):
+        # from https://bugs.pypy.org/issue1514
+        skip("fix me")
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        __import__('marshal')
+        sys.modules['marshal'] = saved
+
+        assert marshal.loads == 42
+        import marshal
+        assert marshal.loads == 42
+        marshal.loads = old
+
+    def test_builtin_reimport_mess(self):
+        # taken from https://bugs.pypy.org/issue1514, with extra cases
+        # that show a difference with CPython: we can get on CPython
+        # several module objects for the same built-in module :-(
+        skip("several built-in module objects: not supported by pypy")
+        import sys, marshal
+
+        old = marshal.loads
+        marshal.loads = 42
+
+        # save, re-import, restore.
+        saved = sys.modules.pop('marshal')
+        marshal2 = __import__('marshal')
+        assert marshal2 is not marshal
+        assert marshal2.loads is old
+        assert marshal2 is sys.modules['marshal']
+        assert marshal is saved
+        assert marshal.loads == 42
+
+        import marshal
+        assert marshal.loads is old
+
+        sys.modules['marshal'] = saved
+        import marshal
+        assert marshal.loads == 42
+
+        marshal.loads = old
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -565,6 +565,43 @@
         assert sys.path is oldpath
         assert 'setdefaultencoding' in dir(sys)
 
+    def test_reload_builtin_doesnt_clear(self):
+        import sys
+        sys.foobar = "baz"
+        reload(sys)
+        assert sys.foobar == "baz"
+
+    def test_reimport_builtin_simple_case_1(self):
+        import sys, time
+        del time.tzset
+        del sys.modules['time']
+        import time
+        assert hasattr(time, 'tzset')
+
+    def test_reimport_builtin_simple_case_2(self):
+        skip("fix me")
+        import sys, time
+        time.foo = "bar"
+        del sys.modules['time']
+        import time
+        assert not hasattr(time, 'foo')
+
+    def test_reimport_builtin(self):
+        skip("fix me")
+        import sys, time
+        oldpath = sys.path
+        time.tzset = "<test_reimport_builtin removed this>"
+
+        del sys.modules['time']
+        import time as time1
+        assert sys.modules['time'] is time1
+
+        assert time.tzset == "<test_reimport_builtin removed this>"
+
+        reload(time1)   # don't leave a broken time.tzset behind
+        import time
+        assert time.tzset != "<test_reimport_builtin removed this>"
+
     def test_reload_infinite(self):
         import infinite_reload
 
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -20,7 +20,7 @@
 def make_sort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
-    
+
     class Repr(object):
         def __init__(self, index_stride_size, stride_size, size, values,
                      indexes, index_start, start):
@@ -71,11 +71,11 @@
         def __init__(self, index_stride_size, stride_size, size):
             start = 0
             dtype = interp_dtype.get_dtype_cache(space).w_longdtype
-            self.indexes = dtype.itemtype.malloc(size*dtype.get_size())
-            self.values = alloc_raw_storage(size * stride_size, 
+            indexes = dtype.itemtype.malloc(size*dtype.get_size())
+            values = alloc_raw_storage(size * stride_size,
                                             track_allocation=False)
-            Repr.__init__(self, index_stride_size, stride_size, 
-                          size, self.values, self.indexes, start, start)
+            Repr.__init__(self, dtype.get_size(), stride_size,
+                          size, values, indexes, start, start)
 
         def __del__(self):
             free_raw_storage(self.indexes, track_allocation=False)
@@ -96,7 +96,7 @@
         for i in range(stop-start):
             retval.setitem(i, lst.getitem(i+start))
         return retval
-    
+
     if count < 2:
         def arg_lt(a, b):
             # Does numpy do <= ?
@@ -108,7 +108,7 @@
                     return True
                 elif a[0][i] > b[0][i]:
                     return False
-            # Does numpy do True?    
+            # Does numpy do True?
             return False
 
     ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
@@ -180,7 +180,7 @@
 
 class SortCache(object):
     built = False
-    
+
     def __init__(self, space):
         if self.built:
             return
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -13,6 +13,9 @@
 from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage
 from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.rstring import StringBuilder
+
 
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
@@ -42,7 +45,23 @@
     return func_with_new_name(new, name + "_box_new"), staticmethod(_get_dtype), func_with_new_name(descr_reduce, "descr_reduce")
 
 
-class PrimitiveBox(object):
+class Box(object):
+    _mixin_ = True
+
+    def reduce(self, space):
+        from rpython.rlib.rstring import StringBuilder
+        from rpython.rtyper.lltypesystem import rffi, lltype
+
+        numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(numpypy, MixedModule)
+        multiarray = numpypy.get("multiarray")
+        assert isinstance(multiarray, MixedModule)
+        scalar = multiarray.get("scalar")
+
+        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
+        return ret
+
+class PrimitiveBox(Box):
     _mixin_ = True
 
     def __init__(self, value):
@@ -54,27 +73,19 @@
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self.value)
 
-    def reduce(self, space):
-        from rpython.rlib.rstring import StringBuilder
-        from rpython.rtyper.lltypesystem import rffi, lltype
-
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
-        assert isinstance(multiarray, MixedModule)
-        scalar = multiarray.get("scalar")
-
+    def raw_str(self):
         value = lltype.malloc(rffi.CArray(lltype.typeOf(self.value)), 1, flavor="raw")
         value[0] = self.value
 
         builder = StringBuilder()
         builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.value)))
+        ret = builder.build()
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(builder.build())])])
         lltype.free(value, flavor="raw")
+
         return ret
 
-class ComplexBox(object):
+class ComplexBox(Box):
     _mixin_ = True
 
     def __init__(self, real, imag=0.):
@@ -90,25 +101,17 @@
     def convert_imag_to(self, dtype):
         return dtype.box(self.imag)
 
-    def reduce(self, space):
-        from rpython.rlib.rstring import StringBuilder
-        from rpython.rtyper.lltypesystem import rffi, lltype
-
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
-        assert isinstance(multiarray, MixedModule)
-        scalar = multiarray.get("scalar")
-
+    def raw_str(self):
         value = lltype.malloc(rffi.CArray(lltype.typeOf(self.real)), 2, flavor="raw")
         value[0] = self.real
         value[1] = self.imag
 
         builder = StringBuilder()
         builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.real)) * 2)
+        ret = builder.build()
 
-        ret = space.newtuple([scalar, space.newtuple([space.wrap(self._get_dtype(space)), space.wrap(builder.build())])])
         lltype.free(value, flavor="raw")
+
         return ret
 
 class W_GenericBox(W_Root):
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -115,9 +115,27 @@
             return space.wrap('=')
         return space.wrap(nonnative_byteorder_prefix)
 
+    def descr_get_str(self, space):
+        size = self.get_size()
+        basic = self.kind
+        if basic == UNICODELTR:
+            size >>= 2
+            endian = byteorder_prefix
+        elif size <= 1:
+            endian = '|'  # ignore
+        elif self.native:
+            endian = byteorder_prefix
+        else:
+            endian = nonnative_byteorder_prefix
+
+        return space.wrap("%s%s%s" % (endian, basic, size))
+
     def descr_get_alignment(self, space):
         return space.wrap(self.itemtype.alignment)
 
+    def descr_get_isnative(self, space):
+        return space.wrap(self.native)
+
     def descr_get_base(self, space):
         return space.wrap(self.base)
 
@@ -421,8 +439,10 @@
     char = interp_attrproperty("char", cls=W_Dtype),
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
     byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
+    str = GetSetProperty(W_Dtype.descr_get_str),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
     alignment = GetSetProperty(W_Dtype.descr_get_alignment),
+    isnative = GetSetProperty(W_Dtype.descr_get_isnative),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
     name = interp_attrproperty('name', cls=W_Dtype),
     fields = GetSetProperty(W_Dtype.descr_get_fields),
@@ -582,7 +602,7 @@
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
                                    ],
-            aliases=["float"],
+            aliases=["float", "double"],
         )
         self.w_complex64dtype = W_ComplexDtype(
             types.Complex64(),
@@ -663,9 +683,10 @@
             char='S',
             w_box_type = space.gettypefor(interp_boxes.W_StringBox),
             alternate_constructors=[space.w_str],
+            aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
-            types.UnicodeType(1),
+            types.UnicodeType(0),
             num=19,
             kind=UNICODELTR,
             name='unicode',
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -418,8 +418,16 @@
         addr = self.implementation.get_storage_as_int(space)
         # will explode if it can't
         w_d = space.newdict()
-        space.setitem_str(w_d, 'data', space.newtuple([space.wrap(addr),
-                                                       space.w_False]))
+        space.setitem_str(w_d, 'data',
+                          space.newtuple([space.wrap(addr), space.w_False]))
+        space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
+        space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
+        if self.implementation.order == 'C':
+            # Array is contiguous, no strides in the interface.
+            strides = space.w_None
+        else:
+            strides = self.descr_get_strides(space)
+        space.setitem_str(w_d, 'strides', strides)
         return w_d
 
     w_pypy_data = None
@@ -786,6 +794,7 @@
         from rpython.rtyper.lltypesystem import rffi
         from rpython.rlib.rstring import StringBuilder
         from pypy.interpreter.mixedmodule import MixedModule
+        from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
 
         numpypy = space.getbuiltinmodule("_numpypy")
         assert isinstance(numpypy, MixedModule)
@@ -796,7 +805,14 @@
         parameters = space.newtuple([space.gettypefor(W_NDimArray), space.newtuple([space.wrap(0)]), space.wrap("b")])
 
         builder = StringBuilder()
-        builder.append_charpsize(self.implementation.get_storage(), self.implementation.get_storage_size())
+        if isinstance(self.implementation, SliceArray):
+            iter = self.implementation.create_iter()
+            while not iter.done():
+                box = iter.getitem()
+                builder.append(box.raw_str())
+                iter.next()
+        else:
+            builder.append_charpsize(self.implementation.get_storage(), self.implementation.get_storage_size())
 
         state = space.newtuple([
                 space.wrap(1),      # version
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -683,6 +683,20 @@
         assert dtype('=i8').byteorder == '='
         assert dtype(byteorder + 'i8').byteorder == '='
 
+    def test_dtype_str(self):
+        from numpypy import dtype
+        byteorder = self.native_prefix
+        assert dtype('i8').str == byteorder + 'i8'
+        assert dtype('<i8').str == '<i8'
+        assert dtype('>i8').str == '>i8'
+        assert dtype('int8').str == '|i1'
+        assert dtype('float').str == byteorder + 'f8'
+        # strange
+        assert dtype('string').str == '|S0'
+        assert dtype('unicode').str == byteorder + 'U0'
+        # assert dtype(('string', 7)).str == '|S7'
+        # assert dtype(('unicode', 7)).str == '<U7'
+
     def test_intp(self):
         from numpypy import dtype
         assert dtype('p') == dtype('intp')
@@ -692,6 +706,11 @@
         from numpypy import dtype
         assert dtype('i4').alignment == 4
 
+    def test_isnative(self):
+        from numpypy import dtype
+        assert dtype('i4').isnative == True
+        assert dtype('>i8').isnative == False
+
     def test_any_all(self):
         import numpypy as numpy
         x = numpy.bool_(True)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1805,7 +1805,7 @@
         from cPickle import loads, dumps
         import numpypy as numpy
 
-        a = numpy.arange(10.)[::2]
+        a = numpy.arange(10.).reshape((5, 2))[::2]
         assert (loads(dumps(a)) == a).all()
 
 class AppTestMultiDim(BaseNumpyAppTest):
@@ -2138,6 +2138,9 @@
         a = array([1, 2, 3])
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
+        assert i['shape'] == (3,)
+        assert i['strides'] == None  # Because array is in C order
+        assert i['typestr'] == a.dtype.str
         a = a[::2]
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
@@ -2496,6 +2499,13 @@
         b = a.argsort()
         assert (b[:3] == [0, 100, 200]).all()
 
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
     def test_argsort_axis(self):
         from numpypy import array
         a = array([[4, 2], [1, 3]])
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -118,7 +118,7 @@
     def look_inside_function(self, func):
         mod = func.__module__ or '?'
 
-        if mod == 'rpython.rlib.rlocale' or mod == 'rpython.rlib.rsocket':
+        if mod == 'rpython.rlib.rbigint' or mod == 'rpython.rlib.rlocale' or mod == 'rpython.rlib.rsocket':
             return False
         if mod.startswith('pypy.interpreter.astcompiler.'):
             return False
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -8,7 +8,12 @@
 
 def test_bigint():
     from rpython.rlib.rbigint import rbigint
-    assert pypypolicy.look_inside_function(rbigint.lt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.eq.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.ne.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.lt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.le.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.gt.im_func)
+    assert not pypypolicy.look_inside_function(rbigint.ge.im_func)
 
 def test_rlocale():
     from rpython.rlib.rlocale import setlocale
diff --git a/pypy/module/pypyjit/test_pypy_c/test_intbound.py b/pypy/module/pypyjit/test_pypy_c/test_intbound.py
--- a/pypy/module/pypyjit/test_pypy_c/test_intbound.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_intbound.py
@@ -231,6 +231,7 @@
         assert loop.match("""
             i8 = int_lt(i6, 300)
             guard_true(i8, descr=...)
+            guard_not_invalidated?
             i10 = int_lshift(i6, 1)
             i12 = int_add_ovf(i5, 1)
             guard_no_overflow(descr=...)
@@ -253,6 +254,7 @@
         assert loop.match("""
             i8 = int_lt(i6, 300)
             guard_true(i8, descr=...)
+            guard_not_invalidated?
             i10 = int_add_ovf(i5, 8)
             guard_no_overflow(descr=...)
             i12 = int_add(i6, 1)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -407,5 +407,4 @@
 
         log = self.run(main, [300])
         loop, = log.loops_by_id("long_op")
-        assert log.match("""
-        """)
+        assert len(loop.ops_by_id("long_op")) == 0
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -103,11 +103,6 @@
         'flags'                 : 'app.null_sysflags',
     }
 
-    def setbuiltinmodule(self, w_module, name):
-        w_name = self.space.wrap(name)
-        w_modules = self.get('modules')
-        self.space.setitem(w_modules, w_name, w_module)
-
     def startup(self, space):
         if space.config.translating and not we_are_translated():
             # don't get the filesystemencoding at translation time
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -29,14 +29,13 @@
     try:
         # first try to print the exception's class name
         stderr = sys.stderr
-        stderr.write(getattr(exctype, '__name__', exctype))
+        stderr.write(str(getattr(exctype, '__name__', exctype)))
         # then attempt to get the str() of the exception
         try:
             s = str(value)
         except:
             s = '<failure of str() on the exception instance>'
-        # then print it, and don't worry too much about the extra space
-        # between the exception class and the ':'
+        # then print it
         if s:
             stderr.write(': %s\n' % (s,))
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -1,5 +1,6 @@
 # Generated by pypy/tool/import_cffi.py
-import py, sys
+import py, sys, platform
+import pytest
 from pypy.module.test_lib_pypy.cffi_tests import backend_tests, test_function, test_ownlib
 from cffi import FFI
 import _cffi_backend
@@ -41,40 +42,43 @@
 
 class TestBitfield:
     def check(self, source, expected_ofs_y, expected_align, expected_size):
+        # NOTE: 'expected_*' is the numbers expected from GCC.
+        # The numbers expected from MSVC are not explicitly written
+        # in this file, and will just be taken from the compiler.
         ffi = FFI()
         ffi.cdef("struct s1 { %s };" % source)
         ctype = ffi.typeof("struct s1")
         # verify the information with gcc
-        if sys.platform != "win32":
-            ffi1 = FFI()
-            ffi1.cdef("""
-                static const int Gofs_y, Galign, Gsize;
-                struct s1 *try_with_value(int fieldnum, long long value);
-            """)
-            fnames = [name for name, cfield in ctype.fields
-                           if name and cfield.bitsize > 0]
-            setters = ['case %d: s.%s = value; break;' % iname
-                       for iname in enumerate(fnames)]
-            lib = ffi1.verify("""
-                struct s1 { %s };
-                struct sa { char a; struct s1 b; };
-                #define Gofs_y  offsetof(struct s1, y)
-                #define Galign  offsetof(struct sa, b)
-                #define Gsize   sizeof(struct s1)
-                struct s1 *try_with_value(int fieldnum, long long value)
-                {
-                    static struct s1 s;
-                    memset(&s, 0, sizeof(s));
-                    switch (fieldnum) { %s }
-                    return &s;
-                }
-            """ % (source, ' '.join(setters)))
-            assert lib.Gofs_y == expected_ofs_y
-            assert lib.Galign == expected_align
-            assert lib.Gsize  == expected_size
+        ffi1 = FFI()
+        ffi1.cdef("""
+            static const int Gofs_y, Galign, Gsize;
+            struct s1 *try_with_value(int fieldnum, long long value);
+        """)
+        fnames = [name for name, cfield in ctype.fields
+                       if name and cfield.bitsize > 0]
+        setters = ['case %d: s.%s = value; break;' % iname
+                   for iname in enumerate(fnames)]
+        lib = ffi1.verify("""
+            struct s1 { %s };
+            struct sa { char a; struct s1 b; };
+            #define Gofs_y  offsetof(struct s1, y)
+            #define Galign  offsetof(struct sa, b)
+            #define Gsize   sizeof(struct s1)
+            struct s1 *try_with_value(int fieldnum, long long value)
+            {
+                static struct s1 s;
+                memset(&s, 0, sizeof(s));
+                switch (fieldnum) { %s }
+                return &s;
+            }
+        """ % (source, ' '.join(setters)))
+        if sys.platform == 'win32':
+            expected_ofs_y = lib.Gofs_y
+            expected_align = lib.Galign
+            expected_size  = lib.Gsize
         else:
-            lib = None
-            fnames = None
+            assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (
+                expected_ofs_y, expected_align, expected_size)
         # the real test follows
         assert ffi.offsetof("struct s1", "y") == expected_ofs_y
         assert ffi.alignof("struct s1") == expected_align
@@ -99,10 +103,9 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
-        if lib is not None:
-            t = lib.try_with_value(fnames.index(name), value)
-            raw2 = ffi.buffer(t, len(raw1))[:]
-            assert raw1 == raw2
+        t = lib.try_with_value(fnames.index(name), value)
+        raw2 = ffi.buffer(t, len(raw1))[:]
+        assert raw1 == raw2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
@@ -120,6 +123,7 @@
         self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8)
         self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4)
 
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_anonymous_no_align(self):
         L = FFI().alignof("long long")
         self.check("char y; int :1;", 0, 1, 2)
@@ -132,13 +136,40 @@
         self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
         self.check("char x; long long  :57; char y;", L + 8, 1, L + 9)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_anonymous_align_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :1;", 0, 4, 4)
+        self.check("char x; int z:1; char y;", 2, 4, 4)
+        self.check("char x; int  :1; char y;", 2, 4, 4)
+        self.check("char x; long long z:48; char y;", 7, L, 8)
+        self.check("char x; long long  :48; char y;", 7, 8, 8)
+        self.check("char x; long long z:56; char y;", 8, L, 8 + L)
+        self.check("char x; long long  :56; char y;", 8, L, 8 + L)
+        self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
+        self.check("char x; long long  :57; char y;", L + 8, L, L + 8 + L)
+
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_zero(self):
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
         self.check("char x; int :0; char y;", 4, 1, 5)
+        self.check("char x; int :0; int :0; char y;", 4, 1, 5)
         self.check("char x; long long :0; char y;", L, 1, L + 1)
         self.check("short x, y; int :0; int :0;", 2, 2, 4)
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
+
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_zero_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :0;", 0, 4, 4)
+        self.check("char x; int :0; char y;", 4, 4, 8)
+        self.check("char x; int :0; int :0; char y;", 4, 4, 8)
+        self.check("char x; long long :0; char y;", L, 8, L + 8)
+        self.check("short x, y; int :0; int :0;", 2, 4, 4)
+        self.check("char x; int :0; short b:1; char y;", 5, 4, 8)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
     def test_error_cases(self):
         ffi = FFI()
diff --git a/pypy/module/test_lib_pypy/test_os_wait.py b/pypy/module/test_lib_pypy/test_os_wait.py
--- a/pypy/module/test_lib_pypy/test_os_wait.py
+++ b/pypy/module/test_lib_pypy/test_os_wait.py
@@ -1,7 +1,7 @@
-# Generates the resource cache
-#from __future__ import absolute_import
-#from lib_pypy.ctypes_config_cache import rebuild
-#rebuild.rebuild_one('resource.ctc.py')
+# Generates the resource cache (it might be there already, but maybe not)
+from __future__ import absolute_import
+from lib_pypy.ctypes_config_cache import rebuild
+rebuild.rebuild_one('resource.ctc.py')
 
 import os
 
diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py
--- a/pypy/module/thread/test/test_fork.py
+++ b/pypy/module/thread/test/test_fork.py
@@ -28,7 +28,7 @@
                 if pid == 0:
                     os._exit(0)
                 else:
-                    self.timeout_killer(pid, 5)
+                    self.timeout_killer(pid, 10)
                     exitcode = os.waitpid(pid, 0)[1]
                     assert exitcode == 0 # if 9, process was killed by timer!
             finally:
@@ -54,7 +54,7 @@
                 thread.start_new_thread(lambda: None, ())
                 os._exit(0)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 assert exitcode == 0 # if 9, process was killed by timer!
 
@@ -73,7 +73,7 @@
                 signal.signal(signal.SIGUSR1, signal.SIG_IGN)
                 os._exit(42)
             else:
-                self.timeout_killer(pid, 5)
+                self.timeout_killer(pid, 10)
                 exitcode = os.waitpid(pid, 0)[1]
                 feedback.append(exitcode)
 
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -693,6 +693,7 @@
             raise OperationError(space.w_ValueError,
                                  space.wrap("list modified during sort"))
 
+find_jmp = jit.JitDriver(greens = [], reds = 'auto', name = 'list.find')
 
 class ListStrategy(object):
     sizehint = -1
@@ -717,6 +718,7 @@
         i = start
         # needs to be safe against eq_w mutating stuff
         while i < stop and i < w_list.length():
+            find_jmp.jit_merge_point()
             if space.eq_w(w_list.getitem(i), w_item):
                 return i
             i += 1
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -4,7 +4,28 @@
 class AppTestMethodCaching(test_typeobject.AppTestTypeObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
 
+    def setup_class(cls):
+        # This is for the following tests, which are a bit fragile and
+        # historically have been failing once in a while.  With this hack,
+        # they are run up to 5 times in a row, saving the frame of the
+        # failed attempt.  This means occasional collisions should work
+        # differently during the retry.
+        cls.w_retry = cls.space.appexec([], """():
+            def retry(run):
+                keepalive = []
+                for i in range(4):
+                    try:
+                        return run()
+                    except AssertionError:
+                        import sys
+                        keepalive.append(sys.exc_info())
+                return run()
+            return retry
+        """)
+
     def test_mix_classes(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -25,6 +46,8 @@
         assert sum(cache_counter) == 30
 
     def test_class_that_cannot_be_cached(self):
+      @self.retry
+      def run():
         import __pypy__
         class X:
             pass
@@ -50,6 +73,8 @@
         assert sum(cache_counter) == 20
  
     def test_change_methods(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -88,6 +113,8 @@
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             def f(self):
@@ -107,6 +134,8 @@
         assert sum(cache_counter) == 30
   
     def test_many_names(self):
+      @self.retry
+      def run():
         import __pypy__
         for j in range(20):
             class A(object):
@@ -160,6 +189,8 @@
         assert e.foo == 3
 
     def test_custom_metaclass(self):
+      @self.retry
+      def run():
         import __pypy__
         for j in range(20):
             class MetaA(type):
@@ -181,6 +212,8 @@
             raise AssertionError("cache_counter = %r" % (cache_counter,))
 
     def test_mutate_class(self):
+      @self.retry
+      def run():
         import __pypy__
         class A(object):
             x = 1
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -113,7 +113,7 @@
         tmpreg = r.lr
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None: # store
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.load_reg(mc, excvalloc, r.ip)
         if on_frame:
             # store exc_value in JITFRAME
@@ -125,7 +125,7 @@
             self.store_reg(mc, r.ip, r.fp, ofs, helper=tmpreg)
         if exctploc is not None:
             # store pos_exception in exctploc
-            assert exctploc.is_reg()
+            assert exctploc.is_core_reg()
             mc.gen_load_int(r.ip.value, self.cpu.pos_exception())
             self.load_reg(mc, exctploc, r.ip, helper=tmpreg)
 
@@ -146,7 +146,7 @@
         tmpreg = r.lr # use lr as a second temporary reg
         mc.gen_load_int(r.ip.value, self.cpu.pos_exc_value())
         if excvalloc is not None:
-            assert excvalloc.is_reg()
+            assert excvalloc.is_core_reg()
             self.store_reg(mc, excvalloc, r.ip)
         else:
             assert exctploc is not r.fp
@@ -947,7 +947,7 @@
     # regalloc support
     def load(self, loc, value):
         """load an immediate value into a register"""
-        assert (loc.is_reg() and value.is_imm()
+        assert (loc.is_core_reg() and value.is_imm()
                     or loc.is_vfp_reg() and value.is_imm_float())
         if value.is_imm():
             self.mc.gen_load_int(loc.value, value.getint())
@@ -958,7 +958,7 @@
     def load_reg(self, mc, target, base, ofs=0, cond=c.AL, helper=r.ip):
         if target.is_vfp_reg():
             return self._load_vfp_reg(mc, target, base, ofs, cond, helper)
-        elif target.is_reg():
+        elif target.is_core_reg():
             return self._load_core_reg(mc, target, base, ofs, cond, helper)
 
     def _load_vfp_reg(self, mc, target, base, ofs, cond=c.AL, helper=r.ip):
@@ -1012,7 +1012,7 @@
     def _mov_imm_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.type == FLOAT:
             raise AssertionError("invalid target for move from imm value")
-        if loc.is_reg():
+        if loc.is_core_reg():
             new_loc = loc
         elif loc.is_stack() or loc.is_raw_sp():
             new_loc = r.lr
@@ -1027,7 +1027,7 @@
     def _mov_reg_to_loc(self, prev_loc, loc, cond=c.AL):
         if loc.is_imm():
             raise AssertionError("mov reg to imm doesn't make sense")
-        if loc.is_reg():
+        if loc.is_core_reg():
             self.mc.MOV_rr(loc.value, prev_loc.value, cond=cond)
         elif loc.is_stack() and loc.type != FLOAT:
             # spill a core register
@@ -1050,7 +1050,7 @@
         helper = None
         offset = prev_loc.value
         tmp = None
-        if loc.is_reg():
+        if loc.is_core_reg():
             assert prev_loc.type != FLOAT, 'trying to load from an \
                 incompatible location into a core register'
             # unspill a core register
@@ -1126,7 +1126,7 @@

From noreply at buildbot.pypy.org  Mon Jun 17 17:59:23 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Mon, 17 Jun 2013 17:59:23 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: hg merge default (Solve
 merge conflicts by discarding remote changes.)
Message-ID: <20130617155923.2DFD61C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64921:9d28dbdb46be
Date: 2013-06-17 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/9d28dbdb46be/

Log:	hg merge default (Solve merge conflicts by discarding remote
	changes.)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -909,6 +909,9 @@
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
+    def newlist_unicode(self, list_u):
+        return self.newlist([self.wrap(u) for u in list_u])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -22,7 +22,8 @@
 from rpython.rlib.objectmodel import (compute_hash, compute_unique_id,
         specialize)
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import StringBuilder, split
+from rpython.rlib.rstring import (StringBuilder, split, rsplit, replace,
+    endswith, startswith)
 
 
 class W_AbstractBytesObject(W_Object):
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -166,6 +166,12 @@
         storage = strategy.erase(list_s)
         return W_ListObject.from_storage_and_strategy(space, storage, strategy)
 
+    @staticmethod
+    def newlist_unicode(space, list_u):
+        strategy = space.fromcache(UnicodeListStrategy)
+        storage = strategy.erase(list_u)
+        return W_ListObject.from_storage_and_strategy(space, storage, strategy)
+
     def __repr__(self):
         """ representation for debugging purposes """
         return "%s(%s, %s)" % (self.__class__.__name__, self.strategy,
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -293,6 +293,9 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
+    def newlist_unicode(self, list_u):
+        return W_ListObject.newlist_unicode(self, list_u)
+
     def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -555,10 +555,30 @@
         try:
             w_l = space.call_method(w_s, "split")
             w_l2 = space.call_method(w_s, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_s, "rsplit")
+            w_l4 = space.call_method(w_s, "rsplit", space.wrap(" "))
         finally:
             del space.newlist
         assert space.listview_str(w_l) == ["a", "b", "c"]
         assert space.listview_str(w_l2) == ["a", "b", "c"]
+        assert space.listview_str(w_l3) == ["a", "b", "c"]
+        assert space.listview_str(w_l4) == ["a", "b", "c"]
+
+    def test_unicode_uses_newlist_unicode(self):
+        space = self.space
+        w_u = space.wrap(u"a b c")
+        space.newlist = None
+        try:
+            w_l = space.call_method(w_u, "split")
+            w_l2 = space.call_method(w_u, "split", space.wrap(" "))
+            w_l3 = space.call_method(w_u, "rsplit")
+            w_l4 = space.call_method(w_u, "rsplit", space.wrap(" "))
+        finally:
+            del space.newlist
+        assert space.listview_unicode(w_l) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l2) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l3) == [u"a", u"b", u"c"]
+        assert space.listview_unicode(w_l4) == [u"a", u"b", u"c"]
 
     def test_pop_without_argument_is_fast(self):
         space = self.space
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -745,21 +745,21 @@
             if (self[pos] == u'\r' and pos + 1 < end and
                 self[pos + 1] == u'\n'):
                 # Count CRLF as one linebreak
-                lines.append(W_UnicodeObject(self[start:pos + keepends * 2]))
+                lines.append(self[start:pos + keepends * 2])
                 pos += 1
             else:
-                lines.append(W_UnicodeObject(self[start:pos + keepends]))
+                lines.append(self[start:pos + keepends])
             pos += 1
             start = pos
         else:
             pos += 1
     if not unicodedb.islinebreak(ord(self[end - 1])):
-        lines.append(W_UnicodeObject(self[start:]))
-    return space.newlist(lines)
+        lines.append(self[start:])
+    return space.newlist_unicode(lines)
 
 def unicode_split__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     length = len(value)
     i = 0
@@ -782,12 +782,12 @@
             maxsplit -= 1   # NB. if it's already < 0, it stays < 0
 
         # the word is value[i:j]
-        res_w.append(W_UnicodeObject(value[i:j]))
+        res.append(value[i:j])
 
         # continue to look from the character following the space after the word
         i = j + 1
 
-    return space.newlist(res_w)
+    return space.newlist_unicode(res)
 
 def unicode_split__Unicode_Unicode_ANY(space, w_self, w_delim, w_maxsplit):
     self = w_self._value
@@ -797,13 +797,13 @@
     if delim_len == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap('empty separator'))
-    parts = _split_with(self, delim, maxsplit)
-    return space.newlist([W_UnicodeObject(part) for part in parts])
+    parts = split(self, delim, maxsplit)
+    return space.newlist_unicode(parts)
 
 
 def unicode_rsplit__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -828,13 +828,13 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(W_UnicodeObject(value[j1:i+1]))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_unicode(res)
 
 def sliced(space, s, start, stop, orig_obj):
     assert start >= 0
@@ -879,8 +879,14 @@
 
 def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old,
                                                  w_new, w_maxsplit):
-    return _unicode_replace(space, w_self, w_old._value, w_new._value,
-                            w_maxsplit)
+    maxsplit = space.int_w(w_maxsplit)
+    try:
+        return W_UnicodeObject(
+                replace(w_self._value, w_old._value, w_new._value, maxsplit))
+    except OverflowError:
+        raise OperationError(
+            space.w_OverflowError,
+            space.wrap("replace string is too long"))
 
 def unicode_replace__Unicode_ANY_ANY_ANY(space, w_self, w_old, w_new,
                                          w_maxsplit):
@@ -892,19 +898,9 @@
         new = unicode(space.bufferstr_w(w_new))
     else:
         new = space.unicode_w(w_new)
-    return _unicode_replace(space, w_self, old, new, w_maxsplit)
-
-def _unicode_replace(space, w_self, old, new, w_maxsplit):
-    if len(old):
-        parts = _split_with(w_self._value, old, space.int_w(w_maxsplit))
-    else:
-        self = w_self._value
-        maxsplit = space.int_w(w_maxsplit)
-        parts = _split_into_chars(self, maxsplit)
-
+    maxsplit = space.int_w(w_maxsplit)
     try:
-        one = ovfcheck(len(parts) * len(new))
-        ovfcheck(one + len(w_self._value))
+        return W_UnicodeObject(replace(w_self._value, old, new, maxsplit))
     except OverflowError:
         raise OperationError(
             space.w_OverflowError,
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -1,25 +1,50 @@
 """ String builder interface and string functions
 """
+import sys
 
 from rpython.annotator.model import (SomeObject, SomeString, s_None, SomeChar,
     SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePtr, SomePBC)
-from rpython.rlib.objectmodel import newlist_hint
+from rpython.rlib.objectmodel import newlist_hint, specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pairtype
+from rpython.rlib import jit
 
 
 # -------------- public API for string functions -----------------------
+
+ at specialize.argtype(0)
 def split(value, by, maxsplit=-1):
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     bylen = len(by)
     if bylen == 0:
         raise ValueError("empty separator")
 
+    start = 0
+    if bylen == 1:
+        # fast path: uses str.rfind(character) and str.count(character)
+        by = by[0]    # annotator hack: string -> char
+        count = value.count(by)
+        if 0 <= maxsplit < count:
+            count = maxsplit
+        res = newlist_hint(count + 1)
+        while count > 0:
+            next = value.find(by, start)
+            assert next >= 0 # cannot fail due to the value.count above
+            res.append(value[start:next])
+            start = next + bylen
+            count -= 1
+        res.append(value[start:len(value)])
+        return res
+
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
         res = []
-    start = 0
+
     while maxsplit != 0:
         next = value.find(by, start)
         if next < 0:
@@ -32,7 +57,12 @@
     return res
 
 
+ at specialize.argtype(0)
 def rsplit(value, by, maxsplit=-1):
+    if isinstance(value, str):
+        assert isinstance(by, str)
+    else:
+        assert isinstance(by, unicode)
     if maxsplit > 0:
         res = newlist_hint(min(maxsplit + 1, len(value)))
     else:
@@ -54,6 +84,109 @@
     res.reverse()
     return res
 
+
+ at specialize.argtype(0)
+def replace(input, sub, by, maxsplit=-1):
+    if isinstance(input, str):
+        assert isinstance(sub, str)
+        assert isinstance(by, str)
+        Builder = StringBuilder
+    else:
+        assert isinstance(sub, unicode)
+        assert isinstance(by, unicode)
+        Builder = UnicodeBuilder
+    if maxsplit == 0:
+        return input
+
+    if not sub:
+        upper = len(input)
+        if maxsplit > 0 and maxsplit < upper + 2:
+            upper = maxsplit - 1
+            assert upper >= 0
+
+        try:
+            result_size = ovfcheck(upper * len(by))
+            result_size = ovfcheck(result_size + upper)
+            result_size = ovfcheck(result_size + len(by))
+            remaining_size = len(input) - upper
+            result_size = ovfcheck(result_size + remaining_size)
+        except OverflowError:
+            raise
+        builder = Builder(result_size)
+        for i in range(upper):
+            builder.append(by)
+            builder.append(input[i])
+        builder.append(by)
+        builder.append_slice(input, upper, len(input))
+    else:
+        # First compute the exact result size
+        count = input.count(sub)
+        if count > maxsplit and maxsplit > 0:
+            count = maxsplit
+        diff_len = len(by) - len(sub)
+        try:
+            result_size = ovfcheck(diff_len * count)
+            result_size = ovfcheck(result_size + len(input))
+        except OverflowError:
+            raise
+
+        builder = Builder(result_size)
+        start = 0
+        sublen = len(sub)
+
+        while maxsplit != 0:
+            next = input.find(sub, start)
+            if next < 0:
+                break
+            builder.append_slice(input, start, next)
+            builder.append(by)
+            start = next + sublen
+            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+        builder.append_slice(input, start, len(input))
+
+    return builder.build()
+
+def _normalize_start_end(length, start, end):
+    if start < 0:
+        start += length
+        if start < 0:
+            start = 0
+    if end < 0:
+        end += length
+        if end < 0:
+            end = 0
+    elif end > length:
+        end = length
+    return start, end
+
+ at specialize.argtype(0)
+ at jit.elidable
+def startswith(u_self, prefix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    stop = start + len(prefix)
+    if stop > end:
+        return False
+    for i in range(len(prefix)):
+        if u_self[start+i] != prefix[i]:
+            return False
+    return True
+
+ at specialize.argtype(0)
+ at jit.elidable
+def endswith(u_self, suffix, start=0, end=sys.maxint):
+    length = len(u_self)
+    start, end = _normalize_start_end(length, start, end)
+    begin = end - len(suffix)
+    if begin < start:
+        return False
+    for i in range(len(suffix)):
+        if u_self[begin+i] != suffix[i]:
+            return False
+    return True
+
+
 # -------------- public API ---------------------------------
 
 INIT_SIZE = 100 # XXX tweak
@@ -271,3 +404,5 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+
+
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -1,6 +1,8 @@
 import sys, py
 
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit
+from rpython.rlib.rstring import replace, startswith, endswith
+from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin
 
 def test_split():
     assert split("", 'x') == ['']
@@ -10,9 +12,21 @@
     assert split('a|b|c|d', '|') == ['a', 'b', 'c', 'd']
     assert split('a|b|c|d', '|', 2) == ['a', 'b', 'c|d']
     assert split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+    assert split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
     assert split('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, split, 'abc', '')
 
+def test_split_unicode():
+    assert split(u"", u'x') == [u'']
+    assert split(u"a", u"a", 1) == [u'', u'']
+    assert split(u" ", u" ", 1) == [u'', u'']
+    assert split(u"aa", u"a", 2) == [u'', u'', u'']
+    assert split(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert split(u'a|b|c|d', u'|', 2) == [u'a', u'b', u'c|d']
+    assert split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert split(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, split, u'abc', u'')
+
 def test_rsplit():
     assert rsplit("a", "a", 1) == ['', '']
     assert rsplit(" ", " ", 1) == ['', '']
@@ -23,6 +37,111 @@
     assert rsplit('endcase test', 'test') == ['endcase ', '']
     py.test.raises(ValueError, rsplit, "abc", '')
 
+def test_rsplit_unicode():
+    assert rsplit(u"a", u"a", 1) == [u'', u'']
+    assert rsplit(u" ", u" ", 1) == [u'', u'']
+    assert rsplit(u"aa", u"a", 2) == [u'', u'', u'']
+    assert rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+    assert rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+    assert rsplit(u'endcase test', u'test') == [u'endcase ', u'']
+    py.test.raises(ValueError, rsplit, u"abc", u'')
+
+def test_string_replace():
+    assert replace('one!two!three!', '!', '@', 1) == 'one at two!three!'
+    assert replace('one!two!three!', '!', '') == 'onetwothree'
+    assert replace('one!two!three!', '!', '@', 2) == 'one at two@three!'
+    assert replace('one!two!three!', '!', '@', 3) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 4) == 'one at two@three@'
+    assert replace('one!two!three!', '!', '@', 0) == 'one!two!three!'
+    assert replace('one!two!three!', '!', '@') == 'one at two@three@'
+    assert replace('one!two!three!', 'x', '@') == 'one!two!three!'
+    assert replace('one!two!three!', 'x', '@', 2) == 'one!two!three!'
+    assert replace('abc', '', '-') == '-a-b-c-'
+    assert replace('abc', '', '-', 3) == '-a-b-c'
+    assert replace('abc', '', '-', 0) == 'abc'
+    assert replace('', '', '') == ''
+    assert replace('', '', 'a') == 'a'
+    assert replace('abc', 'ab', '--', 0) == 'abc'
+    assert replace('abc', 'xy', '--') == 'abc'
+    assert replace('123', '123', '') == ''
+    assert replace('123123', '123', '') == ''
+    assert replace('123x123', '123', '') == 'x'
+
+def test_string_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = "a" * (2**16)
+    with py.test.raises(OverflowError):
+        replace(s, "", s)
+    with py.test.raises(OverflowError):
+        replace(s, "a", s)
+    with py.test.raises(OverflowError):
+        replace(s, "a", s, len(s) - 10)
+
+def test_unicode_replace():
+    assert replace(u'one!two!three!', u'!', u'@', 1) == u'one at two!three!'
+    assert replace(u'one!two!three!', u'!', u'') == u'onetwothree'
+    assert replace(u'one!two!three!', u'!', u'@', 2) == u'one at two@three!'
+    assert replace(u'one!two!three!', u'!', u'@', 3) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 4) == u'one at two@three@'
+    assert replace(u'one!two!three!', u'!', u'@', 0) == u'one!two!three!'
+    assert replace(u'one!two!three!', u'!', u'@') == u'one at two@three@'
+    assert replace(u'one!two!three!', u'x', u'@') == u'one!two!three!'
+    assert replace(u'one!two!three!', u'x', u'@', 2) == u'one!two!three!'
+    assert replace(u'abc', u'', u'-') == u'-a-b-c-'
+    assert replace(u'abc', u'', u'-', 3) == u'-a-b-c'
+    assert replace(u'abc', u'', u'-', 0) == u'abc'
+    assert replace(u'', u'', u'') == u''
+    assert replace(u'', u'', u'a') == u'a'
+    assert replace(u'abc', u'ab', u'--', 0) == u'abc'
+    assert replace(u'abc', u'xy', u'--') == u'abc'
+    assert replace(u'123', u'123', u'') == u''
+    assert replace(u'123123', u'123', u'') == u''
+    assert replace(u'123x123', u'123', u'') == u'x'
+
+def test_unicode_replace_overflow():
+    if sys.maxint > 2**31-1:
+        py.test.skip("Wrong platform")
+    s = u"a" * (2**16)
+    with py.test.raises(OverflowError):
+        replace(s, u"", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s)
+    with py.test.raises(OverflowError):
+        replace(s, u"a", s, len(s) - 10)
+
+def test_startswith():
+    assert startswith('ab', 'ab') is True
+    assert startswith('ab', 'a') is True
+    assert startswith('ab', '') is True
+    assert startswith('x', 'a') is False
+    assert startswith('x', 'x') is True
+    assert startswith('', '') is True
+    assert startswith('', 'a') is False
+    assert startswith('x', 'xx') is False
+    assert startswith('y', 'xx') is False
+    assert startswith('ab', 'a', 0) is True
+    assert startswith('ab', 'a', 1) is False
+    assert startswith('ab', 'b', 1) is True
+    assert startswith('abc', 'bc', 1, 2) is False
+    assert startswith('abc', 'c', -1, 4) is True
+
+def test_endswith():
+    assert endswith('ab', 'ab') is True
+    assert endswith('ab', 'b') is True
+    assert endswith('ab', '') is True
+    assert endswith('x', 'a') is False
+    assert endswith('x', 'x') is True
+    assert endswith('', '') is True
+    assert endswith('', 'a') is False
+    assert endswith('x', 'xx') is False
+    assert endswith('y', 'xx') is False
+    assert endswith('abc', 'ab', 0, 2) is True
+    assert endswith('abc', 'bc', 1) is True
+    assert endswith('abc', 'bc', 2) is False
+    assert endswith('abc', 'b', -3, -1) is True
+
 def test_string_builder():
     s = StringBuilder()
     s.append("a")
@@ -42,4 +161,32 @@
     s.append_multiple_char(u'd', 4)
     assert s.build() == 'aabcbdddd'
     assert isinstance(s.build(), unicode)
-        
+
+
+class TestTranslates(LLRtypeMixin, BaseRtypingTest):
+    def test_split_rsplit(self):
+        def fn():
+            res = True
+            res = res and split('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and split('a//b//c//d', '//', 2) == ['a', 'b', 'c//d']
+            res = res and split(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            res = res and split(u'endcase test', u'test') == [u'endcase ', u'']
+            res = res and rsplit('a|b|c|d', '|', 2) == ['a|b', 'c', 'd']
+            res = res and rsplit('a//b//c//d', '//') == ['a', 'b', 'c', 'd']
+            res = res and rsplit(u'a|b|c|d', u'|') == [u'a', u'b', u'c', u'd']
+            res = res and rsplit(u'a|b|c|d', u'|', 2) == [u'a|b', u'c', u'd']
+            res = res and rsplit(u'a//b//c//d', u'//') == [u'a', u'b', u'c', u'd']
+            return res
+        res = self.interpret(fn, [])
+        assert res
+
+    def test_replace(self):
+        def fn():
+            res = True
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            res = res and replace('abc', 'ab', '--', 0) == 'abc'
+            res = res and replace('abc', 'xy', '--') == 'abc'
+            return res
+        res = self.interpret(fn, [])
+        assert res

From noreply at buildbot.pypy.org  Mon Jun 17 17:59:24 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Mon, 17 Jun 2013 17:59:24 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: Apply changes from default.
Message-ID: <20130617155924.610551C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64922:680e7c196d9f
Date: 2013-06-17 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/680e7c196d9f/

Log:	Apply changes from default.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -134,6 +134,9 @@
         else:
             return ch
 
+    def _newlist_unwrapped(self, space, lst):
+        return space.newlist_str(lst)
+
     @staticmethod
     @unwrap_spec(w_object = WrappedDefault(""))
     def descr_new(space, w_stringtype, w_object):
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import split
+from rpython.rlib.rstring import split, rsplit, replace, startswith, endswith
 
 
 class StringMethods(object):
@@ -473,62 +473,12 @@
         input = self._val()
         sub = self._op_val(space, w_old)
         by = self._op_val(space, w_new)
-
-        if count == 0:
-            return space.wrap(input)
-
-        if not sub:
-            upper = len(input)
-            if count > 0 and count < upper + 2:
-                upper = count - 1
-                assert upper >= 0
-
-            try:
-                result_size = ovfcheck(upper * len(by))
-                result_size = ovfcheck(result_size + upper)
-                result_size = ovfcheck(result_size + len(by))
-                remaining_size = len(input) - upper
-                result_size = ovfcheck(result_size + remaining_size)
-            except OverflowError:
-                raise OperationError(space.w_OverflowError,
-                    space.wrap("replace string is too long")
-                )
-            builder = self._builder(result_size)
-            for i in range(upper):
-                builder.append(by)
-                builder.append(input[i])
-            builder.append(by)
-            builder.append_slice(input, upper, len(input))
-        else:
-            # First compute the exact result size
-            count2 = input.count(sub)
-            if count2 > count and count > 0:
-                count2 = count
-            diff_len = len(by) - len(sub)
-            try:
-                result_size = ovfcheck(diff_len * count2)
-                result_size = ovfcheck(result_size + len(input))
-            except OverflowError:
-                raise OperationError(space.w_OverflowError,
-                    space.wrap("replace string is too long")
-                )
-
-            builder = self._builder(result_size)
-            start = 0
-            sublen = len(sub)
-
-            while count != 0:
-                next = input.find(sub, start)
-                if next < 0:
-                    break
-                builder.append_slice(input, start, next)
-                builder.append(by)
-                start = next + sublen
-                count -= 1   # NB. if it's already < 0, it stays < 0
-
-            builder.append_slice(input, start, len(input))
-
-        return space.wrap(builder.build())
+        try:
+            res = replace(input, sub, by, count)
+        except OverflowError:
+            raise OperationError(space.w_OverflowError,
+                                 space.wrap("replace string is too long"))
+        return space.wrap(res)
 
     @unwrap_spec(maxsplit=int)
     def descr_split(self, space, w_sep=None, maxsplit=-1):
@@ -561,35 +511,18 @@
                 # continue to look from the character following the space after the word
                 i = j + 1
 
-            return space.newlist_str(res)
+            return self._newlist_unwrapped(space, res)
 
         by = self._op_val(space, w_sep)
         bylen = len(by)
         if bylen == 0:
             raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-        if bylen == 1 and maxsplit < 0:
-            res = []
-            start = 0
-            # fast path: uses str.rfind(character) and str.count(character)
-            by = by[0]    # annotator hack: string -> char
-            count = value.count(by)
-            res = [None] * (count + 1)
-            end = len(value)
-            while count >= 0:
-                assert end >= 0
-                prev = value.rfind(by, 0, end)
-                start = prev + 1
-                assert start >= 0
-                res[count] = value[start:end]
-                count -= 1
-                end = prev
-        else:
-            res = split(value, by, maxsplit)
-        return space.newlist_str(res)
+        res = split(value, by, maxsplit)
+        return self._newlist_unwrapped(space, res)
 
     @unwrap_spec(maxsplit=int)
     def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
-        res_w = []
+        res = []
         value = self._val()
         length = len(value)
         if space.is_none(w_sep):
@@ -616,41 +549,26 @@
                 # the word is value[j+1:i+1]
                 j1 = j + 1
                 assert j1 >= 0
-                res_w.append(self._sliced(space, value, j1, i+1, self))
+                res.append(value[j1:i+1])
 
                 # continue to look from the character before the space before the word
                 i = j - 1
 
-            res_w.reverse()
-            return space.newlist(res_w)
+            res.reverse()
+            return self._newlist_unwrapped(space, res)
 
         by = self._op_val(space, w_sep)
         bylen = len(by)
-
-        value = self._value
-        end = len(value)
-        by = w_sep._value
-        bylen = len(by)
         if bylen == 0:
             raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-        while maxsplit != 0:
-            next = value.rfind(by, 0, end)
-            if next < 0:
-                break
-            res_w.append(self._sliced(space, value, next+bylen, end, self))
-            end = next
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        res_w.append(self._sliced(space, value, 0, end, self))
-        res_w.reverse()
-        return space.newlist(res_w)
+        res = rsplit(value, by, maxsplit)
+        return self._newlist_unwrapped(space, res)
 
     @unwrap_spec(keepends=bool)
     def descr_splitlines(self, space, keepends=False):
         data = self._value
         selflen = len(data)
-        strs_w = []
+        strs = []
         i = j = 0
         while i < selflen:
             # Find a line and append it
@@ -663,12 +581,12 @@
                 i += 1
             if keepends:
                 eol = i
-            strs_w.append(self._sliced(space, data, j, eol, self))
+            strs.append(data[j:eol])
             j = i
 
         if j < selflen:
-            strs_w.append(self._sliced(space, data, j, len(data), self))
-        return space.newlist(strs_w)
+            strs.append(data[j:len(data)])
+        return self._newlist_unwrapped(space, strs)
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start, w_end,
@@ -681,14 +599,7 @@
         return space.newbool(self._startswith(space, value, w_prefix, start, end))
 
     def _startswith(self, space, value, w_prefix, start, end):
-        prefix = self._op_val(space, w_prefix)
-        stop = start + len(prefix)
-        if stop > end:
-            return False
-        for i in range(len(prefix)):
-            if value[start+i] != prefix[i]:
-                return False
-        return True
+        return startswith(value, self._op_val(space, w_prefix), start, end)
 
     def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start,
@@ -701,15 +612,8 @@
             return space.w_False
         return space.newbool(self._endswith(space, value, w_suffix, start, end))
 
-    def _endswith(self, space, value, w_suffix, start, end):
-        suffix = self._op_val(space, w_suffix)
-        begin = end - len(suffix)
-        if begin < start:
-            return False
-        for i in range(len(suffix)):
-            if value[begin+i] != suffix[i]:
-                return False
-        return True
+    def _endswith(self, space, value, w_prefix, start, end):
+        return endswith(value, self._op_val(space, w_prefix), start, end)
 
     def _strip(self, space, w_chars, left, right):
         "internal function called by str_xstrip methods"
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -119,6 +119,9 @@
     def _lower(self, ch):
         return unichr(unicodedb.tolower(ord(ch)))
 
+    def _newlist_unwrapped(self, space, lst):
+        return space.newlist_unicode(lst)
+
     def descr_repr(self, space):
         chars = self._value
         size = len(chars)
@@ -863,6 +866,7 @@
     return parts
 
 def _split_with(self, with_, maxsplit=-1):
+    xxx # remove
     parts = []
     start = 0
     end = len(self)
@@ -910,6 +914,7 @@
 
 
 def unicode_expandtabs__Unicode_ANY(space, w_self, tabsize):
+    xxx # remove
     self = w_self._value
     parts = _split_with(self, u'\t')
     result = [parts[0]]

From noreply at buildbot.pypy.org  Mon Jun 17 17:59:26 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Mon, 17 Jun 2013 17:59:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix pypy.tool.pytest.appsupport to work
 with new versions of pytest.
Message-ID: <20130617155926.0D5051C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: 
Changeset: r64923:cbd32584ae93
Date: 2013-06-17 17:55 +0200
http://bitbucket.org/pypy/pypy/changeset/cbd32584ae93/

Log:	Fix pypy.tool.pytest.appsupport to work with new versions of pytest.

diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -1,3 +1,5 @@
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
 import py
 from pypy.interpreter import gateway, pycode
 from pypy.interpreter.error import OperationError
@@ -35,8 +37,13 @@
             return None
     fullsource = property(fullsource, None, None, "Full source of AppCode")
 
-    def getargs(self):
-        return self.raw.co_varnames[:self.raw.co_argcount]
+    def getargs(self, var=False):
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
 
 class AppFrame(py.code.Frame):
 
@@ -70,10 +77,10 @@
     def is_true(self, w_value):
         return self.space.is_true(w_value)
 
-    def getargs(self):
+    def getargs(self, var=False):
         space = self.space
         retval = []
-        for arg in self.code.getargs():
+        for arg in self.code.getargs(var):
             w_val = space.finditem(self.w_locals, space.wrap(arg))
             if w_val is None:
                 w_val = space.wrap('<no value found>')

From noreply at buildbot.pypy.org  Mon Jun 17 17:59:27 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Mon, 17 Jun 2013 17:59:27 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: hg merge default
Message-ID: <20130617155927.39EDB1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64924:92862dc40ad6
Date: 2013-06-17 17:57 +0200
http://bitbucket.org/pypy/pypy/changeset/92862dc40ad6/

Log:	hg merge default

diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -1,3 +1,5 @@
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
 import py
 from pypy.interpreter import gateway, pycode
 from pypy.interpreter.error import OperationError
@@ -35,8 +37,13 @@
             return None
     fullsource = property(fullsource, None, None, "Full source of AppCode")
 
-    def getargs(self):
-        return self.raw.co_varnames[:self.raw.co_argcount]
+    def getargs(self, var=False):
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
 
 class AppFrame(py.code.Frame):
 
@@ -70,10 +77,10 @@
     def is_true(self, w_value):
         return self.space.is_true(w_value)
 
-    def getargs(self):
+    def getargs(self, var=False):
         space = self.space
         retval = []
-        for arg in self.code.getargs():
+        for arg in self.code.getargs(var):
             w_val = space.finditem(self.w_locals, space.wrap(arg))
             if w_val is None:
                 w_val = space.wrap('<no value found>')

From noreply at buildbot.pypy.org  Mon Jun 17 18:01:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 18:01:31 +0200 (CEST)
Subject: [pypy-commit] stmgc default: next test passes
Message-ID: <20130617160131.CD6651C1001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r176:f322b8eb887f
Date: 2013-06-17 17:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/f322b8eb887f/

Log:	next test passes

diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -105,3 +105,19 @@
     assert count_pages() == 1
     major_collect()
     assert count_pages() == 0
+
+def test_free_all_unused_local_pages():
+    def f1(r):
+        p1 = oalloc(HDR)
+        assert count_pages() == 1
+        r.set(1)
+        r.wait(2)
+        assert count_pages() == 0
+    def f2(r):
+        p2 = oalloc(HDR)
+        r.wait(1)
+        assert count_pages() == 1
+        major_collect()
+        assert count_pages() == 0
+        r.set(2)
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Mon Jun 17 18:01:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 18:01:32 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617160132.EC7231C1001@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r177:97a537d7e1db
Date: 2013-06-17 18:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/97a537d7e1db/

Log:	Next test

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -183,6 +183,95 @@
 }
 
 
+/***** Major collections: marking *****/
+
+static struct GcPtrList objects_to_trace;
+
+static void visit(gcptr *pobj)
+{
+    gcptr obj = *pobj;
+    if (obj == NULL)
+        return;
+
+ restart:
+    if (obj->h_tid & GCFLAG_VISITED)
+        return;    /* already seen */
+
+    if (obj->h_tid & (GCFLAG_PUBLIC_TO_PRIVATE | GCFLAG_STUB)) {
+        if (obj->h_revision & 1) { // "is not a ptr", so no more recent version
+            obj->h_tid &= ~GCFLAG_PUBLIC_TO_PRIVATE; // see also fix_outdated()
+        }
+        else {
+            obj = (gcptr)obj->h_revision;   // go visit the more recent version
+            *pobj = obj;
+            goto restart;
+        }
+    }
+    else
+        assert(obj->h_revision & 1);
+
+    obj->h_tid |= GCFLAG_VISITED;
+    gcptrlist_insert(&objects_to_trace, obj);
+}
+
+static void visit_all_objects(void)
+{
+    while (gcptrlist_size(&objects_to_trace) > 0) {
+        gcptr obj = gcptrlist_pop(&objects_to_trace);
+        stmcb_trace(obj, &visit);
+    }
+}
+
+static void mark_roots(gcptr *root, gcptr *end)
+{
+    //assert(*root == END_MARKER);
+    //root++;
+    while (root != end)
+        visit(root++);
+}
+
+static void mark_all_stack_roots(void)
+{
+    struct tx_descriptor *d;
+    for (d = stm_tx_head; d; d = d->tx_next) {
+
+        /* the roots pushed on the shadowstack */
+        mark_roots(d->shadowstack, *d->shadowstack_end_ref);
+
+#if 0
+        /* the thread-local object */
+        visit(d->thread_local_obj_ref);
+#endif
+
+        /* the current transaction's private copies of public objects */
+        wlog_t *item;
+        G2L_LOOP_FORWARD(d->public_to_private, item) {
+
+            /* note that 'item->addr' is also in the read set, so if it was
+               outdated, it will be found at that time */
+            visit(&item->addr);
+            visit(&item->val);
+
+        } G2L_LOOP_END;
+
+        /* make sure that the other lists are empty */
+        assert(gcptrlist_size(&d->public_with_young_copy) == 0);
+        assert(gcptrlist_size(&d->public_descriptor->stolen_objects) == 0);
+        assert(gcptrlist_size(&d->public_descriptor->stolen_young_stubs) == 0);
+        /* NOT NECESSARILY EMPTY:
+           - list_of_read_objects
+           - private_from_protected
+           - public_to_private
+           - old_objects_to_trace
+        */
+        assert(gcptrlist_size(&d->list_of_read_objects) ==
+               d->num_read_objects_known_old);
+        assert(gcptrlist_size(&d->private_from_protected) ==
+               d->num_private_from_protected_known_old);
+    }
+}
+
+
 /***** Major collections: sweeping *****/
 
 static void sweep_pages(struct tx_public_descriptor *gcp, int size_class,
@@ -313,7 +402,9 @@
 
     assert(gcptrlist_size(&objects_to_trace) == 0);
     mark_prebuilt_roots();
+#endif
     mark_all_stack_roots();
+#if 0
     visit_all_objects();
     gcptrlist_delete(&objects_to_trace);
     clean_up_lists_of_read_objects_and_fix_outdated_flags();
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -121,3 +121,15 @@
         assert count_pages() == 0
         r.set(2)
     run_parallel(f1, f2)
+
+def test_keep_local_roots_alive():
+    p1 = oalloc(HDR)
+    assert count_pages() == 1
+    lib.stm_push_root(p1)
+    major_collect()
+    p2 = lib.stm_pop_root()
+    assert p2 == p1    # oalloc() does not use the nursery
+    assert count_pages() == 1
+    p3 = oalloc(HDR)
+    p4 = oalloc(HDR)
+    assert p2 != p3 != p4 != p2

From noreply at buildbot.pypy.org  Mon Jun 17 18:11:22 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Mon, 17 Jun 2013 18:11:22 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: Cleanup.
Message-ID: <20130617161122.AE6B51C03D9@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r64925:b38d402d8513
Date: 2013-06-17 18:09 +0200
http://bitbucket.org/pypy/pypy/changeset/b38d402d8513/

Log:	Cleanup.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -1,29 +1,19 @@
 """The builtin str implementation"""
 
-from sys import maxint
 from pypy.interpreter.buffer import StringBuffer
-from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.interpreter.error import operationerrfmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.objspace.std import newformat, slicetype
+from pypy.objspace.std import newformat
 from pypy.objspace.std.basestringtype import basestring_typedef
 from pypy.objspace.std.formatting import mod_format
-from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import W_Object, registerimplementation
-from pypy.objspace.std.multimethod import FailedToImplement
-from pypy.objspace.std.noneobject import W_NoneObject
-from pypy.objspace.std.register_all import register_all
-from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
-from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
+from pypy.objspace.std.stdtypedef import StdTypeDef
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.unicodeobject import (unicode_from_string,
     decode_object, _get_encoding_and_errors)
-from rpython.rlib import jit
 from rpython.rlib.jit import we_are_jitted
-from rpython.rlib.objectmodel import (compute_hash, compute_unique_id,
-        specialize)
-from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import (StringBuilder, split, rsplit, replace,
-    endswith, startswith)
+from rpython.rlib.objectmodel import compute_hash, compute_unique_id
+from rpython.rlib.rstring import StringBuilder
 
 
 class W_AbstractBytesObject(W_Object):
@@ -331,8 +321,6 @@
     __getnewargs__ = interp2app(W_BytesObject.descr_getnewargs),
 )
 
-str_typedef.registermethods(globals())
-
 
 def string_escape_encode(s, quote):
 
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -524,7 +524,6 @@
     def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
         res = []
         value = self._val()
-        length = len(value)
         if space.is_none(w_sep):
             i = len(value)-1
             while True:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -11,11 +11,9 @@
 from pypy.objspace.std.stdtypedef import StdTypeDef
 from pypy.objspace.std.stringmethods import StringMethods
 from rpython.rlib.objectmodel import compute_hash, compute_unique_id
-from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.rstring import UnicodeBuilder
 from rpython.rlib.runicode import (str_decode_utf_8, str_decode_ascii,
     unicode_encode_utf_8, unicode_encode_ascii, make_unicode_escape_function)
-from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode',
            'encode_object', 'decode_object', 'unicode_from_object',
@@ -350,11 +348,6 @@
         # raising UnicodeDecodeError is messy, "please crash for me"
         return unicode_from_encoded_object(space, w_str, "ascii", "strict")
 
-def unicode_decode__unitypedef_ANY_ANY(space, w_unicode, w_encoding=None,
-                                       w_errors=None):
-    return space.call_method(space.str(w_unicode), 'decode',
-                             w_encoding, w_errors)
-
 
 @unwrap_spec(w_string = WrappedDefault(""))
 def descr_new_(space, w_unicodetype, w_string, w_encoding=None, w_errors=None):
@@ -503,444 +496,6 @@
                 raise OperationError(space.w_UnicodeEncodeError, space.newtuple([w_encoding, w_unistr, w_start, w_end, w_reason]))
     return ''.join(result)
 
-# checks if should trigger an unicode warning
-def _unicode_string_comparison(space, w_uni, w_str, inverse, uni_from_str):
-    try:
-        w_uni2 = uni_from_str(space, w_str)
-    except OperationError, e:
-        if e.match(space, space.w_UnicodeDecodeError):
-            msg = ("Unicode %s comparison failed to convert both arguments to "
-                   "Unicode - interpreting them as being unequal" %
-                   "unequal" if inverse else "equal")
-            space.warn(space.wrap(msg), space.w_UnicodeWarning)
-            return space.newbool(inverse)
-        raise
-    result = space.eq(w_uni, w_uni2)
-    if inverse:
-        return space.not_(result)
-    return result
-
-def _isspace(uchar):
-    return unicodedb.isspace(ord(uchar))
-
-def make_generic(funcname):
-    def func(space, w_self):
-        v = w_self._value
-        if len(v) == 0:
-            return space.w_False
-        for idx in range(len(v)):
-            if not getattr(unicodedb, funcname)(ord(v[idx])):
-                return space.w_False
-        return space.w_True
-    return func_with_new_name(func, "unicode_%s__Unicode" % (funcname, ))
-
-unicode_isspace__Unicode = make_generic("isspace")
-unicode_isalpha__Unicode = make_generic("isalpha")
-unicode_isalnum__Unicode = make_generic("isalnum")
-unicode_isdecimal__Unicode = make_generic("isdecimal")
-unicode_isdigit__Unicode = make_generic("isdigit")
-unicode_isnumeric__Unicode = make_generic("isnumeric")
-
-def unicode_islower__Unicode(space, w_unicode):
-    cased = False
-    for uchar in w_unicode._value:
-        if (unicodedb.isupper(ord(uchar)) or
-            unicodedb.istitle(ord(uchar))):
-            return space.w_False
-        if not cased and unicodedb.islower(ord(uchar)):
-            cased = True
-    return space.newbool(cased)
-
-def unicode_isupper__Unicode(space, w_unicode):
-    cased = False
-    for uchar in w_unicode._value:
-        if (unicodedb.islower(ord(uchar)) or
-            unicodedb.istitle(ord(uchar))):
-            return space.w_False
-        if not cased and unicodedb.isupper(ord(uchar)):
-            cased = True
-    return space.newbool(cased)
-
-def unicode_istitle__Unicode(space, w_unicode):
-    cased = False
-    previous_is_cased = False
-    for uchar in w_unicode._value:
-        if (unicodedb.isupper(ord(uchar)) or
-            unicodedb.istitle(ord(uchar))):
-            if previous_is_cased:
-                return space.w_False
-            previous_is_cased = cased = True
-        elif unicodedb.islower(ord(uchar)):
-            if not previous_is_cased:
-                return space.w_False
-            previous_is_cased = cased = True
-        else:
-            previous_is_cased = False
-    return space.newbool(cased)
-
-def _strip(space, w_self, w_chars, left, right):
-    "internal function called by str_xstrip methods"
-    u_self = w_self._value
-    u_chars = w_chars._value
-
-    lpos = 0
-    rpos = len(u_self)
-
-    if left:
-        while lpos < rpos and u_self[lpos] in u_chars:
-           lpos += 1
-
-    if right:
-        while rpos > lpos and u_self[rpos - 1] in u_chars:
-           rpos -= 1
-
-    assert rpos >= 0
-    result = u_self[lpos: rpos]
-    return W_UnicodeObject(result)
-
-def _strip_none(space, w_self, left, right):
-    "internal function called by str_xstrip methods"
-    u_self = w_self._value
-
-    lpos = 0
-    rpos = len(u_self)
-
-    if left:
-        while lpos < rpos and _isspace(u_self[lpos]):
-           lpos += 1
-
-    if right:
-        while rpos > lpos and _isspace(u_self[rpos - 1]):
-           rpos -= 1
-
-    assert rpos >= 0
-    result = u_self[lpos: rpos]
-    return W_UnicodeObject(result)
-
-
-def unicode_capitalize__Unicode(space, w_self):
-    input = w_self._value
-    if len(input) == 0:
-        return W_UnicodeObject.EMPTY
-    builder = UnicodeBuilder(len(input))
-    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
-    for i in range(1, len(input)):
-        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
-    return W_UnicodeObject(builder.build())
-
-def unicode_title__Unicode(space, w_self):
-    input = w_self._value
-    if len(input) == 0:
-        return w_self
-
-    builder = UnicodeBuilder(len(input))
-    previous_is_cased = False
-    for i in range(len(input)):
-        unichar = ord(input[i])
-        if previous_is_cased:
-            builder.append(unichr(unicodedb.tolower(unichar)))
-        else:
-            builder.append(unichr(unicodedb.totitle(unichar)))
-        previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(builder.build())
-
-def unicode_lower__Unicode(space, w_self):
-    input = w_self._value
-    builder = UnicodeBuilder(len(input))
-    for i in range(len(input)):
-        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
-    return W_UnicodeObject(builder.build())
-
-def unicode_upper__Unicode(space, w_self):
-    input = w_self._value
-    builder = UnicodeBuilder(len(input))
-    for i in range(len(input)):
-        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
-    return W_UnicodeObject(builder.build())
-
-def unicode_swapcase__Unicode(space, w_self):
-    input = w_self._value
-    builder = UnicodeBuilder(len(input))
-    for i in range(len(input)):
-        unichar = ord(input[i])
-        if unicodedb.islower(unichar):
-            builder.append(unichr(unicodedb.toupper(unichar)))
-        elif unicodedb.isupper(unichar):
-            builder.append(unichr(unicodedb.tolower(unichar)))
-        else:
-            builder.append(input[i])
-    return W_UnicodeObject(builder.build())
-
-def _normalize_index(length, index):
-    if index < 0:
-        index += length
-        if index < 0:
-            index = 0
-    elif index > length:
-        index = length
-    return index
-
-def _to_unichar_w(space, w_char):
-    try:
-        unistr = space.unicode_w(w_char)
-    except OperationError, e:
-        if e.match(space, space.w_TypeError):
-            msg = 'The fill character cannot be converted to Unicode'
-            raise OperationError(space.w_TypeError, space.wrap(msg))
-        else:
-            raise
-
-    if len(unistr) != 1:
-        raise OperationError(space.w_TypeError, space.wrap('The fill character must be exactly one character long'))
-    return unistr[0]
-
-def unicode_center__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar):
-    self = w_self._value
-    width = space.int_w(w_width)
-    fillchar = _to_unichar_w(space, w_fillchar)
-    padding = width - len(self)
-    if padding < 0:
-        return w_self.create_if_subclassed()
-    leftpad = padding // 2 + (padding & width & 1)
-    result = [fillchar] * width
-    for i in range(len(self)):
-        result[leftpad + i] = self[i]
-    return W_UnicodeObject(u''.join(result))
-
-def unicode_ljust__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar):
-    self = w_self._value
-    width = space.int_w(w_width)
-    fillchar = _to_unichar_w(space, w_fillchar)
-    padding = width - len(self)
-    if padding < 0:
-        return w_self.create_if_subclassed()
-    result = [fillchar] * width
-    for i in range(len(self)):
-        result[i] = self[i]
-    return W_UnicodeObject(u''.join(result))
-
-def unicode_rjust__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar):
-    self = w_self._value
-    width = space.int_w(w_width)
-    fillchar = _to_unichar_w(space, w_fillchar)
-    padding = width - len(self)
-    if padding < 0:
-        return w_self.create_if_subclassed()
-    result = [fillchar] * width
-    for i in range(len(self)):
-        result[padding + i] = self[i]
-    return W_UnicodeObject(u''.join(result))
-
-def unicode_splitlines__Unicode_ANY(space, w_self, w_keepends):
-    self = w_self._value
-    keepends = 0
-    if space.int_w(w_keepends):
-        keepends = 1
-    if len(self) == 0:
-        return space.newlist([])
-
-    start = 0
-    end = len(self)
-    pos = 0
-    lines = []
-    while pos < end:
-        if unicodedb.islinebreak(ord(self[pos])):
-            if (self[pos] == u'\r' and pos + 1 < end and
-                self[pos + 1] == u'\n'):
-                # Count CRLF as one linebreak
-                lines.append(self[start:pos + keepends * 2])
-                pos += 1
-            else:
-                lines.append(self[start:pos + keepends])
-            pos += 1
-            start = pos
-        else:
-            pos += 1
-    if not unicodedb.islinebreak(ord(self[end - 1])):
-        lines.append(self[start:])
-    return space.newlist_unicode(lines)
-
-def unicode_split__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
-    maxsplit = space.int_w(w_maxsplit)
-    res = []
-    value = w_self._value
-    length = len(value)
-    i = 0
-    while True:
-        # find the beginning of the next word
-        while i < length:
-            if not _isspace(value[i]):
-                break   # found
-            i += 1
-        else:
-            break  # end of string, finished
-
-        # find the end of the word
-        if maxsplit == 0:
-            j = length   # take all the rest of the string
-        else:
-            j = i + 1
-            while j < length and not _isspace(value[j]):
-                j += 1
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        # the word is value[i:j]
-        res.append(value[i:j])
-
-        # continue to look from the character following the space after the word
-        i = j + 1
-
-    return space.newlist_unicode(res)
-
-def unicode_split__Unicode_Unicode_ANY(space, w_self, w_delim, w_maxsplit):
-    self = w_self._value
-    delim = w_delim._value
-    maxsplit = space.int_w(w_maxsplit)
-    delim_len = len(delim)
-    if delim_len == 0:
-        raise OperationError(space.w_ValueError,
-                             space.wrap('empty separator'))
-    parts = split(self, delim, maxsplit)
-    return space.newlist_unicode(parts)
-
-
-def unicode_rsplit__Unicode_None_ANY(space, w_self, w_none, w_maxsplit):
-    maxsplit = space.int_w(w_maxsplit)
-    res = []
-    value = w_self._value
-    i = len(value)-1
-    while True:
-        # starting from the end, find the end of the next word
-        while i >= 0:
-            if not _isspace(value[i]):
-                break   # found
-            i -= 1
-        else:
-            break  # end of string, finished
-
-        # find the start of the word
-        # (more precisely, 'j' will be the space character before the word)
-        if maxsplit == 0:
-            j = -1   # take all the rest of the string
-        else:
-            j = i - 1
-            while j >= 0 and not _isspace(value[j]):
-                j -= 1
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        # the word is value[j+1:i+1]
-        j1 = j + 1
-        assert j1 >= 0
-        res.append(value[j1:i+1])
-
-        # continue to look from the character before the space before the word
-        i = j - 1
-
-    res.reverse()
-    return space.newlist_unicode(res)
-
-def sliced(space, s, start, stop, orig_obj):
-    assert start >= 0
-    assert stop >= 0
-    if start == 0 and stop == len(s) and space.is_w(space.type(orig_obj), space.w_unicode):
-        return orig_obj
-    return space.wrap( s[start:stop])
-
-#unicode_rsplit__Unicode_Unicode_ANY = make_rsplit_with_delim('unicode_rsplit__Unicode_Unicode_ANY',
-#                                                             sliced)
-
-def _split_into_chars(self, maxsplit):
-    if maxsplit == 0:
-        return [self]
-    index = 0
-    end = len(self)
-    parts = [u'']
-    maxsplit -= 1
-    while maxsplit != 0:
-        if index >= end:
-            break
-        parts.append(self[index])
-        index += 1
-        maxsplit -= 1
-    parts.append(self[index:])
-    return parts
-
-def _split_with(self, with_, maxsplit=-1):
-    xxx # remove
-    parts = []
-    start = 0
-    end = len(self)
-    length = len(with_)
-    while maxsplit != 0:
-        index = self.find(with_, start, end)
-        if index < 0:
-            break
-        parts.append(self[start:index])
-        start = index + length
-        maxsplit -= 1
-    parts.append(self[start:])
-    return parts
-
-def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old,
-                                                 w_new, w_maxsplit):
-    maxsplit = space.int_w(w_maxsplit)
-    try:
-        return W_UnicodeObject(
-                replace(w_self._value, w_old._value, w_new._value, maxsplit))
-    except OverflowError:
-        raise OperationError(
-            space.w_OverflowError,
-            space.wrap("replace string is too long"))
-
-def unicode_replace__Unicode_ANY_ANY_ANY(space, w_self, w_old, w_new,
-                                         w_maxsplit):
-    if not space.isinstance_w(w_old, space.w_unicode):
-        old = unicode(space.bufferstr_w(w_old))
-    else:
-        old = space.unicode_w(w_old)
-    if not space.isinstance_w(w_new, space.w_unicode):
-        new = unicode(space.bufferstr_w(w_new))
-    else:
-        new = space.unicode_w(w_new)
-    maxsplit = space.int_w(w_maxsplit)
-    try:
-        return W_UnicodeObject(replace(w_self._value, old, new, maxsplit))
-    except OverflowError:
-        raise OperationError(
-            space.w_OverflowError,
-            space.wrap("replace string is too long"))
-
-    return W_UnicodeObject(new.join(parts))
-
-
-def unicode_expandtabs__Unicode_ANY(space, w_self, tabsize):
-    xxx # remove
-    self = w_self._value
-    parts = _split_with(self, u'\t')
-    result = [parts[0]]
-    prevsize = 0
-    for ch in parts[0]:
-        prevsize += 1
-        if ch == u"\n" or ch ==  u"\r":
-            prevsize = 0
-    totalsize = prevsize
-
-    for i in range(1, len(parts)):
-        pad = tabsize - prevsize % tabsize
-        nextpart = parts[i]
-        try:
-            totalsize = ovfcheck(totalsize + pad)
-            totalsize = ovfcheck(totalsize + len(nextpart))
-            result.append(u' ' * pad)
-        except OverflowError:
-            raise OperationError(space.w_OverflowError, space.wrap('new string is too long'))
-        result.append(nextpart)
-        prevsize = 0
-        for ch in nextpart:
-            prevsize += 1
-            if ch in (u"\n", u"\r"):
-                prevsize = 0
-    return space.wrap(u''.join(result))
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=False, unicode_output=False, quotes=True, prefix='u')

From noreply at buildbot.pypy.org  Mon Jun 17 18:15:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 18:15:43 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test,
 and the simplest fix so far: move a "#if 0" one line.
Message-ID: <20130617161543.957AE1C03D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r178:58c077a3a7f6
Date: 2013-06-17 18:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/58c077a3a7f6/

Log:	Next test, and the simplest fix so far: move a "#if 0" one line.

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -404,8 +404,8 @@
     mark_prebuilt_roots();
 #endif
     mark_all_stack_roots();
+    visit_all_objects();
 #if 0
-    visit_all_objects();
     gcptrlist_delete(&objects_to_trace);
     clean_up_lists_of_read_objects_and_fix_outdated_flags();
 #endif
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -133,3 +133,13 @@
     p3 = oalloc(HDR)
     p4 = oalloc(HDR)
     assert p2 != p3 != p4 != p2
+
+def test_trace_simple():
+    p1 = oalloc_refs(1)
+    p2 = oalloc(HDR)
+    rawsetptr(p1, 0, p2)
+    lib.stm_push_root(p1)
+    major_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1    # oalloc() does not use the nursery
+    assert count_pages() == 2   # one for p1, one for p2, which have != sizes

From noreply at buildbot.pypy.org  Mon Jun 17 18:20:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 18:20:19 +0200 (CEST)
Subject: [pypy-commit] stmgc default: update_next_threshold()
Message-ID: <20130617162019.66B821C03D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r179:0531f4f549cb
Date: 2013-06-17 18:19 +0200
http://bitbucket.org/pypy/stmgc/changeset/0531f4f549cb/

Log:	update_next_threshold()

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -392,6 +392,35 @@
 
 /***** Major collections: main *****/
 
+void update_next_threshold(void)
+{
+    uintptr_t free_space_in_pages, next;
+
+    /* the limit will be reached when we have allocated 0.82 times mc_total */
+    next = (uintptr_t)(mc_total_in_use * (GC_MAJOR_COLLECT-1.0));
+
+    /* this limit should be at least GC_MIN */
+    if (next < GC_MIN)
+        next = GC_MIN;
+
+    /* this difference gives the size allocated in pages but unused so far */
+    assert(mc_total_in_use <= mc_total_reserved);
+    free_space_in_pages = mc_total_reserved - mc_total_in_use;
+
+    /* decrement 'next' by this much, because it will not be accounted for */
+    if (next >= free_space_in_pages)
+        next -= free_space_in_pages;
+    else
+        next = 0;
+
+    /* allow for headroom: enforce the smallest allowed value */
+    if (next < GC_EXPAND)
+        next = GC_EXPAND;
+
+    /* done */
+    countdown_next_major_coll = next;
+}
+
 void stm_major_collect(void)
 {
     stmgcpage_acquire_global_lock();
@@ -416,9 +445,7 @@
     free_unused_global_pages();
 #endif
     free_closed_thread_descriptors();
-#if 0
     update_next_threshold();
-#endif
 
     fprintf(stderr, "| %lu bytes alive, %lu not used, countdown %lu\n`-----\n",
             (unsigned long)mc_total_in_use,

From noreply at buildbot.pypy.org  Mon Jun 17 18:20:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 18:20:20 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617162020.81DB61C03D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r180:409c5810395a
Date: 2013-06-17 18:19 +0200
http://bitbucket.org/pypy/stmgc/changeset/409c5810395a/

Log:	Next test

diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -143,3 +143,17 @@
     p1b = lib.stm_pop_root()
     assert p1b == p1    # oalloc() does not use the nursery
     assert count_pages() == 2   # one for p1, one for p2, which have != sizes
+
+def test_keep_global_roots_alive_2():
+    p = oalloc_refs(3)
+    rawsetptr(p, 0, p)
+    rawsetptr(p, 1, ffi.NULL)
+    rawsetptr(p, 2, p)
+    lib.stm_push_root(p)
+    for i in range(3):
+        major_collect()
+        check_not_free(p)
+        assert rawgetptr(p, 0) == p
+        assert rawgetptr(p, 1) == ffi.NULL
+        assert rawgetptr(p, 2) == p
+    lib.stm_pop_root()

From noreply at buildbot.pypy.org  Mon Jun 17 20:26:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 20:26:23 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Port more code, next test passes.
Message-ID: <20130617182623.8DC121C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r181:ad80f196abeb
Date: 2013-06-17 20:26 +0200
http://bitbucket.org/pypy/stmgc/changeset/ad80f196abeb/

Log:	Port more code, next test passes.

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -271,6 +271,74 @@
     }
 }
 
+static void cleanup_for_thread(struct tx_descriptor *d)
+{
+    long i;
+    gcptr *items = d->list_of_read_objects.items;
+
+    if (d->active < 0)
+        return;       /* already "aborted" during forced minor collection */
+
+    for (i = d->list_of_read_objects.size - 1; i >= 0; --i) {
+        gcptr obj = items[i];
+
+        /* Warning: in case the object listed is outdated and has been
+           replaced with a more recent revision, then it might be the
+           case that obj->h_revision doesn't have GCFLAG_VISITED, but
+           just removing it is very wrong --- we want 'd' to abort.
+        */
+        revision_t v = obj->h_revision;
+        if (IS_POINTER(v)) {    /* has a more recent revision.  Oups. */
+            fprintf(stderr,
+                    "ABRT_COLLECT_MAJOR: %p was read but modified already\n",
+                    obj);
+            AbortTransactionAfterCollect(d, ABRT_COLLECT_MAJOR);
+            return;
+        }
+
+        /* on the other hand, if we see a non-visited object in the read
+           list, then we need to remove it --- it's wrong to just abort.
+           Consider the following case: the transaction is inevitable,
+           and since it started, it popped objects out of its shadow
+           stack.  Some popped objects might become free even if they
+           have been read from.  We must not abort such transactions
+           (and cannot anyway: they are inevitable!). */
+        if (!(obj->h_tid & GCFLAG_VISITED)) {
+            items[i] = items[--d->list_of_read_objects.size];
+        }
+    }
+
+    d->num_read_objects_known_old = d->list_of_read_objects.size;
+    fxcache_clear(&d->recent_reads_cache);
+
+    /* We are now after visiting all objects, and we know the
+     * transaction isn't aborting because of this collection.  We have
+     * cleared GCFLAG_PUBLIC_TO_PRIVATE from public objects at the end
+     * of the chain.  Now we have to set it again on public objects that
+     * have a private copy.
+     */
+    wlog_t *item;
+
+    G2L_LOOP_FORWARD(d->public_to_private, item) {
+
+        assert(item->addr->h_tid & GCFLAG_PUBLIC);
+        /* assert(is_private(item->val)); but in the other thread,
+           which becomes: */
+        assert((item->val->h_revision == *d->private_revision_ref) ||
+               (item->val->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
+
+        item->addr->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
+
+    } G2L_LOOP_END;
+}
+
+static void clean_up_lists_of_read_objects_and_fix_outdated_flags(void)
+{
+    struct tx_descriptor *d;
+    for (d = stm_tx_head; d; d = d->tx_next)
+        cleanup_for_thread(d);
+}
+
 
 /***** Major collections: sweeping *****/
 
@@ -434,10 +502,8 @@
 #endif
     mark_all_stack_roots();
     visit_all_objects();
-#if 0
     gcptrlist_delete(&objects_to_trace);
     clean_up_lists_of_read_objects_and_fix_outdated_flags();
-#endif
 
     mc_total_in_use = mc_total_reserved = 0;
     free_all_unused_local_pages();
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -499,17 +499,13 @@
     assert 42 < (p.h_tid & 0xFFFF) < 521
     assert p.h_tid & GCFLAG_PREBUILT_ORIGINAL
 
-def make_global(p1):
-    assert p1.h_revision == lib.get_local_revision()
-    p1.h_revision = (lib.stm_global_cur_time() | 1) - 2
-
-def delegate(p1, p2):
-    assert p1.h_revision != lib.get_local_revision()
-    assert p2.h_revision != lib.get_local_revision()
-    p1.h_revision = ffi.cast("revision_t", p2)
-    p1.h_tid |= GCFLAG_PUBLIC_TO_PRIVATE
-    if p1.h_tid & GCFLAG_PREBUILT_ORIGINAL:
-        lib.stmgcpage_add_prebuilt_root(p1)
+def make_public(p1):
+    """Hack at an object returned by oalloc() to force it public."""
+    assert classify(p1) == "protected"
+    assert p1.h_tid & GCFLAG_OLD
+    p1.h_tid |= GCFLAG_PUBLIC
+    assert classify(p1) == "public"
+    assert (p1.h_tid & GCFLAG_PUBLIC_TO_PRIVATE) == 0
 
 def transaction_break():
     lib.stm_commit_transaction()
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -157,3 +157,28 @@
         assert rawgetptr(p, 1) == ffi.NULL
         assert rawgetptr(p, 2) == p
     lib.stm_pop_root()
+
+def test_local_copy_from_global_obj():
+    p1 = oalloc(HDR); make_public(p1)
+    p2n = lib.stm_write_barrier(p1)
+    assert p2n != p1
+    assert lib.stm_write_barrier(p1) == p2n
+    check_not_free(p1)
+    check_not_free(p2n)
+    assert p1.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
+    lib.stm_push_root(p1)
+    lib.stm_push_root(p2n)
+    minor_collect()    # move p2n out of the nursery
+    p2 = lib.stm_pop_root()
+    assert p2 != p2n
+    assert p1.h_tid & GCFLAG_PUBLIC_TO_PRIVATE
+    assert lib.stm_write_barrier(p1) == p2
+    print p1, p2
+    major_collect()
+    major_collect()
+    p1a = lib.stm_pop_root()
+    assert p1a == p1
+    check_not_free(p1)
+    check_not_free(p2)
+    p3 = lib.stm_write_barrier(p1)
+    assert p3 == p2

From noreply at buildbot.pypy.org  Mon Jun 17 20:38:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 20:38:39 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617183839.7FF191C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r182:75aaf6902952
Date: 2013-06-17 20:38 +0200
http://bitbucket.org/pypy/stmgc/changeset/75aaf6902952/

Log:	Next test

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -160,7 +160,7 @@
     }
 }
 
-static unsigned char random_char = 42;
+static unsigned char random_char = 0x55;
 
 void stmgcpage_free(gcptr obj)
 {
@@ -171,7 +171,8 @@
         assert(0 < size_class && size_class < GC_SMALL_REQUESTS);
 
         /* We simply re-add the object to the right chained list */
-        assert(obj->h_tid = DEBUG_WORD(random_char++));
+        assert(obj->h_tid = DEBUG_WORD(random_char));
+        assert(random_char ^= (0xAA ^ 0x55));
         obj->h_revision = (revision_t)gcp->free_loc_for_size[size_class];
         gcp->free_loc_for_size[size_class] = obj;
         //stm_dbgmem_not_used(obj, size_class * WORD, 0);
@@ -197,19 +198,22 @@
     if (obj->h_tid & GCFLAG_VISITED)
         return;    /* already seen */
 
-    if (obj->h_tid & (GCFLAG_PUBLIC_TO_PRIVATE | GCFLAG_STUB)) {
-        if (obj->h_revision & 1) { // "is not a ptr", so no more recent version
-            obj->h_tid &= ~GCFLAG_PUBLIC_TO_PRIVATE; // see also fix_outdated()
+    if (obj->h_revision & 1) {
+        obj->h_tid &= ~GCFLAG_PUBLIC_TO_PRIVATE;  /* see also fix_outdated() */
+    }
+    else {
+        /* h_revision is a ptr: we have a more recent version */
+        assert(!(obj->h_tid & GCFLAG_STUB));
+        gcptr prev_obj = obj;
+        obj = (gcptr)obj->h_revision;   /* go visit the more recent version */
+
+        if (!(obj->h_tid & GCFLAG_VISITED) && IS_POINTER(obj->h_revision)) {
+            obj = (gcptr)obj->h_revision;
+            prev_obj->h_revision = (revision_t)obj;
         }
-        else {
-            obj = (gcptr)obj->h_revision;   // go visit the more recent version
-            *pobj = obj;
-            goto restart;
-        }
+        *pobj = obj;
+        goto restart;
     }
-    else
-        assert(obj->h_revision & 1);
-
     obj->h_tid |= GCFLAG_VISITED;
     gcptrlist_insert(&objects_to_trace, obj);
 }
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -495,10 +495,28 @@
 def check_inaccessible(p):
     assert not lib._stm_can_access_memory(p)
 
+def DEBUG_WORD(char):
+    return int(ffi.cast("revision_t", char * 0x0101010101010101))
+
+def check_free_old(p):
+    assert not lib._stm_can_access_memory(p) or p.h_tid == DEBUG_WORD(0xDD)
+
+def check_free_explicitly(p):
+    assert not lib._stm_can_access_memory(p) or p.h_tid in (
+        DEBUG_WORD(0x55), DEBUG_WORD(0xAA))
+
 def check_prebuilt(p):
     assert 42 < (p.h_tid & 0xFFFF) < 521
     assert p.h_tid & GCFLAG_PREBUILT_ORIGINAL
 
+def delegate(p1, p2):
+    assert classify(p1) == "public"
+    assert classify(p2) == "public"
+    p1.h_revision = ffi.cast("revision_t", p2)
+    p1.h_tid |= GCFLAG_PUBLIC_TO_PRIVATE
+    if p1.h_tid & GCFLAG_PREBUILT_ORIGINAL:
+        lib.stmgcpage_add_prebuilt_root(p1)
+
 def make_public(p1):
     """Hack at an object returned by oalloc() to force it public."""
     assert classify(p1) == "protected"
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -182,3 +182,20 @@
     check_not_free(p2)
     p3 = lib.stm_write_barrier(p1)
     assert p3 == p2
+
+def test_new_version():
+    p1 = oalloc(HDR); make_public(p1)
+    p2 = oalloc(HDR); make_public(p2)
+    delegate(p1, p2)
+    check_not_free(p1)
+    check_not_free(p2)
+    lib.stm_push_root(p1)
+    major_collect()
+    major_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p2
+    check_free_old(p1)
+    check_not_free(p2)
+    p3 = lib.stm_write_barrier(p2)
+    assert p3 != p2
+    assert p3 == lib.stm_write_barrier(p2)

From noreply at buildbot.pypy.org  Mon Jun 17 20:39:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 20:39:58 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next test
Message-ID: <20130617183958.B1DF51C300C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r183:bef700bb02ad
Date: 2013-06-17 20:39 +0200
http://bitbucket.org/pypy/stmgc/changeset/bef700bb02ad/

Log:	Next test

diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -199,3 +199,25 @@
     p3 = lib.stm_write_barrier(p2)
     assert p3 != p2
     assert p3 == lib.stm_write_barrier(p2)
+
+def test_new_version_kill_intermediate():
+    p1 = oalloc(HDR); make_public(p1)
+    p2 = oalloc(HDR); make_public(p2)
+    p3 = oalloc(HDR); make_public(p3)
+    p4 = oalloc(HDR); make_public(p4)
+    delegate(p1, p2)
+    delegate(p2, p3)
+    delegate(p3, p4)
+    lib.stm_push_root(p2)
+    major_collect()
+    major_collect()
+    p2b = lib.stm_pop_root()
+    assert p2b == p4
+    check_free_old(p1)
+    check_free_old(p2)
+    check_free_old(p3)
+    check_not_free(p4)
+    p5 = lib.stm_write_barrier(p4)
+    assert p5 != p4
+    assert p5 == lib.stm_write_barrier(p4)
+    assert p5 == lib.stm_write_barrier(p5)

From noreply at buildbot.pypy.org  Mon Jun 17 21:50:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Jun 2013 21:50:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix link
Message-ID: <20130617195033.96BD31C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64926:2f71b398f811
Date: 2013-06-17 21:49 +0200
http://bitbucket.org/pypy/pypy/changeset/2f71b398f811/

Log:	Fix link

diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -76,4 +76,4 @@
 The rest of directories serve specific niche goal and are unlikely a good
 entry point.
 
-.. _`introduction to RPython`: getting-started-dev.rst
+.. _`introduction to RPython`: getting-started-dev.html

From noreply at buildbot.pypy.org  Mon Jun 17 23:05:14 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 17 Jun 2013 23:05:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: adapt to the new methods (fix
	translation)
Message-ID: <20130617210515.155CB1C0268@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r64927:e0ba4acfd3c2
Date: 2013-06-17 14:02 -0700
http://bitbucket.org/pypy/pypy/changeset/e0ba4acfd3c2/

Log:	adapt to the new methods (fix translation)

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -12,8 +12,8 @@
 from pypy.module.cpyext.stringobject import PyString_Check
 from pypy.module.sys.interp_encoding import setdefaultencoding
 from pypy.module._codecs.interp_codecs import CodecState
-from pypy.objspace.std import unicodeobject, unicodetype, stringtype
-from rpython.rlib import runicode
+from pypy.objspace.std import unicodeobject, unicodetype
+from rpython.rlib import rstring, runicode
 from rpython.tool.sourcetools import func_renamer
 import sys
 
@@ -684,9 +684,9 @@
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
     if rffi.cast(lltype.Signed, direction) <= 0:
-        return stringtype.stringstartswith(str, substr, start, end)
+        return rstring.startswith(str, substr, start, end)
     else:
-        return stringtype.stringendswith(str, substr, start, end)
+        return rstring.endswith(str, substr, start, end)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_Count(space, w_str, w_substr, start, end):

From noreply at buildbot.pypy.org  Mon Jun 17 23:05:17 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 17 Jun 2013 23:05:17 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130617210517.67D0A1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64928:f648d00a3854
Date: 2013-06-17 14:04 -0700
http://bitbucket.org/pypy/pypy/changeset/f648d00a3854/

Log:	merge default

diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,6 +134,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -168,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -278,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -8,7 +8,12 @@
 class TclError(Exception):
     pass
 
-from .tklib import tklib, tkffi
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
 from .app import TkApp
 
 TK_VERSION = tkffi.string(tklib.get_tk_version())
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -31,13 +31,13 @@
     char *bytes;
     int length;
     Tcl_ObjType *typePtr;
-    union {			/* The internal representation: */
-	long longValue;		/*   - an long integer value. */
-	double doubleValue;	/*   - a double-precision floating value. */
-	struct {		/*   - internal rep as two pointers. */
-	    void *ptr1;
-	    void *ptr2;
-	} twoPtrValue;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
     } internalRep;
     ...;
 } Tcl_Obj;
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -76,4 +76,4 @@
 The rest of directories serve specific niche goal and are unlikely a good
 entry point.
 
-.. _`introduction to RPython`: getting-started-dev.rst
+.. _`introduction to RPython`: getting-started-dev.html
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,3 +52,8 @@
 
 .. branch: ctypes-byref
 Add the '_obj' attribute on ctypes pointer() and byref() objects
+
+.. branch: argsort-segfault
+Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
+
+.. branch: dtype-isnative
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 
 from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr,
     ctypearray, ctypestruct, ctypevoid, ctypeenum)
@@ -115,13 +116,15 @@
 # ____________________________________________________________
 
 SF_MSVC_BITFIELDS = 1
+SF_GCC_ARM_BITFIELDS = 2
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+elif rffi_platform.getdefined('__arm__', ''):
+    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
     DEFAULT_SFLAGS = 0
 
-
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -180,7 +183,7 @@
         # field is an anonymous bitfield
         falign = ftype.alignof()
         do_align = True
-        if fbitsize >= 0:
+        if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
                 # GCC: anonymous bitfields (of any size) don't cause alignment
                 do_align = (fname != '')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2767,10 +2767,10 @@
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
@@ -2783,7 +2783,10 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
@@ -2795,10 +2798,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2807,6 +2815,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -11,8 +11,8 @@
     make_typedescr, get_typedescr)
 from pypy.module.cpyext.bytesobject import PyBytes_Check, PyBytes_FromObject
 from pypy.module._codecs.interp_codecs import CodecState
-from pypy.objspace.std import unicodeobject, unicodetype, stringtype
-from rpython.rlib import runicode, rstring
+from pypy.objspace.std import unicodeobject, unicodetype
+from rpython.rlib import rstring, runicode
 from rpython.tool.sourcetools import func_renamer
 import sys
 
@@ -824,9 +824,9 @@
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
     if rffi.cast(lltype.Signed, direction) <= 0:
-        return stringtype.stringstartswith(str, substr, start, end)
+        return rstring.startswith(str, substr, start, end)
     else:
-        return stringtype.stringendswith(str, substr, start, end)
+        return rstring.endswith(str, substr, start, end)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_Count(space, w_str, w_substr, start, end):
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -20,7 +20,7 @@
 def make_sort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
-    
+
     class Repr(object):
         def __init__(self, index_stride_size, stride_size, size, values,
                      indexes, index_start, start):
@@ -71,11 +71,11 @@
         def __init__(self, index_stride_size, stride_size, size):
             start = 0
             dtype = interp_dtype.get_dtype_cache(space).w_longdtype
-            self.indexes = dtype.itemtype.malloc(size*dtype.get_size())
-            self.values = alloc_raw_storage(size * stride_size, 
+            indexes = dtype.itemtype.malloc(size*dtype.get_size())
+            values = alloc_raw_storage(size * stride_size,
                                             track_allocation=False)
-            Repr.__init__(self, index_stride_size, stride_size, 
-                          size, self.values, self.indexes, start, start)
+            Repr.__init__(self, dtype.get_size(), stride_size,
+                          size, values, indexes, start, start)
 
         def __del__(self):
             free_raw_storage(self.indexes, track_allocation=False)
@@ -96,7 +96,7 @@
         for i in range(stop-start):
             retval.setitem(i, lst.getitem(i+start))
         return retval
-    
+
     if count < 2:
         def arg_lt(a, b):
             # Does numpy do <= ?
@@ -108,7 +108,7 @@
                     return True
                 elif a[0][i] > b[0][i]:
                     return False
-            # Does numpy do True?    
+            # Does numpy do True?
             return False
 
     ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
@@ -180,7 +180,7 @@
 
 class SortCache(object):
     built = False
-    
+
     def __init__(self, space):
         if self.built:
             return
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -115,9 +115,27 @@
             return space.wrap('=')
         return space.wrap(nonnative_byteorder_prefix)
 
+    def descr_get_str(self, space):
+        size = self.get_size()
+        basic = self.kind
+        if basic == UNICODELTR:
+            size >>= 2
+            endian = byteorder_prefix
+        elif size <= 1:
+            endian = '|'  # ignore
+        elif self.native:
+            endian = byteorder_prefix
+        else:
+            endian = nonnative_byteorder_prefix
+
+        return space.wrap("%s%s%s" % (endian, basic, size))
+
     def descr_get_alignment(self, space):
         return space.wrap(self.itemtype.alignment)
 
+    def descr_get_isnative(self, space):
+        return space.wrap(self.native)
+
     def descr_get_base(self, space):
         return space.wrap(self.base)
 
@@ -421,8 +439,10 @@
     char = interp_attrproperty("char", cls=W_Dtype),
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
     byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
+    str = GetSetProperty(W_Dtype.descr_get_str),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
     alignment = GetSetProperty(W_Dtype.descr_get_alignment),
+    isnative = GetSetProperty(W_Dtype.descr_get_isnative),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
     name = interp_attrproperty('name', cls=W_Dtype),
     fields = GetSetProperty(W_Dtype.descr_get_fields),
@@ -582,7 +602,7 @@
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
                                    ],
-            aliases=["float"],
+            aliases=["float", "double"],
         )
         self.w_complex64dtype = W_ComplexDtype(
             types.Complex64(),
@@ -663,9 +683,10 @@
             char='S',
             w_box_type = space.gettypefor(interp_boxes.W_StringBox),
             alternate_constructors=[space.w_str],
+            aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
-            types.UnicodeType(1),
+            types.UnicodeType(0),
             num=19,
             kind=UNICODELTR,
             name='unicode',
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -418,8 +418,16 @@
         addr = self.implementation.get_storage_as_int(space)
         # will explode if it can't
         w_d = space.newdict()
-        space.setitem_str(w_d, 'data', space.newtuple([space.wrap(addr),
-                                                       space.w_False]))
+        space.setitem_str(w_d, 'data',
+                          space.newtuple([space.wrap(addr), space.w_False]))
+        space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
+        space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
+        if self.implementation.order == 'C':
+            # Array is contiguous, no strides in the interface.
+            strides = space.w_None
+        else:
+            strides = self.descr_get_strides(space)
+        space.setitem_str(w_d, 'strides', strides)
         return w_d
 
     w_pypy_data = None
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -682,6 +682,20 @@
         assert dtype('=i8').byteorder == '='
         assert dtype(byteorder + 'i8').byteorder == '='
 
+    def test_dtype_str(self):
+        from numpypy import dtype
+        byteorder = self.native_prefix
+        assert dtype('i8').str == byteorder + 'i8'
+        assert dtype('<i8').str == '<i8'
+        assert dtype('>i8').str == '>i8'
+        assert dtype('int8').str == '|i1'
+        assert dtype('float').str == byteorder + 'f8'
+        # strange
+        assert dtype('string').str == '|S0'
+        assert dtype('unicode').str == byteorder + 'U0'
+        # assert dtype(('string', 7)).str == '|S7'
+        # assert dtype(('unicode', 7)).str == '<U7'
+
     def test_intp(self):
         from numpypy import dtype
         assert dtype('p') == dtype('intp')
@@ -691,6 +705,11 @@
         from numpypy import dtype
         assert dtype('i4').alignment == 4
 
+    def test_isnative(self):
+        from numpypy import dtype
+        assert dtype('i4').isnative == True
+        assert dtype('>i8').isnative == False
+
     def test_any_all(self):
         import numpypy as numpy
         x = numpy.bool_(True)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2138,6 +2138,9 @@
         a = array([1, 2, 3])
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
+        assert i['shape'] == (3,)
+        assert i['strides'] == None  # Because array is in C order
+        assert i['typestr'] == a.dtype.str
         a = a[::2]
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
@@ -2496,6 +2499,13 @@
         b = a.argsort()
         assert (b[:3] == [0, 100, 200]).all()
 
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
     def test_argsort_axis(self):
         from numpypy import array
         a = array([[4, 2], [1, 3]])
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -1,5 +1,6 @@
 # Generated by pypy/tool/import_cffi.py
-import py, sys
+import py, sys, platform
+import pytest
 from pypy.module.test_lib_pypy.cffi_tests import backend_tests, test_function, test_ownlib
 from cffi import FFI
 import _cffi_backend
@@ -122,6 +123,7 @@
         self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8)
         self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4)
 
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_anonymous_no_align(self):
         L = FFI().alignof("long long")
         self.check("char y; int :1;", 0, 1, 2)
@@ -134,6 +136,20 @@
         self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
         self.check("char x; long long  :57; char y;", L + 8, 1, L + 9)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_anonymous_align_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :1;", 0, 4, 4)
+        self.check("char x; int z:1; char y;", 2, 4, 4)
+        self.check("char x; int  :1; char y;", 2, 4, 4)
+        self.check("char x; long long z:48; char y;", 7, L, 8)
+        self.check("char x; long long  :48; char y;", 7, 8, 8)
+        self.check("char x; long long z:56; char y;", 8, L, 8 + L)
+        self.check("char x; long long  :56; char y;", 8, L, 8 + L)
+        self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
+        self.check("char x; long long  :57; char y;", L + 8, L, L + 8 + L)
+
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_zero(self):
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
@@ -144,6 +160,17 @@
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
         self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_zero_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :0;", 0, 4, 4)
+        self.check("char x; int :0; char y;", 4, 4, 8)
+        self.check("char x; int :0; int :0; char y;", 4, 4, 8)
+        self.check("char x; long long :0; char y;", L, 8, L + 8)
+        self.check("short x, y; int :0; int :0;", 2, 4, 4)
+        self.check("char x; int :0; short b:1; char y;", 5, 4, 8)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
+
     def test_error_cases(self):
         ffi = FFI()
         py.test.raises(TypeError,
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -675,6 +675,7 @@
             raise OperationError(space.w_ValueError,
                                  space.wrap("list modified during sort"))
 
+find_jmp = jit.JitDriver(greens = [], reds = 'auto', name = 'list.find')
 
 class ListStrategy(object):
     sizehint = -1
@@ -699,6 +700,7 @@
         i = start
         # needs to be safe against eq_w mutating stuff
         while i < stop and i < w_list.length():
+            find_jmp.jit_merge_point()
             if space.eq_w(w_list.getitem(i), w_item):
                 return i
             i += 1
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -1,3 +1,5 @@
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
 import py
 from pypy.interpreter import gateway, pycode
 from pypy.interpreter.error import OperationError
@@ -35,8 +37,13 @@
             return None
     fullsource = property(fullsource, None, None, "Full source of AppCode")
 
-    def getargs(self):
-        return self.raw.co_varnames[:self.raw.co_argcount]
+    def getargs(self, var=False):
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
 
 class AppFrame(py.code.Frame):
 
@@ -70,10 +77,10 @@
     def is_true(self, w_value):
         return self.space.is_true(w_value)
 
-    def getargs(self):
+    def getargs(self, var=False):
         space = self.space
         retval = []
-        for arg in self.code.getargs():
+        for arg in self.code.getargs(var):
             w_val = space.finditem(self.w_locals, space.wrap(arg))
             if w_val is None:
                 w_val = space.wrap('<no value found>')
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -1,13 +1,12 @@
+import sys
+
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.platform import platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.unroll import unrolling_iterable
 
-import sys, os
 
-link_files = []
-include_dirs = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32', 'zlib1',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -24,7 +23,7 @@
     includes = []
 
 includes += [
-    'openssl/ssl.h', 
+    'openssl/ssl.h',
     'openssl/err.h',
     'openssl/rand.h',
     'openssl/evp.h',
@@ -33,16 +32,14 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
-    link_files = link_files,
     includes = includes,
-    include_dirs = include_dirs,
     export_symbols = [],
     post_include_bits = [
         # Unnamed structures are not supported by rffi_platform.
         # So we replace an attribute access with a macro call.
         '#define pypy_GENERAL_NAME_dirn(name) (name->d.dirn)',
-        ],
-    )
+    ],
+)
 
 eci = rffi_platform.configure_external_library(
     'openssl', eci,
@@ -114,7 +111,7 @@
     X509_extension_st = rffi_platform.Struct(
         'struct X509_extension_st',
         [('value', ASN1_STRING)])
-    X509V3_EXT_D2I = lltype.FuncType([rffi.VOIDP, rffi.CCHARPP, rffi.LONG], 
+    X509V3_EXT_D2I = lltype.FuncType([rffi.VOIDP, rffi.CCHARPP, rffi.LONG],
                                      rffi.VOIDP)
     v3_ext_method = rffi_platform.Struct(
         'struct v3_ext_method',
@@ -137,7 +134,7 @@
         'OBJ_NAME',
         [('alias', rffi.INT),
          ('name', rffi.CCHARP),
-         ]) 
+         ])
 
 
 for k, v in rffi_platform.configure(CConfig).items():
@@ -259,11 +256,11 @@
 ssl_external('ASN1_STRING_to_UTF8', [rffi.CCHARPP, ASN1_STRING], rffi.INT)
 ssl_external('ASN1_TIME_print', [BIO, ASN1_TIME], rffi.INT)
 ssl_external('i2a_ASN1_INTEGER', [BIO, ASN1_INTEGER], rffi.INT)
-ssl_external('ASN1_item_d2i', 
+ssl_external('ASN1_item_d2i',
              [rffi.VOIDP, rffi.CCHARPP, rffi.LONG, ASN1_ITEM], rffi.VOIDP)
-if OPENSSL_EXPORT_VAR_AS_FUNCTION:             
+if OPENSSL_EXPORT_VAR_AS_FUNCTION:
     ssl_external('ASN1_ITEM_ptr', [ASN1_ITEM_EXP], ASN1_ITEM, macro=True)
-else:    
+else:
     ssl_external('ASN1_ITEM_ptr', [rffi.VOIDP], ASN1_ITEM, macro=True)
 
 ssl_external('sk_GENERAL_NAME_num', [GENERAL_NAMES], rffi.INT,

From noreply at buildbot.pypy.org  Mon Jun 17 23:07:29 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 17 Jun 2013 23:07:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: apply 26a150624b95 from default
Message-ID: <20130617210729.C7C8A1C0268@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64929:475d275aa4f8
Date: 2013-06-17 14:06 -0700
http://bitbucket.org/pypy/pypy/changeset/475d275aa4f8/

Log:	apply 26a150624b95 from default

diff --git a/lib-python/3/logging/__init__.py b/lib-python/3/logging/__init__.py
--- a/lib-python/3/logging/__init__.py
+++ b/lib-python/3/logging/__init__.py
@@ -129,6 +129,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -161,7 +166,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/3/test/test_logging.py b/lib-python/3/test/test_logging.py
--- a/lib-python/3/test/test_logging.py
+++ b/lib-python/3/test/test_logging.py
@@ -299,6 +299,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""

From noreply at buildbot.pypy.org  Tue Jun 18 01:46:25 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue, 18 Jun 2013 01:46:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: cpython returns a formal iter here
Message-ID: <20130617234625.7CC1A1C1001@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64931:f187810ad0ec
Date: 2013-06-17 16:44 -0700
http://bitbucket.org/pypy/pypy/changeset/f187810ad0ec/

Log:	cpython returns a formal iter here

diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -431,5 +431,6 @@
 
            This API is used by pickle.py and copy.py.
         """
-        return (type(self), (self.default_factory,), None, None, self.items())
+        return (type(self), (self.default_factory,), None, None,
+                iter(self.items()))
 
diff --git a/pypy/module/_collections/app_defaultdict.py b/pypy/module/_collections/app_defaultdict.py
--- a/pypy/module/_collections/app_defaultdict.py
+++ b/pypy/module/_collections/app_defaultdict.py
@@ -63,4 +63,5 @@
 
            This API is used by pickle.py and copy.py.
         """
-        return (type(self), (self._default_factory,), None, None, self.items())
+        return (type(self), (self._default_factory,), None, None,
+                iter(self.items()))
diff --git a/pypy/module/_collections/test/test_defaultdict.py b/pypy/module/_collections/test/test_defaultdict.py
--- a/pypy/module/_collections/test/test_defaultdict.py
+++ b/pypy/module/_collections/test/test_defaultdict.py
@@ -56,3 +56,9 @@
         assert len(d2) == 1
         assert d2[2] == 3
         assert d2[3] == 42
+
+    def test_reduce(self):
+        import _collections
+        d = _collections.defaultdict(None, {3: 4})
+        dict_iter = d.__reduce__()[4]
+        assert type(dict_iter) is type(iter(d.items()))

From noreply at buildbot.pypy.org  Tue Jun 18 01:46:24 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue, 18 Jun 2013 01:46:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20130617234624.2C5051C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64930:10bf1805e3e8
Date: 2013-06-17 15:34 -0700
http://bitbucket.org/pypy/pypy/changeset/10bf1805e3e8/

Log:	2to3

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -626,9 +626,10 @@
         assert 'settrace' in dir(sys)
 
     def test_reload_builtin_doesnt_clear(self):
+        import imp
         import sys
         sys.foobar = "baz"
-        reload(sys)
+        imp.reload(sys)
         assert sys.foobar == "baz"
 
     def test_reimport_builtin_simple_case_1(self):
@@ -648,7 +649,7 @@
 
     def test_reimport_builtin(self):
         skip("fix me")
-        import sys, time
+        import imp, sys, time
         oldpath = sys.path
         time.tzset = "<test_reimport_builtin removed this>"
 
@@ -658,7 +659,7 @@
 
         assert time.tzset == "<test_reimport_builtin removed this>"
 
-        reload(time1)   # don't leave a broken time.tzset behind
+        imp.reload(time1)   # don't leave a broken time.tzset behind
         import time
         assert time.tzset != "<test_reimport_builtin removed this>"
 

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:05 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:05 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added interpreter proxy minor
	3 stubs
Message-ID: <20130618140405.953911C1001@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r448:ce8d75cdb2fc
Date: 2013-06-11 11:43 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/ce8d75cdb2fc/

Log:	added interpreter proxy minor 3 stubs

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -660,10 +660,15 @@
 # #endif
 
 # #if VM_PROXY_MINOR > 3
+ at expose_on_virtual_machine_proxy([str, str], bool, minor=3)
+def ioLoadFunctionFrom(fnName, modName):
+    print 'Called InterpreterProxy >> ioLoadFunctionFrom'
+    raise ProxyFunctionFailed
 
-#     void *(*ioLoadFunctionFrom)(char *fnName, char *modName);
-#     sqInt (*ioMicroMSecs)(void);
-
+ at expose_on_virtual_machine_proxy([], bool, minor=3)
+def ioMicroMSecs():
+    print 'Called InterpreterProxy >> ioMicroMSecs'
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 4

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:06 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:06 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added interpreter proxy minor
	5 & 6 stubs
Message-ID: <20130618140406.B1C7D1C300C@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r449:c4ab2046ee2a
Date: 2013-06-11 11:44 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/c4ab2046ee2a/

Log:	added interpreter proxy minor 5 & 6 stubs

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -705,15 +705,44 @@
 # #endif
 
 # #if VM_PROXY_MINOR > 5
-#     sqInt (*isArray)(sqInt oop);
-#     sqInt (*forceInterruptCheck)(void);
+ at expose_on_virtual_machine_proxy([oop], bool, minor=5)
+def isArray(w_object):
+    if not isinstance(w_object, model.W_PointersObject):
+        return False
+    space = IProxy.space
+    s_class = w_object.shadow_of_my_class(space)
+    if s_class.instsize() == 0 and s_class.isvariable():
+        return True
+    else:
+        return False
+
+ at expose_on_virtual_machine_proxy([], int, minor=5)
+def forceInterruptCheck():
+    print 'Called InterpreterProxy >> forceInterruptCheck'
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 6
-#     sqInt  (*fetchLong32ofObject)(sqInt fieldFieldIndex, sqInt oop);
-#     sqInt  (*getThisSessionID)(void);
-#     sqInt     (*ioFilenamefromStringofLengthresolveAliases)(char* aCharBuffer, char* filenameIndex, sqInt filenameLength, sqInt resolveFlag);
-#     sqInt  (*vmEndianness)(void);
+ at expose_on_virtual_machine_proxy([int, oop], oop, minor=6)
+def fetchLong32ofObject(fieldFieldIndex, oop):
+    print 'Called InterpreterProxy >> fetchLong32ofObject'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, oop], int, minor=6)
+def getThisSessionID(fieldFieldIndex, oop):
+    # return random int
+    print 'Called InterpreterProxy >> getThisSessionID'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([str, str, int, int], oop, minor=6)
+def ioFilenamefromStringofLengthresolveAliases(aCharBuffer, filenameIndex, filenameLength, resolveFlag):
+    print 'Called InterpreterProxy >> ioFilenamefromStringofLengthresolveAliases'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], int, minor=6)
+def vmEndianness():
+    print 'Called InterpreterProxy >> vmEndianness'
+    return 0 # return 1 for big endian
 # #endif
 
 # #if VM_PROXY_MINOR > 7

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:04 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:04 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added long and interpreter
	proxy minor 4 stubs
Message-ID: <20130618140404.7CA941C094A@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r447:679647db7890
Date: 2013-06-11 11:43 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/679647db7890/

Log:	added long and interpreter proxy minor 4 stubs

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -34,7 +34,8 @@
     pass
 
 def expose_on_virtual_machine_proxy(unwrap_spec, result_type, minor=0, major=1):
-    mapping = {oop: sqInt, int: sqInt, list: sqIntArrayPtr, bool: sqInt, float: sqDouble, str: sqStr}
+    mapping = {oop: sqInt, int: sqInt, list: sqIntArrayPtr, bool: sqInt,
+                float: sqDouble, str: sqStr, long: sqLong}
     f_ptr = Ptr(FuncType([mapping[spec] for spec in unwrap_spec], mapping[result_type]))
     if minor < minor:
         minor = minor
@@ -75,6 +76,9 @@
                     return 0.0
                 elif mapping[result_type] is sqIntArrayPtr:
                     return rffi.cast(sqIntArrayPtr, 0)
+                elif mapping[result_type] is sqLong:
+                    # XXX: how to return a long 0?
+                    return 0
                 else:
                     raise NotImplementedError(
                         "InterpreterProxy: unknown result_type %s" % (result_type, ))
@@ -674,11 +678,25 @@
 # #   endif
 # #  endif
 
-#     sqInt  (*positive64BitIntegerFor)(sqLong integerValue);
-#     sqLong (*positive64BitValueOf)(sqInt oop);
-#     sqInt  (*signed64BitIntegerFor)(sqLong integerValue);
-#     sqLong (*signed64BitValueOf)(sqInt oop);
+ at expose_on_virtual_machine_proxy([long], oop, minor=4)
+def positive64BitIntegerFor(integerValue):
+    print 'Called InterpreterProxy >> positive64BitIntegerFor'
+    raise ProxyFunctionFailed
 
+ at expose_on_virtual_machine_proxy([oop], long, minor=4)
+def positive64BitValueOf(w_number):
+    print 'Called InterpreterProxy >> positive64BitValueOf'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([long], oop, minor=4)
+def signed64BitIntegerFor(integerValue):
+    print 'Called InterpreterProxy >> signed64BitIntegerFor'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], long, minor=4)
+def signed64BitValueOf(w_number):
+    print 'Called InterpreterProxy >> signed64BitValueOf'
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 5

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:03 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:03 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added interpreter-proxy minor
	2 stubs
Message-ID: <20130618140403.48A171C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r446:e64194e6ea6c
Date: 2013-06-11 11:43 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/e64194e6ea6c/

Log:	added interpreter-proxy minor 2 stubs

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -459,7 +459,6 @@
 def classString():
     return IProxy.space.w_String
 
-
 #     /* InterpreterProxy methodsFor: 'instance creation' */
 
 @expose_on_virtual_machine_proxy([oop], oop)
@@ -558,7 +557,6 @@
 
 #     /* InterpreterProxy methodsFor: 'compiler' */
 
-
 @expose_on_virtual_machine_proxy([], int)
 def compilerHookVector():
     print 'Called InterpreterProxy >> compilerHookVector'
@@ -590,24 +588,71 @@
     raise ProxyFunctionFailed
 
 # #if VM_PROXY_MINOR > 2
+ at expose_on_virtual_machine_proxy([], oop, minor=2)
+def classLargeNegativeInteger():
+    print 'Called InterpreterProxy >> classLargeNegativeInteger'
+    raise ProxyFunctionFailed
 
-#     sqInt (*classLargeNegativeInteger)(void);
-#     sqInt (*signed32BitIntegerFor)(sqInt integerValue);
-#     sqInt (*signed32BitValueOf)(sqInt oop);
-#     sqInt (*includesBehaviorThatOf)(sqInt aClass, sqInt aSuperClass);
-#     sqInt (*primitiveMethod)(void);
+ at expose_on_virtual_machine_proxy([int], oop, minor=2)
+def signed32BitIntegerFor(n):
+    print 'Called InterpreterProxy >> signed32BitIntegerFor'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], int, minor=2)
+def signed32BitValueOf(w_number):
+    print 'Called InterpreterProxy >> signed32BitValueOf'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop, oop], int, minor=2)
+def includesBehaviorThatOf(w_class, w_superclass):
+    print 'Called InterpreterProxy >> includesBehaviorThatOf'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=2)
+def primitiveMethod():
+    return IProxy.s_method.w_self()
 
 #     /* InterpreterProxy methodsFor: 'FFI support' */
 
-#     sqInt (*classExternalAddress)(void);
-#     sqInt (*classExternalData)(void);
-#     sqInt (*classExternalFunction)(void);
-#     sqInt (*classExternalLibrary)(void);
-#     sqInt (*classExternalStructure)(void);
-#     sqInt (*ioLoadModuleOfLength)(sqInt modIndex, sqInt modLength);
-#     sqInt (*ioLoadSymbolOfLengthFromModule)(sqInt fnIndex, sqInt fnLength, sqInt handle);
-#     sqInt (*isInMemory)(sqInt address);
+ at expose_on_virtual_machine_proxy([], oop, minor=2)
+def classExternalAddress():
+    print 'Called InterpreterProxy >> classExternalAddress'
+    raise ProxyFunctionFailed
 
+ at expose_on_virtual_machine_proxy([], oop, minor=2)
+def classExternalData():
+    print 'Called InterpreterProxy >> classExternalData'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=2)
+def classExternalFunction():
+    print 'Called InterpreterProxy >> classExternalFunction'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=2)
+def classExternalLibrary():
+    print 'Called InterpreterProxy >> classExternalLibrary'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=2)
+def classExternalStructure():
+    print 'Called InterpreterProxy >> classExternalStructure'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, int], oop, minor=2)
+def ioLoadModuleOfLength(modIndex, modLength):
+    print 'Called InterpreterProxy >> ioLoadModuleOfLength'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, int, int], int, minor=2)
+def ioLoadSymbolOfLengthFromModule(fnIndex, fnLength, handle):
+    print 'Called InterpreterProxy >> ioLoadSymbolOfLengthFromModule'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], bool, minor=2)
+def isInMemory(address):
+    print 'Called InterpreterProxy >> isInMemory'
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 3

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:08 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:08 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added interpreter proxy stubs
	for minor 7
Message-ID: <20130618140408.9E62F1C300E@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r450:38261a187782
Date: 2013-06-11 11:45 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/38261a187782/

Log:	added interpreter proxy stubs for minor 7

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -754,12 +754,20 @@
 #                    used in callbackLeave
 #      Returns: True if successful, false otherwise */
 #   sqInt (*callbackEnter)(sqInt *callbackID);
+ at expose_on_virtual_machine_proxy([int], bool, minor=7)
+def callbackEnter(callbackID):
+    print 'Called InterpreterProxy >> callbackEnter'
+    raise ProxyFunctionFailed
 
 #   /* callbackLeave: Leave the interpreter from a previous callback
 #      Arguments:
 #        callbackID: The ID of the callback received from callbackEnter()
 #      Returns: True if succcessful, false otherwise. */
 #   sqInt (*callbackLeave)(sqInt  callbackID);
+ at expose_on_virtual_machine_proxy([int], bool, minor=7)
+def callbackLeave(callbackID):
+    print 'Called InterpreterProxy >> callbackLeave'
+    raise ProxyFunctionFailed
 
 #   /* addGCRoot: Add a variable location to the garbage collector.
 #      The contents of the variable location will be updated accordingly.
@@ -767,6 +775,10 @@
 #        varLoc: Pointer to the variable location
 #      Returns: True if successful, false otherwise. */
 #   sqInt (*addGCRoot)(sqInt *varLoc);
+ at expose_on_virtual_machine_proxy([oop], bool, minor=7)
+def addGCRoot(callbackID):
+    print 'Called InterpreterProxy >> addGCRoot'
+    raise ProxyFunctionFailed
 
 #   /* removeGCRoot: Remove a variable location from the garbage collector.
 #      Arguments:
@@ -774,6 +786,10 @@
 #      Returns: True if successful, false otherwise.
 #   */
 #   sqInt (*removeGCRoot)(sqInt *varLoc);
+ at expose_on_virtual_machine_proxy([oop], bool, minor=7)
+def removeGCRoot(callbackID):
+    print 'Called InterpreterProxy >> removeGCRoot'
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 8

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:09 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:09 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added the last stubs of
	InterpreterProxy
Message-ID: <20130618140409.B2B331C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r451:a218c2dfae8f
Date: 2013-06-14 12:59 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/a218c2dfae8f/

Log:	added the last stubs of InterpreterProxy

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -24,6 +24,7 @@
 sqDouble = rffi.DOUBLE
 sqIntArrayPtr = Ptr(rffi.CArray(sqInt))
 sqStr = rffi.CCHARP
+void = lltype.Void
 
 major = minor = 0
 functions = []
@@ -35,7 +36,7 @@
 
 def expose_on_virtual_machine_proxy(unwrap_spec, result_type, minor=0, major=1):
     mapping = {oop: sqInt, int: sqInt, list: sqIntArrayPtr, bool: sqInt,
-                float: sqDouble, str: sqStr, long: sqLong}
+                float: sqDouble, str: sqStr, long: sqLong, void: void}
     f_ptr = Ptr(FuncType([mapping[spec] for spec in unwrap_spec], mapping[result_type]))
     if minor < minor:
         minor = minor
@@ -79,12 +80,14 @@
                 elif mapping[result_type] is sqLong:
                     # XXX: how to return a long 0?
                     return 0
+                elif mapping[result_type] is sqStr:
+                    return rffi.cast(sqStr, "")
                 else:
                     raise NotImplementedError(
                         "InterpreterProxy: unknown result_type %s" % (result_type, ))
         wrapped.func_name = "wrapped_ipf_" + func.func_name
         functions.append(("c_" + func.func_name, f_ptr, wrapped))
-        return wrapped
+        return func
     return decorator
 
 @expose_on_virtual_machine_proxy([], int)
@@ -167,8 +170,8 @@
 
 @expose_on_virtual_machine_proxy([oop], list)
 def arrayValueOf(w_array):
-    if w_array.is_array_object():
-        raise NotImplementedError
+    # if w_array.is_array_object():
+    #     return w_array.as_c_array(IProxy)
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], int)
@@ -186,8 +189,8 @@
     # arrayOop := self fetchPointer: fieldIndex ofObject: objectPointer.
     # ^ self arrayValueOf: arrayOop
     w_array = w_object.fetch(IProxy.space, fieldIndex)
-    if w_array.is_array_object():
-        raise NotImplementedError
+    # if w_array.is_array_object():
+    #     return w_array.as_c_array(IProxy)
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], oop)
@@ -793,55 +796,200 @@
 # #endif
 
 # #if VM_PROXY_MINOR > 8
-#     /* See interp.h and above for standard error codes. */
-#     sqInt  (*primitiveFailFor)(sqInt code);
-#     void (*(*setInterruptCheckChain)(void (*aFunction)(void)))();
-#     sqInt  (*classAlien)(void);
-#     sqInt  (*classUnsafeAlien)(void);
-#     sqInt  (*sendInvokeCallbackStackRegistersJmpbuf)(sqInt thunkPtrAsInt, sqInt stackPtrAsInt, sqInt regsPtrAsInt, sqInt jmpBufPtrAsInt);
-#     sqInt  (*reestablishContextPriorToCallback)(sqInt callbackContext);
-#     sqInt *(*getStackPointer)(void);
-#     sqInt  (*isOopImmutable)(sqInt oop);
-#     sqInt  (*isOopMutable)(sqInt oop);
+ at expose_on_virtual_machine_proxy([int], bool, minor=8)
+def primitiveFailFor(code):
+    if code > 0:
+        IProxy.failed(code)
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], void, minor=8)
+def setInterruptCheckChain():
+    print 'Called InterpreterProxy >> setInterruptCheckChain'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=8)
+def classAlien():
+    print 'Called InterpreterProxy >> classAlien'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=8)
+def classUnsafeAlien():
+    print 'Called InterpreterProxy >> classUnsafeAlien'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, int, int, int], int, minor=8)
+def sendInvokeCallbackStackRegistersJmpbuf(thunkPtrAsInt, stackPtrAsInt, regsPtrAsInt, jmpBufPtrAsInt):
+    print 'Called InterpreterProxy >> sendInvokeCallbackStackRegistersJmpbuf'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], int, minor=8)
+def reestablishContextPriorToCallback(callbackContext):
+    print 'Called InterpreterProxy >> reestablishContextPriorToCallback'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], int, minor=8)
+def getStackPointer():
+    print 'Called InterpreterProxy >> getStackPointer'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], bool, minor=8)
+def isOopImmutable(w_object):
+    print 'Called InterpreterProxy >> isOopImmutable'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], bool, minor=8)
+def isOopMutable(w_object):
+    print 'Called InterpreterProxy >> isOopMutable'
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 9
-#   sqInt  (*methodArg)  (sqInt index);
-#   sqInt  (*objectArg)  (sqInt index);
-#   sqInt  (*integerArg) (sqInt index);
-#   double (*floatArg)   (sqInt index);
-#   sqInt  (*methodReturnValue) (sqInt oop);
-#   sqInt  (*topRemappableOop)  (void);
+ at expose_on_virtual_machine_proxy([int], oop, minor=9)
+def methodArg(n):
+    print 'Called InterpreterProxy >> methodArg'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], oop, minor=9)
+def objectArg(n):
+    print 'Called InterpreterProxy >> objectArg'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], int, minor=9)
+def integerArg(n):
+    print 'Called InterpreterProxy >> integerArg'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], float, minor=9)
+def floatArg(n):
+    print 'Called InterpreterProxy >> floatArg'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop], int, minor=9)
+def methodReturnValue(w_object):
+    print 'Called InterpreterProxy >> methodReturnValue'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=9)
+def topRemappableOop():
+    print 'Called InterpreterProxy >> topRemappableOop'
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 10
 # # define DisownVMLockOutFullGC 1
-#   sqInt (*disownVM)(sqInt flags);
-#   sqInt (*ownVM)   (sqInt threadIdAndFlags);
+ at expose_on_virtual_machine_proxy([int], int, minor=10)
+def disownVM(flags):
+    print 'Called InterpreterProxy >> disownVM'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], int, minor=10)
+def ownVM(threadIdAndFlags):
+    print 'Called InterpreterProxy >> ownVM'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, int], int, minor=10)
+def addHighPriorityTickee(ticker, periodms):
+    print 'Called InterpreterProxy >> addHighPriorityTickee'
+    raise ProxyFunctionFailed
 #   void  (*addHighPriorityTickee)(void (*ticker)(void), unsigned periodms);
+ at expose_on_virtual_machine_proxy([int, int, int], int, minor=10)
+def addSynchronousTickee(ticker, periodms, roundms):
+    print 'Called InterpreterProxy >> addSynchronousTickee'
+    raise ProxyFunctionFailed
 #   void  (*addSynchronousTickee)(void (*ticker)(void), unsigned periodms, unsigned roundms);
-#   usqLong (*utcMicroseconds)(void);
+ at expose_on_virtual_machine_proxy([], long, minor=10)
+def utcMicroseconds():
+    print 'Called InterpreterProxy >> utcMicroseconds'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], int, minor=10)
+def tenuringIncrementalGC():
+    print 'Called InterpreterProxy >> tenuringIncrementalGC'
+    raise ProxyFunctionFailed
 #   sqInt (*tenuringIncrementalGC)(void);
-#   sqInt (*isYoung) (sqInt anOop);
-#   sqInt (*isKindOfClass)(sqInt oop, sqInt aClass);
+ at expose_on_virtual_machine_proxy([oop], bool, minor=10)
+def isYoung(w_object):
+    print 'Called InterpreterProxy >> isYoung'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([oop, oop], bool, minor=10)
+def isKindOfClass(w_object, w_class):
+    print 'Called InterpreterProxy >> isKindOfClass'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], oop, minor=10)
+def primitiveErrorTable():
+    print 'Called InterpreterProxy >> primitiveErrorTable'
+    raise ProxyFunctionFailed
 #   sqInt (*primitiveErrorTable)(void);
-#   sqInt (*primitiveFailureCode)(void);
-#   sqInt (*instanceSizeOf)(sqInt aClass);
+ at expose_on_virtual_machine_proxy([], int, minor=10)
+def primitiveFailureCode():
+    return IProxy.fail_reason
+
+ at expose_on_virtual_machine_proxy([oop], int, minor=10)
+def instanceSizeOf(w_class):
+    if isinstance(w_class, model.W_PointersObject):
+        s_class = w_class.as_class_get_shadow(IProxy.space)
+        return s_class.instsize()
+    raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 11
 # /* VMCallbackContext opaque type avoids all including setjmp.h & vmCallback.h */
-#   sqInt (*sendInvokeCallbackContext)(vmccp);
+ at expose_on_virtual_machine_proxy([int], int, minor=11)
+def sendInvokeCallbackContext(vmccp):
+    print 'Called InterpreterProxy >> sendInvokeCallbackContext'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int, int, int], int, minor=11)
+def returnAsThroughCallbackContext(n, vmccp, m):
+    print 'Called InterpreterProxy >> returnAsThroughCallbackContext'
+    raise ProxyFunctionFailed
 #   sqInt (*returnAsThroughCallbackContext)(int, vmccp, sqInt);
-#   long  (*signedMachineIntegerValueOf)(sqInt);
-#   long  (*stackSignedMachineIntegerValue)(sqInt);
-#   unsigned long  (*positiveMachineIntegerValueOf)(sqInt);
-#   unsigned long  (*stackPositiveMachineIntegerValue)(sqInt);
-#   sqInt  (*getInterruptPending)(void);
-#   char  *(*cStringOrNullFor)(sqInt);
+ at expose_on_virtual_machine_proxy([int], long, minor=11)
+def signedMachineIntegerValueOf(n):
+    print 'Called InterpreterProxy >> signedMachineIntegerValueOf'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], long, minor=11)
+def stackSignedMachineIntegerValue(n):
+    print 'Called InterpreterProxy >> stackSignedMachineIntegerValue'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], long, minor=11)
+def positiveMachineIntegerValueOf(n):
+    print 'Called InterpreterProxy >> positiveMachineIntegerValueOf'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], long, minor=11)
+def stackPositiveMachineIntegerValue(n):
+    print 'Called InterpreterProxy >> stackPositiveMachineIntegerValue'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([], int, minor=11)
+def getInterruptPending():
+    print 'Called InterpreterProxy >> getInterruptPending'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], str, minor=11)
+def cStringOrNullFor(n):
+    print 'Called InterpreterProxy >> cStringOrNullFor'
+    raise ProxyFunctionFailed
+
+ at expose_on_virtual_machine_proxy([int], int, minor=11)
+def startOfAlienData(n):
+    print 'Called InterpreterProxy >> startOfAlienData'
+    raise ProxyFunctionFailed
 #   void  *(*startOfAlienData)(sqInt);
+ at expose_on_virtual_machine_proxy([int], int, minor=11)
+def sizeOfAlienData(n):
+    print 'Called InterpreterProxy >> sizeOfAlienData'
+    raise ProxyFunctionFailed
 #   usqInt (*sizeOfAlienData)(sqInt);
-#   sqInt  (*signalNoResume)(sqInt);
+ at expose_on_virtual_machine_proxy([int], int, minor=11)
+def signalNoResume(n):
+    print 'Called InterpreterProxy >> signalNoResume'
+    raise ProxyFunctionFailed
 # #endif
 
 # ##############################################################################
@@ -902,6 +1050,7 @@
             self.reset()
 
     def failed(self, reason=1):
+        assert reason != 0
         self.fail_reason = reason
 
     def oop_to_object(self, oop):
@@ -936,3 +1085,17 @@
         return w_object
 
 IProxy = _InterpreterProxy()
+
+# # Class extensions for Array conversion
+# class __extend__(model.W_PointersObject):
+#     def as_c_array(self, proxy):
+#         return map(lambda x: proxy.object_to_oop(x), self.vars[self.instsize(space):])
+
+# class __extend__(model.W_BytesObject):
+#     def as_c_array(self, proxy):
+#         print "InterpreterProxy >> as_c_array on BytesObject"
+#         raise ProxyFunctionFailed
+
+# class __extend__(model.W_WordsObject):
+#     def as_c_array(self, proxy):
+#         return map(lambda x: proxy.object_to_oop(proxy.space.wrap_positive_32bit_int(x), self.words)

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:10 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:10 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed minor/major-bug
Message-ID: <20130618140410.C28C11C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r452:80ccad6058ee
Date: 2013-06-14 16:44 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/80ccad6058ee/

Log:	fixed minor/major-bug

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -26,7 +26,7 @@
 sqStr = rffi.CCHARP
 void = lltype.Void
 
-major = minor = 0
+MAJOR = MINOR = 0
 functions = []
 
 oop = object()
@@ -35,13 +35,14 @@
     pass
 
 def expose_on_virtual_machine_proxy(unwrap_spec, result_type, minor=0, major=1):
+    global MINOR, MAJOR
     mapping = {oop: sqInt, int: sqInt, list: sqIntArrayPtr, bool: sqInt,
                 float: sqDouble, str: sqStr, long: sqLong, void: void}
     f_ptr = Ptr(FuncType([mapping[spec] for spec in unwrap_spec], mapping[result_type]))
-    if minor < minor:
-        minor = minor
-    if major < major:
-        major = major
+    if MINOR < minor:
+        MINOR = minor
+    if MAJOR < major:
+        MAJOR = major
     def decorator(func):
         len_unwrap_spec = len(unwrap_spec)
         assert (len_unwrap_spec == len(inspect.getargspec(func)[0]) + 1,
@@ -92,11 +93,11 @@
 
 @expose_on_virtual_machine_proxy([], int)
 def minorVersion():
-    return minor
+    return MINOR
 
 @expose_on_virtual_machine_proxy([], int)
 def majorVersion():
-    return major
+    return MAJOR
 
 @expose_on_virtual_machine_proxy([int], int)
 def pop(nItems):
@@ -224,12 +225,14 @@
 @expose_on_virtual_machine_proxy([oop], list)
 def firstFixedField(w_object):
     # return a list with oops (?) of w_objects instVars
-    raise NotImplementedError
+    print "InterpreterProxy >> firstFixedField"
+    raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], list)
 def firstIndexableField(w_object):
     # return a list with values (?) of w_objects variable-parts
-    raise NotImplementedError
+    print "InterpreterProxy >> firstIndexableField"
+    raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, oop], oop)
 def literalofMethod(offset, w_method):

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:11 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:11 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added static tracing for
 called interpreter_proxy functions and
Message-ID: <20130618140411.E02491C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r453:6740391af94e
Date: 2013-06-14 16:45 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/6740391af94e/

Log:	added static tracing for called interpreter_proxy functions and
	fixed bool-return type

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -51,6 +51,7 @@
         def wrapped(*c_arguments):
             assert len_unwrap_spec == len(c_arguments)
             args = ()
+            print "Called InterpreterProxy >> %s" % func.func_name
             try:
                 for i, spec in unrolling_unwrap_spec:
                     c_arg = c_arguments[i]
@@ -65,9 +66,15 @@
                 elif result_type is list:
                     assert isinstance(result, list)
                     return IProxy.list_to_carray(result)
-                elif result_type in (int, float, bool):
+                elif result_type in (int, float):
                     assert isinstance(result, result_type)
                     return result
+                elif result_type is bool:
+                    assert isinstance(result, bool)
+                    if result:
+                        return 1
+                    else:
+                        return 0
                 else:
                     return result
             except error.PrimitiveFailedError:

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:13 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:13 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: (tfel, lwassermann):
Message-ID: <20130618140413.056C51C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r454:40875946b055
Date: 2013-06-14 16:54 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/40875946b055/

Log:	(tfel, lwassermann): added dynamic loading capabilities fixed
	problem with execution folder identification (adapted from topaz)
	fixed bool-function return values removed external bitblt hard link
	to now actually call external function 'copyBits'

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -94,8 +94,8 @@
                     raise NotImplementedError(
                         "InterpreterProxy: unknown result_type %s" % (result_type, ))
         wrapped.func_name = "wrapped_ipf_" + func.func_name
-        functions.append(("c_" + func.func_name, f_ptr, wrapped))
-        return func
+        functions.append((func.func_name, f_ptr, wrapped))
+        return wrapped
     return decorator
 
 @expose_on_virtual_machine_proxy([], int)
@@ -1005,8 +1005,7 @@
 # ##############################################################################
 
 VirtualMachine = lltype.Struct("VirtualMachine",
-        *map(lambda x: (x[0], x[1]), functions),
-        hints={'c_name': 'VirtualMachine'})
+        *map(lambda x: (x[0], x[1]), functions))
 VMPtr = Ptr(VirtualMachine)
 
 proxy_functions = unrolling_iterable(functions)
@@ -1023,6 +1022,10 @@
 
 # rffi.llexternal is supposed to represent c-functions.
 
+func_str_void = Ptr(FuncType([], sqStr))
+func_bool_vm = Ptr(FuncType([VMPtr], sqInt))
+func_bool_void = Ptr(FuncType([], sqInt))
+
 class _InterpreterProxy(object):
     _immutable_fields_ = ['vm_initialized?']
 
@@ -1032,6 +1035,7 @@
         self._next_oop = 0
         self.oop_map = {}
         self.object_map = {}
+        self.loaded_modules = {}
         self.remappable_objects = []
         self.reset()
 
@@ -1043,6 +1047,30 @@
         self.fail_reason = 0
 
     def call(self, signature, interp, s_frame, argcount, s_method):
+        from rpython.rlib.rdynload import dlsym
+        self.initialize_from_call(signature, interp, s_frame, argcount, s_method)
+        try:
+            if signature[0] not in self.loaded_modules:
+                module = self.load_and_initialize(signature[0])
+                print "Successfully loaded: %s" % signature[0]
+            else:
+                module = self.loaded_modules[signature[0]]
+
+            # call the correct function in it...
+            try:
+                _external_function = dlsym(module, signature[1])
+            except KeyError:
+                self.failed()
+            else:
+                external_function = rffi.cast(func_bool_void, _external_function)
+                external_function()
+
+            if not self.fail_reason == 0:
+                raise error.PrimitiveFailedError
+        finally:
+            self.reset()
+
+    def initialize_from_call(self, signature, interp, s_frame, argcount, s_method):
         self.interp = interp
         self.s_frame = s_frame
         self.argcount = argcount
@@ -1050,14 +1078,6 @@
         self.space = interp.space
         # ensure that space.w_nil gets the first possible oop
         self.object_to_oop(self.space.w_nil)
-        try:
-            # Load the correct DLL
-            self.failed()
-            # call the correct function in it...
-            if not self.fail_reason == 0:
-                raise error.PrimitiveFailedError
-        finally:
-            self.reset()
 
     def failed(self, reason=1):
         assert reason != 0
@@ -1094,6 +1114,57 @@
         self.remappable_objects.append(w_object)
         return w_object
 
+    def top_remappable(self):
+        if len(self.remappable_objects) == 0:
+            raise ProxyFunctionFailed
+        return self.remappable_objects[-1]
+
+    def load_and_initialize(self, module_name):
+        from rpython.rlib.rdynload import dlopen, dlsym, dlclose, DLOpenError
+        import os
+        c_name = rffi.str2charp(os.path.join(IProxy.space.executable_path(), module_name))
+        try:
+            module = dlopen(c_name)
+        except DLOpenError, e:
+            print "Missing library: %s" % e
+            raise error.PrimitiveFailedError
+        try:
+            try:
+                _getModuleName = dlsym(module, "getModuleName")
+            except KeyError:
+                pass # the method does not need to exist
+            else:
+                getModuleName = rffi.cast(func_str_void, _getModuleName)
+                if not rffi.charp2str(getModuleName()).startswith(module_name):
+                    raise error.PrimitiveFailedError
+
+            try:
+                _setInterpreter = dlsym(module, "setInterpreter")
+            except KeyError:
+                raise error.PrimitiveFailedError
+            else:
+                setInterpreter = rffi.cast(func_bool_vm, _setInterpreter)
+                if not setInterpreter(sqGetInterpreterProxy()):
+                    print "Failed setting interpreter on: %s" % module_name
+                    raise error.PrimitiveFailedError
+
+            try:
+                _initialiseModule = dlsym(module, "initialiseModule")
+            except KeyError:
+                pass # the method does not need to exist
+            else:
+                initialiseModule = rffi.cast(func_bool_void, _initialiseModule)
+                if not initialiseModule():
+                    print "Failed initialization of: %s" % module_name
+                    raise error.PrimitiveFailedError
+
+            self.loaded_modules[module_name] = module
+            return module
+        except error.PrimitiveFailedError:
+            dlclose(module)
+            raise
+
+
 IProxy = _InterpreterProxy()
 
 # # Class extensions for Array conversion
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -1,15 +1,40 @@
+import os
+
 from spyvm import constants, model, shadow, wrapper
 from spyvm.error import UnwrappingError, WrappingError, PrimitiveFailedError
-from rpython.rlib import jit
+from rpython.rlib import jit, rpath
 from rpython.rlib.objectmodel import instantiate, specialize
 from rpython.rlib.rarithmetic import intmask, r_uint, int_between
 
 class ObjSpace(object):
     def __init__(self):
         self.classtable = {}
+        self._executable_path = [""] # XXX: we cannot set the attribute
+                                  # directly on the frozen objectspace
         self.make_bootstrap_classes()
         self.make_bootstrap_objects()
 
+    def find_executable(self, executable):
+        if os.sep in executable or (os.name == "nt" and ":" in executable):
+            return executable
+        path = os.environ.get("PATH")
+        if path:
+            for dir in path.split(os.pathsep):
+                f = os.path.join(dir, executable)
+                if os.path.isfile(f):
+                    executable = f
+                    break
+        return rpath.rabspath(executable)
+
+    def runtime_setup(self, executable):
+        fullpath = rpath.rabspath(self.find_executable(executable))
+        i = fullpath.rfind(os.path.sep) + 1
+        assert i > 0
+        self._executable_path[0] = fullpath[:i]
+
+    def executable_path(self):
+        return self._executable_path[0]
+
     def make_bootstrap_classes(self):
         def define_core_cls(name, w_superclass, w_metaclass):
             assert name.startswith('w_')
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -839,9 +839,9 @@
         raise PrimitiveFailedError
     signature = (w_modulename.as_string(), w_functionname.as_string())
 
-    if signature == ('BitBltPlugin', 'primitiveCopyBits'):
-        return prim_holder.prim_table[BITBLT_COPY_BITS](interp, s_frame, argcount, s_method)
-    elif signature[0] == "SocketPlugin":
+    # if signature == ('BitBltPlugin', 'primitiveCopyBits'):
+    #     return prim_holder.prim_table[BITBLT_COPY_BITS](interp, s_frame, argcount, s_method)
+    if signature[0] == "SocketPlugin":
         from spyvm.plugins.socket import SocketPlugin
         return SocketPlugin.call(signature[1], interp, s_frame, argcount, s_method)
     elif signature[0] == "FilePlugin":
@@ -994,7 +994,7 @@
 
 @expose_primitive(VM_PATH, unwrap_spec=[object])
 def func(interp, s_frame, w_receiver):
-    return interp.space.wrap_string(os.path.join(os.getcwd(), ''))
+    return interp.space.wrap_string("%s%s" % (interp.space.executable_path(), os.path.sep))
 
 @expose_primitive(SHORT_AT, unwrap_spec=[object, index1_0])
 def func(interp, s_frame, w_receiver, n0):
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -136,6 +136,7 @@
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
     interp = interpreter.Interpreter(space, image, image_name=path, trace=trace)
+    space.runtime_setup(argv[0])
     if benchmark is not None:
         return _run_benchmark(interp, number, benchmark)
     else:

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:14 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:14 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: increased minor version by
 one to be able to load shared libraries compiled with recent headers
Message-ID: <20130618140414.B17CB1C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r455:568172d7577c
Date: 2013-06-14 16:54 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/568172d7577c/

Log:	increased minor version by one to be able to load shared libraries
	compiled with recent headers

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -996,7 +996,7 @@
     print 'Called InterpreterProxy >> sizeOfAlienData'
     raise ProxyFunctionFailed
 #   usqInt (*sizeOfAlienData)(sqInt);
- at expose_on_virtual_machine_proxy([int], int, minor=11)
+ at expose_on_virtual_machine_proxy([int], int, minor=12)
 def signalNoResume(n):
     print 'Called InterpreterProxy >> signalNoResume'
     raise ProxyFunctionFailed

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:16 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:16 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added correct identification
 of saved window size and communication to the image (via primitive
 SCREEN_SIZE)
Message-ID: <20130618140416.A11101C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r456:4f4372c774a9
Date: 2013-06-17 13:20 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/4f4372c774a9/

Log:	added correct identification of saved window size and communication
	to the image (via primitive SCREEN_SIZE) needed to add the
	lastWindowSize word to SqueakImage and its maintenance when setting
	the display object (BE_DISPLAY)

diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -698,8 +698,9 @@
         w_rcvr.store(interp.space, 0, w_display_bitmap)
 
     w_display_bitmap.flush_to_screen()
+    interp.image.lastWindowSize = (width << 16)  + height
+    interp.space.objtable['w_display'] = w_rcvr
 
-    interp.space.objtable['w_display'] = w_rcvr
     return w_rcvr
 
 @expose_primitive(STRING_REPLACE, unwrap_spec=[object, index1_0, index1_0, object, index1_0])
@@ -730,11 +731,14 @@
 
 @expose_primitive(SCREEN_SIZE, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    # XXX get the real screen size
+    # We need to have the indirection via interp.image, because when the image
+    # is saved, the display form size is always reduced to 240 at 120.
+    if not interp.image:
+        raise PrimitiveFailedError
     w_res = interp.space.w_Point.as_class_get_shadow(interp.space).new(2)
     point = wrapper.PointWrapper(interp.space, w_res)
-    point.store_x(640)
-    point.store_y(480)
+    point.store_x((interp.image.lastWindowSize >> 16) & 0xffff)
+    point.store_y(interp.image.lastWindowSize & 0xffff)
     return w_res
 
 @expose_primitive(MOUSE_BUTTONS, unwrap_spec=[object])
diff --git a/spyvm/squeakimage.py b/spyvm/squeakimage.py
--- a/spyvm/squeakimage.py
+++ b/spyvm/squeakimage.py
@@ -218,6 +218,8 @@
         # cache wrapper integers
         self.intcache = {}
 
+        self.lastWindowSize = 0
+
     def initialize(self):
         # XXX should be called something like read_full_image
         self.read_header()
@@ -247,8 +249,8 @@
         self.specialobjectspointer = self.stream.next()
         # 1 word last used hash
         lasthash = self.stream.next()
-        savedwindowssize = self.stream.next()
-        # print "savedwindowssize", savedwindowssize
+        self.lastWindowSize = savedwindowssize = self.stream.next()
+        # print "savedwindowssize: ", savedwindowssize >> 16, "@", savedwindowssize & 0xffff
         fullscreenflag = self.stream.next()
         extravmmemory = self.stream.next()
         self.stream.skipbytes(headersize - self.stream.pos)
@@ -371,6 +373,7 @@
 
         self.w_asSymbol = self.find_symbol(space, reader, "asSymbol")
         self.w_simulateCopyBits = self.find_symbol(space, reader, "simulateCopyBits")
+        self.lastWindowSize = reader.lastWindowSize
 
     def find_symbol(self, space, reader, symbol):
         w_dnu = self.special(constants.SO_DOES_NOT_UNDERSTAND)

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:17 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:17 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: improved printing for
	interpreter proxy
Message-ID: <20130618140417.C14AB1C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r457:19c1a78e9e83
Date: 2013-06-17 13:22 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/19c1a78e9e83/

Log:	improved printing for interpreter proxy removed individual
	printing statements in favor of printing in the wrapper
	added printing of return values to allow easier tracking

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -51,7 +51,7 @@
         def wrapped(*c_arguments):
             assert len_unwrap_spec == len(c_arguments)
             args = ()
-            print "Called InterpreterProxy >> %s" % func.func_name
+            print 'Called InterpreterProxy >> %s' % func.func_name,
             try:
                 for i, spec in unrolling_unwrap_spec:
                     c_arg = c_arguments[i]
@@ -60,6 +60,7 @@
                     else:
                         args += (c_arg, )
                 result = func(*args)
+                print '\t-> %s' % result
                 if result_type is oop:
                     assert isinstance(result, model.W_Object)
                     return IProxy.object_to_oop(result)
@@ -78,6 +79,7 @@
                 else:
                     return result
             except error.PrimitiveFailedError:
+                print '\t-> failed'
                 IProxy.failed()
                 if mapping[result_type] is sqInt:
                     return 0
@@ -188,7 +190,7 @@
     size = s_class.instsize()
     if s_class.isvariable():
         size += w_object.primsize(IProxy.space)
-    if isinstance(w_object, model.W_BytesObject):
+    if not isinstance(w_object, model.W_BytesObject):
         size *= 4
     return size
 
@@ -232,13 +234,11 @@
 @expose_on_virtual_machine_proxy([oop], list)
 def firstFixedField(w_object):
     # return a list with oops (?) of w_objects instVars
-    print "InterpreterProxy >> firstFixedField"
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], list)
 def firstIndexableField(w_object):
     # return a list with values (?) of w_objects variable-parts
-    print "InterpreterProxy >> firstIndexableField"
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, oop], oop)
@@ -320,14 +320,12 @@
 @expose_on_virtual_machine_proxy([oop, str], bool)
 def isKindOf(w_object, name):
     # XXX: stub, until used
-    print "InterpreterProxy >> isKindOf(object, name)"
     return False
 #     sqInt (*isKindOf)(sqInt oop, char *aString);
 
 @expose_on_virtual_machine_proxy([oop, str], bool)
 def isMemberOf(w_object, name):
     # XXX: stub, until used
-    print "InterpreterProxy >> isMemberOf(object, name)"
     return False
 #     sqInt (*isMemberOf)(sqInt oop, char *aString);
 
@@ -510,7 +508,6 @@
 @expose_on_virtual_machine_proxy([list, list], int)
 def becomewith(w_array1, w_array2):
     # XXX: stub, until used
-    print "InterpreterProxy >> becomewith(list, list)"
     return 0
 
 @expose_on_virtual_machine_proxy([int], int)
@@ -546,13 +543,11 @@
 
 @expose_on_virtual_machine_proxy([oop, int, int, int, int], int)
 def showDisplayBitsLeftTopRightBottom(w_form, l, t, r, b):
-    print 'Called InterpreterProxy >> showDisplayBitsLeftTopRightBottom'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], int)
 def signalSemaphoreWithIndex(n):
     # ((Smalltalk externalObjects) at: n) signal
-    print 'Called InterpreterProxy >> signalSemaphoreWithIndex'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([bool], int)
@@ -576,12 +571,10 @@
 
 @expose_on_virtual_machine_proxy([], int)
 def compilerHookVector():
-    print 'Called InterpreterProxy >> compilerHookVector'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], int)
 def setCompilerInitialized(n):
-    print 'Called InterpreterProxy >> setCompilerInitialized'
     raise ProxyFunctionFailed
 
 #     /* InterpreterProxy methodsFor: 'BitBlt support' */
@@ -589,40 +582,33 @@
 @expose_on_virtual_machine_proxy([int], int, minor=1)
 def loadBitBltFrom(w_bitBlit):
     # bb := bbOop
-    print 'Called InterpreterProxy >> loadBitBltFrom'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], int, minor=1)
 def copyBits():
     # bb copyBits
-    print 'Called InterpreterProxy >> copyBits'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, int, int], int, minor=1)
 def copyBitsFromtoat(x0, x1, y):
     # bb copyBitsFrom: x0 to: x1 at: y
-    print 'Called InterpreterProxy >> copyBitsFromtoat'
     raise ProxyFunctionFailed
 
 # #if VM_PROXY_MINOR > 2
 @expose_on_virtual_machine_proxy([], oop, minor=2)
 def classLargeNegativeInteger():
-    print 'Called InterpreterProxy >> classLargeNegativeInteger'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], oop, minor=2)
 def signed32BitIntegerFor(n):
-    print 'Called InterpreterProxy >> signed32BitIntegerFor'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], int, minor=2)
 def signed32BitValueOf(w_number):
-    print 'Called InterpreterProxy >> signed32BitValueOf'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop, oop], int, minor=2)
 def includesBehaviorThatOf(w_class, w_superclass):
-    print 'Called InterpreterProxy >> includesBehaviorThatOf'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=2)
@@ -633,54 +619,44 @@
 
 @expose_on_virtual_machine_proxy([], oop, minor=2)
 def classExternalAddress():
-    print 'Called InterpreterProxy >> classExternalAddress'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=2)
 def classExternalData():
-    print 'Called InterpreterProxy >> classExternalData'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=2)
 def classExternalFunction():
-    print 'Called InterpreterProxy >> classExternalFunction'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=2)
 def classExternalLibrary():
-    print 'Called InterpreterProxy >> classExternalLibrary'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=2)
 def classExternalStructure():
-    print 'Called InterpreterProxy >> classExternalStructure'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, int], oop, minor=2)
 def ioLoadModuleOfLength(modIndex, modLength):
-    print 'Called InterpreterProxy >> ioLoadModuleOfLength'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, int, int], int, minor=2)
 def ioLoadSymbolOfLengthFromModule(fnIndex, fnLength, handle):
-    print 'Called InterpreterProxy >> ioLoadSymbolOfLengthFromModule'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], bool, minor=2)
 def isInMemory(address):
-    print 'Called InterpreterProxy >> isInMemory'
     raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 3
 @expose_on_virtual_machine_proxy([str, str], bool, minor=3)
 def ioLoadFunctionFrom(fnName, modName):
-    print 'Called InterpreterProxy >> ioLoadFunctionFrom'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], bool, minor=3)
 def ioMicroMSecs():
-    print 'Called InterpreterProxy >> ioMicroMSecs'
     raise ProxyFunctionFailed
 # #endif
 
@@ -698,22 +674,18 @@
 
 @expose_on_virtual_machine_proxy([long], oop, minor=4)
 def positive64BitIntegerFor(integerValue):
-    print 'Called InterpreterProxy >> positive64BitIntegerFor'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], long, minor=4)
 def positive64BitValueOf(w_number):
-    print 'Called InterpreterProxy >> positive64BitValueOf'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([long], oop, minor=4)
 def signed64BitIntegerFor(integerValue):
-    print 'Called InterpreterProxy >> signed64BitIntegerFor'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], long, minor=4)
 def signed64BitValueOf(w_number):
-    print 'Called InterpreterProxy >> signed64BitValueOf'
     raise ProxyFunctionFailed
 # #endif
 
@@ -731,30 +703,25 @@
 
 @expose_on_virtual_machine_proxy([], int, minor=5)
 def forceInterruptCheck():
-    print 'Called InterpreterProxy >> forceInterruptCheck'
     raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 6
 @expose_on_virtual_machine_proxy([int, oop], oop, minor=6)
 def fetchLong32ofObject(fieldFieldIndex, oop):
-    print 'Called InterpreterProxy >> fetchLong32ofObject'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, oop], int, minor=6)
 def getThisSessionID(fieldFieldIndex, oop):
     # return random int
-    print 'Called InterpreterProxy >> getThisSessionID'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([str, str, int, int], oop, minor=6)
 def ioFilenamefromStringofLengthresolveAliases(aCharBuffer, filenameIndex, filenameLength, resolveFlag):
-    print 'Called InterpreterProxy >> ioFilenamefromStringofLengthresolveAliases'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], int, minor=6)
 def vmEndianness():
-    print 'Called InterpreterProxy >> vmEndianness'
     return 0 # return 1 for big endian
 # #endif
 
@@ -769,7 +736,6 @@
 #   sqInt (*callbackEnter)(sqInt *callbackID);
 @expose_on_virtual_machine_proxy([int], bool, minor=7)
 def callbackEnter(callbackID):
-    print 'Called InterpreterProxy >> callbackEnter'
     raise ProxyFunctionFailed
 
 #   /* callbackLeave: Leave the interpreter from a previous callback
@@ -779,7 +745,6 @@
 #   sqInt (*callbackLeave)(sqInt  callbackID);
 @expose_on_virtual_machine_proxy([int], bool, minor=7)
 def callbackLeave(callbackID):
-    print 'Called InterpreterProxy >> callbackLeave'
     raise ProxyFunctionFailed
 
 #   /* addGCRoot: Add a variable location to the garbage collector.
@@ -790,7 +755,6 @@
 #   sqInt (*addGCRoot)(sqInt *varLoc);
 @expose_on_virtual_machine_proxy([oop], bool, minor=7)
 def addGCRoot(callbackID):
-    print 'Called InterpreterProxy >> addGCRoot'
     raise ProxyFunctionFailed
 
 #   /* removeGCRoot: Remove a variable location from the garbage collector.
@@ -801,7 +765,6 @@
 #   sqInt (*removeGCRoot)(sqInt *varLoc);
 @expose_on_virtual_machine_proxy([oop], bool, minor=7)
 def removeGCRoot(callbackID):
-    print 'Called InterpreterProxy >> removeGCRoot'
     raise ProxyFunctionFailed
 # #endif
 
@@ -814,74 +777,60 @@
 
 @expose_on_virtual_machine_proxy([], void, minor=8)
 def setInterruptCheckChain():
-    print 'Called InterpreterProxy >> setInterruptCheckChain'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=8)
 def classAlien():
-    print 'Called InterpreterProxy >> classAlien'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=8)
 def classUnsafeAlien():
-    print 'Called InterpreterProxy >> classUnsafeAlien'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, int, int, int], int, minor=8)
 def sendInvokeCallbackStackRegistersJmpbuf(thunkPtrAsInt, stackPtrAsInt, regsPtrAsInt, jmpBufPtrAsInt):
-    print 'Called InterpreterProxy >> sendInvokeCallbackStackRegistersJmpbuf'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], int, minor=8)
 def reestablishContextPriorToCallback(callbackContext):
-    print 'Called InterpreterProxy >> reestablishContextPriorToCallback'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], int, minor=8)
 def getStackPointer():
-    print 'Called InterpreterProxy >> getStackPointer'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], bool, minor=8)
 def isOopImmutable(w_object):
-    print 'Called InterpreterProxy >> isOopImmutable'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], bool, minor=8)
 def isOopMutable(w_object):
-    print 'Called InterpreterProxy >> isOopMutable'
     raise ProxyFunctionFailed
 # #endif
 
 # #if VM_PROXY_MINOR > 9
 @expose_on_virtual_machine_proxy([int], oop, minor=9)
 def methodArg(n):
-    print 'Called InterpreterProxy >> methodArg'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], oop, minor=9)
 def objectArg(n):
-    print 'Called InterpreterProxy >> objectArg'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], int, minor=9)
 def integerArg(n):
-    print 'Called InterpreterProxy >> integerArg'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], float, minor=9)
 def floatArg(n):
-    print 'Called InterpreterProxy >> floatArg'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop], int, minor=9)
 def methodReturnValue(w_object):
-    print 'Called InterpreterProxy >> methodReturnValue'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=9)
 def topRemappableOop():
-    print 'Called InterpreterProxy >> topRemappableOop'
     raise ProxyFunctionFailed
 # #endif
 
@@ -889,47 +838,38 @@
 # # define DisownVMLockOutFullGC 1
 @expose_on_virtual_machine_proxy([int], int, minor=10)
 def disownVM(flags):
-    print 'Called InterpreterProxy >> disownVM'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], int, minor=10)
 def ownVM(threadIdAndFlags):
-    print 'Called InterpreterProxy >> ownVM'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, int], int, minor=10)
 def addHighPriorityTickee(ticker, periodms):
-    print 'Called InterpreterProxy >> addHighPriorityTickee'
     raise ProxyFunctionFailed
 #   void  (*addHighPriorityTickee)(void (*ticker)(void), unsigned periodms);
 @expose_on_virtual_machine_proxy([int, int, int], int, minor=10)
 def addSynchronousTickee(ticker, periodms, roundms):
-    print 'Called InterpreterProxy >> addSynchronousTickee'
     raise ProxyFunctionFailed
 #   void  (*addSynchronousTickee)(void (*ticker)(void), unsigned periodms, unsigned roundms);
 @expose_on_virtual_machine_proxy([], long, minor=10)
 def utcMicroseconds():
-    print 'Called InterpreterProxy >> utcMicroseconds'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], int, minor=10)
 def tenuringIncrementalGC():
-    print 'Called InterpreterProxy >> tenuringIncrementalGC'
     raise ProxyFunctionFailed
 #   sqInt (*tenuringIncrementalGC)(void);
 @expose_on_virtual_machine_proxy([oop], bool, minor=10)
 def isYoung(w_object):
-    print 'Called InterpreterProxy >> isYoung'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop, oop], bool, minor=10)
 def isKindOfClass(w_object, w_class):
-    print 'Called InterpreterProxy >> isKindOfClass'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], oop, minor=10)
 def primitiveErrorTable():
-    print 'Called InterpreterProxy >> primitiveErrorTable'
     raise ProxyFunctionFailed
 #   sqInt (*primitiveErrorTable)(void);
 @expose_on_virtual_machine_proxy([], int, minor=10)
@@ -948,57 +888,46 @@
 # /* VMCallbackContext opaque type avoids all including setjmp.h & vmCallback.h */
 @expose_on_virtual_machine_proxy([int], int, minor=11)
 def sendInvokeCallbackContext(vmccp):
-    print 'Called InterpreterProxy >> sendInvokeCallbackContext'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, int, int], int, minor=11)
 def returnAsThroughCallbackContext(n, vmccp, m):
-    print 'Called InterpreterProxy >> returnAsThroughCallbackContext'
     raise ProxyFunctionFailed
 #   sqInt (*returnAsThroughCallbackContext)(int, vmccp, sqInt);
 @expose_on_virtual_machine_proxy([int], long, minor=11)
 def signedMachineIntegerValueOf(n):
-    print 'Called InterpreterProxy >> signedMachineIntegerValueOf'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], long, minor=11)
 def stackSignedMachineIntegerValue(n):
-    print 'Called InterpreterProxy >> stackSignedMachineIntegerValue'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], long, minor=11)
 def positiveMachineIntegerValueOf(n):
-    print 'Called InterpreterProxy >> positiveMachineIntegerValueOf'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], long, minor=11)
 def stackPositiveMachineIntegerValue(n):
-    print 'Called InterpreterProxy >> stackPositiveMachineIntegerValue'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([], int, minor=11)
 def getInterruptPending():
-    print 'Called InterpreterProxy >> getInterruptPending'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], str, minor=11)
 def cStringOrNullFor(n):
-    print 'Called InterpreterProxy >> cStringOrNullFor'
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int], int, minor=11)
 def startOfAlienData(n):
-    print 'Called InterpreterProxy >> startOfAlienData'
     raise ProxyFunctionFailed
 #   void  *(*startOfAlienData)(sqInt);
 @expose_on_virtual_machine_proxy([int], int, minor=11)
 def sizeOfAlienData(n):
-    print 'Called InterpreterProxy >> sizeOfAlienData'
     raise ProxyFunctionFailed
 #   usqInt (*sizeOfAlienData)(sqInt);
 @expose_on_virtual_machine_proxy([int], int, minor=12)
 def signalNoResume(n):
-    print 'Called InterpreterProxy >> signalNoResume'
     raise ProxyFunctionFailed
 # #endif
 
@@ -1052,7 +981,6 @@
         try:
             if signature[0] not in self.loaded_modules:
                 module = self.load_and_initialize(signature[0])
-                print "Successfully loaded: %s" % signature[0]
             else:
                 module = self.loaded_modules[signature[0]]
 
@@ -1063,6 +991,7 @@
                 self.failed()
             else:
                 external_function = rffi.cast(func_bool_void, _external_function)
+                print "Calling %s >> %s" % signature
                 external_function()
 
             if not self.fail_reason == 0:
@@ -1094,6 +1023,7 @@
             return self.object_map[w_object]
         except KeyError:
             new_index = self.next_oop()
+            print "Mapping new Object: %d -> %s" % (new_index, w_object)
             self.oop_map[new_index] = w_object
             self.object_map[w_object] = new_index
             return new_index
@@ -1126,7 +1056,6 @@
         try:
             module = dlopen(c_name)
         except DLOpenError, e:
-            print "Missing library: %s" % e
             raise error.PrimitiveFailedError
         try:
             try:

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:19 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:19 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added the possibility of
 having a single string arg to targetimageloadingsmalltalk
Message-ID: <20130618140419.29B291C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r458:08c73a56a895
Date: 2013-06-17 14:13 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/08c73a56a895/

Log:	added the possibility of having a single string arg to
	targetimageloadingsmalltalk

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -10,7 +10,7 @@
 from spyvm.interpreter_proxy import VirtualMachine
 
 
-def _run_benchmark(interp, number, benchmark):
+def _run_benchmark(interp, number, benchmark, arg):
     from spyvm.plugins.vmdebugging import stop_ui_process
     stop_ui_process()
 
@@ -25,15 +25,7 @@
         len(w_hpp._vars)
     )
 
-    # XXX: Copied from interpreter >> perform
-    w_receiver = interp.space.wrap_int(number)
-    w_selector = interp.perform(interp.space.wrap_string(benchmark), "asSymbol")
-    w_method = model.W_CompiledMethod(header=512)
-    w_method.literalatput0(interp.space, 1, w_selector)
-    w_method.setbytes([chr(131), chr(0), chr(124)]) #returnTopFromMethod
-    s_method = w_method.as_compiledmethod_get_shadow(interp.space)
-    s_frame = shadow.MethodContextShadow.make_context(interp.space, s_method, w_receiver, [], None)
-    s_frame.push(w_receiver)
+    s_frame = context_for(interp, number, benchmark, arg)
     # second variable is suspended context
     w_benchmark_proc.store(space, 1, s_frame.w_self())
 
@@ -69,13 +61,28 @@
 
 space = objspace.ObjSpace()
 
+def context_for(interp, number, benchmark, stringarg):
+    # XXX: Copied from interpreter >> perform
+    argcount = 0 if stringarg == "" else 1
+    w_receiver = interp.space.wrap_int(number)
+    w_selector = interp.perform(interp.space.wrap_string(benchmark), "asSymbol")
+    w_method = model.W_CompiledMethod(header=512)
+    w_method.literalatput0(interp.space, 1, w_selector)
+    w_method.setbytes([chr(131), chr(argcount << 5), chr(124)]) #returnTopFromMethod
+    s_method = w_method.as_compiledmethod_get_shadow(interp.space)
+    s_frame = shadow.MethodContextShadow.make_context(interp.space, s_method, w_receiver, [], None)
+    s_frame.push(w_receiver)
+    if not stringarg == "":
+        s_frame.push(interp.space.wrap_string(stringarg))
+    return s_frame
 
 def _usage(argv):
     print """
     Usage: %s
           -j|--jit [jitargs]
-          -n|--number [smallint]
+          -n|--number [smallint, default: 0]
           -m|--method [benchmark on smallint]
+          -a|--arg [string argument to #method]
           [image path, default: Squeak.image]
     """ % argv[0]
 
@@ -91,6 +98,7 @@
     number = 0
     benchmark = None
     trace = False
+    stringarg = ""
 
     while idx < len(argv):
         arg = argv[idx]
@@ -112,6 +120,10 @@
             idx += 1
         elif arg in ["-t", "--trace"]:
             trace = True
+        elif arg in ["-a", "--arg"]:
+            _arg_missing(argv, idx, arg)
+            stringarg = argv[idx + 1]
+            idx += 1
         elif path is None:
             path = argv[idx]
         else:
@@ -138,7 +150,7 @@
     interp = interpreter.Interpreter(space, image, image_name=path, trace=trace)
     space.runtime_setup(argv[0])
     if benchmark is not None:
-        return _run_benchmark(interp, number, benchmark)
+        return _run_benchmark(interp, number, benchmark, stringarg)
     else:
         _run_image(interp)
         return 0

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:20 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:20 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed error in quo primitive:
 round towards 0 instead of -inf
Message-ID: <20130618140420.31DB61C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r459:7c7a00b215d5
Date: 2013-06-17 16:01 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/7c7a00b215d5/

Log:	fixed error in quo primitive: round towards 0 instead of -inf

diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -235,7 +235,11 @@
 def func(interp, s_frame, receiver, argument):
     if argument == 0:
         raise PrimitiveFailedError()
-    return interp.space.wrap_int(receiver // argument)
+    res = receiver // argument
+    # see http://python-history.blogspot.de/2010/08/why-pythons-integer-division-floors.html
+    if res < 0:
+        res = res + 1
+    return interp.space.wrap_int(res)
 
 # #bitShift: -- return the shifted value
 @expose_primitive(BIT_SHIFT, unwrap_spec=[object, int])
diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py
--- a/spyvm/test/test_primitives.py
+++ b/spyvm/test/test_primitives.py
@@ -116,6 +116,7 @@
 def test_small_int_quo():
     assert prim(primitives.QUO, [12,3]).value == 4
     assert prim(primitives.QUO, [12,7]).value == 1
+    assert prim(primitives.QUO, [-9,4]).value == -2
 
 def test_small_int_quo_fail():
     prim_fails(primitives.QUO, [12, 0])

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:22 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:22 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed another problem with
	primitive QUO
Message-ID: <20130618140422.6C4DC1C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r460:6dde3110edf1
Date: 2013-06-17 17:14 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/6dde3110edf1/

Log:	fixed another problem with primitive QUO added more test-cases for
	quo

diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -237,7 +237,7 @@
         raise PrimitiveFailedError()
     res = receiver // argument
     # see http://python-history.blogspot.de/2010/08/why-pythons-integer-division-floors.html
-    if res < 0:
+    if res < 0 and not abs(receiver) == abs(argument):
         res = res + 1
     return interp.space.wrap_int(res)
 
@@ -702,7 +702,8 @@
         w_rcvr.store(interp.space, 0, w_display_bitmap)
 
     w_display_bitmap.flush_to_screen()
-    interp.image.lastWindowSize = (width << 16)  + height
+    if interp.image:
+        interp.image.lastWindowSize = (width << 16)  + height
     interp.space.objtable['w_display'] = w_rcvr
 
     return w_rcvr
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -989,8 +989,8 @@
                 self.pc() + 1
             )
         args = '%d' % argcount
-        for i in range(argcount - 1, -1, -1):
-            args += ': %s' % self.peek(i).as_repr_string()
+        for i in range(argcount):
+            args += ': %s' % self.peek(argcount -1 - i).as_repr_string()
         return '%s (rcvr: %s) [pc: %d] (%s)' % (
             self.method_str(),
             self.w_receiver().as_repr_string(),
diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py
--- a/spyvm/test/test_primitives.py
+++ b/spyvm/test/test_primitives.py
@@ -117,6 +117,12 @@
     assert prim(primitives.QUO, [12,3]).value == 4
     assert prim(primitives.QUO, [12,7]).value == 1
     assert prim(primitives.QUO, [-9,4]).value == -2
+    assert prim(primitives.QUO, [-12,12]).value == -1
+    assert prim(primitives.QUO, [-12,11]).value == -1
+    assert prim(primitives.QUO, [-12,13]).value == 0
+    assert prim(primitives.QUO, [-12,-12]).value == 1
+    assert prim(primitives.QUO, [12,-11]).value == -1
+    assert prim(primitives.QUO, [12,-13]).value == 0
 
 def test_small_int_quo_fail():
     prim_fails(primitives.QUO, [12, 0])

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:23 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:23 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added fileplugin
	primitiveGetPosition
Message-ID: <20130618140423.86E9F1C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r461:8bb0308ecb3d
Date: 2013-06-17 17:15 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/8bb0308ecb3d/

Log:	added fileplugin primitiveGetPosition

diff --git a/spyvm/plugins/fileplugin.py b/spyvm/plugins/fileplugin.py
--- a/spyvm/plugins/fileplugin.py
+++ b/spyvm/plugins/fileplugin.py
@@ -101,6 +101,15 @@
         target.setchar(start + i, contents[i])
     return space.wrap_int(len_read)
 
+ at FilePlugin.expose_primitive(unwrap_spec=[object, int])
+def primitiveFileGetPosition(interp, s_frame, w_rcvr, fd):
+    try:
+        pos = os.lseek(fd, 0, os.SEEK_CUR)
+    except OSError:
+        raise PrimitiveFailedError
+    else:
+        return interp.space.wrap_int(pos)
+
 @FilePlugin.expose_primitive(unwrap_spec=[object, int, int])
 def primitiveFileSetPosition(interp, s_frame, w_rcvr, fd, position):
     try:

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:24 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:24 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: restructured
 sqGetInterpreterProxy,
 because the JIT can't be generated when exposed functions are called in
 rpython code
Message-ID: <20130618140424.994DE1C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r462:87173d05b6c2
Date: 2013-06-18 15:55 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/87173d05b6c2/

Log:	restructured sqGetInterpreterProxy, because the JIT can't be
	generated when exposed functions are called in rpython code

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -941,6 +941,11 @@
 
 @entrypoint('main', [], c_name='sqGetInterpreterProxy')
 def sqGetInterpreterProxy():
+    return getInterpreterProxy()
+
+# Redirect of sqGetInterpreterProxy, because the JIT doesn't allow calling of
+# functions annotated with @entrypoint.
+def getInterpreterProxy():
     if not IProxy.vm_initialized:
         vm_proxy = lltype.malloc(VirtualMachine, flavor='raw')
         for func_name, signature, func in proxy_functions:
@@ -1073,7 +1078,7 @@
                 raise error.PrimitiveFailedError
             else:
                 setInterpreter = rffi.cast(func_bool_vm, _setInterpreter)
-                if not setInterpreter(sqGetInterpreterProxy()):
+                if not setInterpreter(getInterpreterProxy()):
                     print "Failed setting interpreter on: %s" % module_name
                     raise error.PrimitiveFailedError
 

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:25 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:25 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed the JIT-bug:
Message-ID: <20130618140425.AEFFA1C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r463:52d5e8f2ef2d
Date: 2013-06-18 15:57 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/52d5e8f2ef2d/

Log:	fixed the JIT-bug: wrap_int was annotated with longlong
	type, although it should have been int fixed all such send
	sites added assert to ensure proper errors on future
	mistakes added try:except: block in fileplugin, to be able
	to run the vm in the meta-interpreter

diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -199,6 +199,7 @@
 
     def wrap_int(self, val):
         from spyvm import constants
+        assert isinstance(val, int)
         if int_between(constants.TAGGED_MININT, val,
                         constants.TAGGED_MAXINT + 1):
             return model.W_SmallInteger(val)
diff --git a/spyvm/plugins/fileplugin.py b/spyvm/plugins/fileplugin.py
--- a/spyvm/plugins/fileplugin.py
+++ b/spyvm/plugins/fileplugin.py
@@ -1,19 +1,21 @@
 import os, stat, sys
 
-from rpython.rlib import jit
+from rpython.rlib import jit, rarithmetic
 from rpython.rlib.listsort import TimSort
 
 from spyvm import model, error
 from spyvm.plugins.plugin import Plugin
 from spyvm.primitives import PrimitiveFailedError, index1_0
 
-
 FilePlugin = Plugin()
 os.stat_float_times(False)
 
-std_fds = [sys.stdin.fileno(),
+try:
+    std_fds = [sys.stdin.fileno(),
            sys.stdout.fileno(),
            sys.stderr.fileno()]
+except ValueError:
+    std_fds = [0, 1, 2]
 
 @FilePlugin.expose_primitive(unwrap_spec=[object])
 def primitiveDirectoryDelimitor(interp, s_frame, w_rcvr):
@@ -44,7 +46,7 @@
     w_creationTime = smalltalk_timestamp(space, file_info.st_ctime)
     w_modificationTime = smalltalk_timestamp(space, file_info.st_mtime)
     w_dirFlag = space.w_true if stat.S_IFDIR & file_info.st_mode else space.w_false
-    w_fileSize = space.wrap_int(file_info.st_size)
+    w_fileSize = space.wrap_int(rarithmetic.intmask(file_info.st_size))
 
     return space.wrap_list([w_name, w_creationTime, w_modificationTime,
                             w_dirFlag, w_fileSize])
@@ -70,7 +72,6 @@
 
 @FilePlugin.expose_primitive(unwrap_spec=[object, int])
 def primitiveFileClose(interp, s_frame, w_rcvr, fd):
-    from rpython.rlib.rarithmetic import intmask
     try:
         os.close(fd)
     except OSError:
@@ -108,7 +109,7 @@
     except OSError:
         raise PrimitiveFailedError
     else:
-        return interp.space.wrap_int(pos)
+        return interp.space.wrap_positive_32bit_int(rarithmetic.intmask(pos))
 
 @FilePlugin.expose_primitive(unwrap_spec=[object, int, int])
 def primitiveFileSetPosition(interp, s_frame, w_rcvr, fd, position):
@@ -124,7 +125,7 @@
         file_info = os.fstat(fd)
     except OSError:
         raise PrimitiveFailedError
-    return interp.space.wrap_int(file_info.st_size)
+    return interp.space.wrap_positive_32bit_int(rarithmetic.intmask(file_info.st_size))
 
 @FilePlugin.expose_primitive(unwrap_spec=[object])
 def primitiveFileStdioHandles(interp, s_frame, w_rcvr):
diff --git a/spyvm/test/test_objectspace.py b/spyvm/test/test_objectspace.py
--- a/spyvm/test/test_objectspace.py
+++ b/spyvm/test/test_objectspace.py
@@ -55,5 +55,17 @@
     # byteobj.bytes.append('\x01')
     # num = space.unwrap_uint(byteobj)
     # should not raise. see docstring.
-  
 
+
+def test_wrap_int():
+    for num in [-10, 1, 15, 0x3fffffff]:
+        assert space.wrap_int(num).value == num
+
+    for num in [2L, -5L]:
+        with py.test.raises(AssertionError):
+            space.wrap_int(num)
+
+    from rpython.rlib.rarithmetic import intmask
+    for num in [0x7fffffff, intmask(0x80000000)]:
+        with py.test.raises(objspace.WrappingError):
+            space.wrap_int(num)

From noreply at buildbot.pypy.org  Tue Jun 18 16:04:26 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 18 Jun 2013 16:04:26 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added
	fileplugin-writefileprimitive
Message-ID: <20130618140426.D96E61C0619@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r464:1aa209705f3a
Date: 2013-06-18 15:58 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/1aa209705f3a/

Log:	added fileplugin-writefileprimitive

diff --git a/spyvm/plugins/fileplugin.py b/spyvm/plugins/fileplugin.py
--- a/spyvm/plugins/fileplugin.py
+++ b/spyvm/plugins/fileplugin.py
@@ -134,6 +134,19 @@
     space = interp.space
     return space.wrap_list([space.wrap_int(fd) for fd in std_fds])
 
+ at FilePlugin.expose_primitive(unwrap_spec=[object, int, str, index1_0, int])
+def primitiveFileWrite(interp, s_frame, w_rcvr, fd, a_string, start, count):
+    space = interp.space
+    end = min(start + 1 + count, len(a_string))
+    if not (start >= 0 and end > start):
+        return space.wrap_int(0)
+    try:
+        written = os.write(fd, a_string[start:end])
+    except OSError:
+        raise PrimitiveFailedError
+    else:
+        return space.wrap_positive_32bit_int(rarithmetic.intmask(written))
+
 @jit.elidable
 def smalltalk_timestamp(space, sec_since_epoch):
     import time

From noreply at buildbot.pypy.org  Tue Jun 18 19:28:24 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Jun 2013 19:28:24 +0200 (CEST)
Subject: [pypy-commit] stmgc default: passes
Message-ID: <20130618172824.E11E71C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r184:7bceb80ecbef
Date: 2013-06-18 10:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/7bceb80ecbef/

Log:	passes

diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -221,3 +221,30 @@
     assert p5 != p4
     assert p5 == lib.stm_write_barrier(p4)
     assert p5 == lib.stm_write_barrier(p5)
+
+def test_new_version_kill_intermediate_non_root():
+    p1 = oalloc_refs(1); make_public(p1)
+    p2 = oalloc(HDR);    make_public(p2)
+    p3 = oalloc(HDR);    make_public(p3)
+    p4 = oalloc(HDR);    make_public(p4)
+    p5 = oalloc(HDR);    make_public(p5)
+    delegate(p2, p3)
+    delegate(p3, p4)
+    delegate(p4, p5)
+    rawsetptr(p1, 0, p3)
+    assert rawgetptr(p1, 0) == p3
+    lib.stm_push_root(p1)
+    major_collect()
+    lib.stm_pop_root()
+    check_not_free(p1)
+    check_free_old(p2)
+    check_free_old(p3)
+    check_free_old(p4)
+    check_not_free(p5)
+    print 'p1:', p1
+    print '      containing:', rawgetptr(p1, 0)
+    print 'p2:', p2
+    print 'p3:', p3
+    print 'p4:', p4
+    print 'p5:', p5
+    assert rawgetptr(p1, 0) == p5

From noreply at buildbot.pypy.org  Tue Jun 18 19:28:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Jun 2013 19:28:26 +0200 (CEST)
Subject: [pypy-commit] stmgc default: minor stuff and a multi_thread test
	that still fails
Message-ID: <20130618172826.1D5621C07B6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r185:b82d04da5e98
Date: 2013-06-18 19:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/b82d04da5e98/

Log:	minor stuff and a multi_thread test that still fails

diff --git a/c4/doc-stmgc.txt b/c4/doc-stmgc.txt
--- a/c4/doc-stmgc.txt
+++ b/c4/doc-stmgc.txt
@@ -292,7 +292,7 @@
 as hinted above, we can compact chains of public object copies.
 
 Note that our design relies on the following property: in a given copy
-of an object which was committed at revision N, all pointers points to
+of an object which was committed at revision N, all pointers point to
 copies of objects which were committed at or before revision N.  This
 property is true by construction, but we must be careful not to break it
 by "optimizing" the content of a copy.  In particular, major collections
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -248,3 +248,9 @@
     print 'p4:', p4
     print 'p5:', p5
     assert rawgetptr(p1, 0) == p5
+
+def test_prebuilt_version_1():
+    p1 = lib.pseudoprebuilt(HDR, 42 + HDR)
+    check_prebuilt(p1)
+    major_collect()
+    check_prebuilt(p1)
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -505,6 +505,6 @@
 
 
 def test_more_multi_thread():
-    py.test.skip("more random tests")
-    for i in range(100):
-        yield test_multi_thread, 1200 + i
+    #py.test.skip("more random tests")
+    for i in range(200):
+        yield test_multi_thread, 1624 + i

From noreply at buildbot.pypy.org  Tue Jun 18 19:28:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Jun 2013 19:28:58 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Update
Message-ID: <20130618172858.ACB891C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r186:54e7428842a1
Date: 2013-06-18 19:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/54e7428842a1/

Log:	Update

diff --git a/README.txt b/README.txt
--- a/README.txt
+++ b/README.txt
@@ -18,7 +18,7 @@
 
 Run tests with "py.test".
 
-A demo program will be found in "c4/demo1.c" (not there yet).
+A demo program can be found in "c4/demo1.c".
 It can be built with "make debug-demo1" or "make build-demo1".
 
 The plan is to use this C code directly with PyPy, and not write

From noreply at buildbot.pypy.org  Tue Jun 18 20:36:40 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue, 18 Jun 2013 20:36:40 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: typo
Message-ID: <20130618183640.28E4D1C0268@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: extradoc
Changeset: r4976:0bcb68bccad2
Date: 2013-06-18 11:36 -0700
http://bitbucket.org/pypy/extradoc/changeset/0bcb68bccad2/

Log:	typo

diff --git a/blog/draft/py3k-status-update-11.rst b/blog/draft/py3k-status-update-11.rst
--- a/blog/draft/py3k-status-update-11.rst
+++ b/blog/draft/py3k-status-update-11.rst
@@ -52,7 +52,7 @@
 to provide a more fully-fledged initial release.
 
 Thanks to the following for their contributions: Manuel Jacob, Amaury Forgeot
-d'Arc, Karl Ramm, Jason Chu and Christian Hudson.
+d'Arc, Karl Ramm, Jason Chu and Christian Hudon.
 
 cheers,
 Phil

From noreply at buildbot.pypy.org  Tue Jun 18 21:05:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Jun 2013 21:05:49 +0200 (CEST)
Subject: [pypy-commit] stmgc fix-1: Going back to a pre-major-GC time,
	found an issue
Message-ID: <20130618190549.878271C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-1
Changeset: r187:58dccb421a8c
Date: 2013-06-18 20:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/58dccb421a8c/

Log:	Going back to a pre-major-GC time, found an issue

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -505,6 +505,6 @@
 
 
 def test_more_multi_thread():
-    py.test.skip("more random tests")
+    #py.test.skip("more random tests")
     for i in range(100):
-        yield test_multi_thread, 1200 + i
+        yield test_multi_thread, 1624 + i

From noreply at buildbot.pypy.org  Tue Jun 18 21:05:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Jun 2013 21:05:51 +0200 (CEST)
Subject: [pypy-commit] stmgc fix-1: Fix.  See "subtle".
Message-ID: <20130618190551.56C001C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-1
Changeset: r188:3e503e6cec3e
Date: 2013-06-18 21:03 +0200
http://bitbucket.org/pypy/stmgc/changeset/3e503e6cec3e/

Log:	Fix. See "subtle".

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -202,6 +202,7 @@
   gcptrlist_insert(&d->list_of_read_objects, P);
 
  add_in_recent_reads_cache:
+  assert(!(P->h_tid & GCFLAG_NURSERY_MOVED));
   fxcache_add(&d->recent_reads_cache, P);
   return P;
 
@@ -235,7 +236,8 @@
     }
 }
 
-static gcptr _match_public_to_private(gcptr P, gcptr pubobj, gcptr privobj)
+static gcptr _match_public_to_private(gcptr P, gcptr pubobj, gcptr privobj,
+                                      int from_stolen)
 {
   gcptr org_pubobj = pubobj;
   while ((pubobj->h_revision & 3) == 0)
@@ -251,6 +253,10 @@
       assert(is_private(privobj));
       if (P != org_pubobj)
         fprintf(stderr, "| actually %p ", org_pubobj);
+      if (from_stolen)
+        fprintf(stderr, "-stolen");
+      else
+        assert(org_pubobj->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
       fprintf(stderr, "-public_to_private-> %p private\n", privobj);
       return privobj;
     }
@@ -265,7 +271,8 @@
 
   G2L_LOOP_FORWARD(d->public_to_private, item)
     {
-      R = _match_public_to_private(P, item->addr, item->val);
+      assert(item->addr->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
+      R = _match_public_to_private(P, item->addr, item->val, 0);
       if (R != NULL)
         return R;
 
@@ -276,7 +283,9 @@
 
   for (i = 0; i < size; i += 2)
     {
-      R = _match_public_to_private(P, items[i], items[i + 1]);
+      if (items[i + 1] == NULL)
+        continue;
+      R = _match_public_to_private(P, items[i], items[i + 1], 1);
       if (R != NULL)
         return R;
     }
@@ -532,11 +541,12 @@
       */
       if (R->h_tid & GCFLAG_NURSERY_MOVED)
         {
-          /* Bah, the object turned into this kind of stub while we
-             were waiting for the collection_lock, because it was
-             stolen by someone else.  Use R->h_revision instead. */
+          /* Bah, the object turned into this kind of stub, possibly
+             while we were waiting for the collection_lock, because it
+             was stolen by someone else.  Use R->h_revision instead. */
           assert(IS_POINTER(R->h_revision));
           R = (gcptr)R->h_revision;
+          assert(R->h_tid & GCFLAG_PUBLIC);
         }
       assert(R->h_tid & GCFLAG_OLD);
       W = LocalizePublic(d, R);
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -153,6 +153,13 @@
             gcptr O = stmgc_duplicate_old(L);
             L->h_revision = (revision_t)O;
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
+            /* subtle: we need to remove L from the fxcache of the target
+               thread, otherwise its read barrier might not trigger on it.
+               It is mostly fine because it is anyway identical to O.  But
+               the issue is if the target thread adds a public_to_private
+               off O later: the write barrier will miss it if it only sees
+               L. */
+            gcptrlist_insert2(&foreign_pd->stolen_objects, L, NULL);
             L = O;
             fprintf(stderr, "\t---> %p\n", L);
         }
@@ -216,17 +223,20 @@
 
     for (i = 0; i < size; i += 2) {
         gcptr B = items[i];
+        assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
+
+        /* to be on the safe side --- but actually needed, see the
+           gcptrlist_insert2(L, NULL) above */
+        fxcache_remove(&d->recent_reads_cache, B);
+
         gcptr L = items[i + 1];
-
+        if (L == NULL)
+            continue;
         assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
-        assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
 
         g2l_insert(&d->public_to_private, B, L);
 
-        /* to be on the safe side */
-        fxcache_remove(&d->recent_reads_cache, B);
-
-        /* but this is definitely needed: all keys in public_to_private
+        /* this is definitely needed: all keys in public_to_private
            must appear in list_of_read_objects */
         fprintf(stderr, "n.readobj: %p\n", B);
         gcptrlist_insert(&d->list_of_read_objects, B);
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -77,6 +77,9 @@
 
 gcptr stm_read_barrier(gcptr obj)
 {
+    //if (FXCACHE_AT(obj) == obj)
+    //    fprintf(stderr, "read_barrier: in cache: %p\n", obj);
+
     /* XXX inline in the caller, optimize to get the smallest code */
     if (UNLIKELY((obj->h_revision != stm_private_rev_num) &&
                  (FXCACHE_AT(obj) != obj)))

From noreply at buildbot.pypy.org  Tue Jun 18 21:05:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Jun 2013 21:05:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: hg merge fix-1
Message-ID: <20130618190552.CFD821C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r189:d849db4efe62
Date: 2013-06-18 21:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/d849db4efe62/

Log:	hg merge fix-1

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -202,6 +202,7 @@
   gcptrlist_insert(&d->list_of_read_objects, P);
 
  add_in_recent_reads_cache:
+  assert(!(P->h_tid & GCFLAG_NURSERY_MOVED));
   fxcache_add(&d->recent_reads_cache, P);
   return P;
 
@@ -235,7 +236,8 @@
     }
 }
 
-static gcptr _match_public_to_private(gcptr P, gcptr pubobj, gcptr privobj)
+static gcptr _match_public_to_private(gcptr P, gcptr pubobj, gcptr privobj,
+                                      int from_stolen)
 {
   gcptr org_pubobj = pubobj;
   while ((pubobj->h_revision & 3) == 0)
@@ -251,6 +253,10 @@
       assert(is_private(privobj));
       if (P != org_pubobj)
         fprintf(stderr, "| actually %p ", org_pubobj);
+      if (from_stolen)
+        fprintf(stderr, "-stolen");
+      else
+        assert(org_pubobj->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
       fprintf(stderr, "-public_to_private-> %p private\n", privobj);
       return privobj;
     }
@@ -265,7 +271,8 @@
 
   G2L_LOOP_FORWARD(d->public_to_private, item)
     {
-      R = _match_public_to_private(P, item->addr, item->val);
+      assert(item->addr->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
+      R = _match_public_to_private(P, item->addr, item->val, 0);
       if (R != NULL)
         return R;
 
@@ -276,7 +283,9 @@
 
   for (i = 0; i < size; i += 2)
     {
-      R = _match_public_to_private(P, items[i], items[i + 1]);
+      if (items[i + 1] == NULL)
+        continue;
+      R = _match_public_to_private(P, items[i], items[i + 1], 1);
       if (R != NULL)
         return R;
     }
@@ -532,11 +541,12 @@
       */
       if (R->h_tid & GCFLAG_NURSERY_MOVED)
         {
-          /* Bah, the object turned into this kind of stub while we
-             were waiting for the collection_lock, because it was
-             stolen by someone else.  Use R->h_revision instead. */
+          /* Bah, the object turned into this kind of stub, possibly
+             while we were waiting for the collection_lock, because it
+             was stolen by someone else.  Use R->h_revision instead. */
           assert(IS_POINTER(R->h_revision));
           R = (gcptr)R->h_revision;
+          assert(R->h_tid & GCFLAG_PUBLIC);
         }
       assert(R->h_tid & GCFLAG_OLD);
       W = LocalizePublic(d, R);
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -154,6 +154,13 @@
             gcptr O = stmgc_duplicate_old(L);
             L->h_revision = (revision_t)O;
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
+            /* subtle: we need to remove L from the fxcache of the target
+               thread, otherwise its read barrier might not trigger on it.
+               It is mostly fine because it is anyway identical to O.  But
+               the issue is if the target thread adds a public_to_private
+               off O later: the write barrier will miss it if it only sees
+               L. */
+            gcptrlist_insert2(&foreign_pd->stolen_objects, L, NULL);
             L = O;
             fprintf(stderr, "\t---> %p\n", L);
         }
@@ -217,17 +224,20 @@
 
     for (i = 0; i < size; i += 2) {
         gcptr B = items[i];
+        assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
+
+        /* to be on the safe side --- but actually needed, see the
+           gcptrlist_insert2(L, NULL) above */
+        fxcache_remove(&d->recent_reads_cache, B);
+
         gcptr L = items[i + 1];
-
+        if (L == NULL)
+            continue;
         assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
-        assert(!(B->h_tid & GCFLAG_BACKUP_COPY));  /* already removed */
 
         g2l_insert(&d->public_to_private, B, L);
 
-        /* to be on the safe side */
-        fxcache_remove(&d->recent_reads_cache, B);
-
-        /* but this is definitely needed: all keys in public_to_private
+        /* this is definitely needed: all keys in public_to_private
            must appear in list_of_read_objects */
         fprintf(stderr, "n.readobj: %p\n", B);
         gcptrlist_insert(&d->list_of_read_objects, B);
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -77,6 +77,9 @@
 
 gcptr stm_read_barrier(gcptr obj)
 {
+    //if (FXCACHE_AT(obj) == obj)
+    //    fprintf(stderr, "read_barrier: in cache: %p\n", obj);
+
     /* XXX inline in the caller, optimize to get the smallest code */
     if (UNLIKELY((obj->h_revision != stm_private_rev_num) &&
                  (FXCACHE_AT(obj) != obj)))

From noreply at buildbot.pypy.org  Tue Jun 18 21:07:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Jun 2013 21:07:15 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Skip this again. Moved the problematic
 iteration into its own test.
Message-ID: <20130618190715.DD6E41C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r190:011a0cd9be1e
Date: 2013-06-18 21:07 +0200
http://bitbucket.org/pypy/stmgc/changeset/011a0cd9be1e/

Log:	Skip this again. Moved the problematic iteration into its own test.

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -504,7 +504,10 @@
     assert fine == [True, True]
 
 
+def test_specific_issue_1():
+    test_multi_thread(1624)
+
 def test_more_multi_thread():
-    #py.test.skip("more random tests")
+    py.test.skip("more random tests")
     for i in range(200):
         yield test_multi_thread, 1624 + i

From noreply at buildbot.pypy.org  Wed Jun 19 02:17:31 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 19 Jun 2013 02:17:31 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Add a test for the multi_index
	flag
Message-ID: <20130619001731.D5D901C0619@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64932:7245e102beb5
Date: 2013-06-19 02:08 +0200
http://bitbucket.org/pypy/pypy/changeset/7245e102beb5/

Log:	Add a test for the multi_index flag

diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -248,4 +248,22 @@
         assert (it.operands[1] == [[6, 22, 38], [54, 70, 86]]).all()
         assert (it.operands[1] == a.sum(axis=2)).all()
 
+    def test_get_dtypes(self):
+        from numpypy import array, dtype, nditer
+        x = array([1, 2])
+        y = array([1.0, 2.0])
+        assert nditer([x, y]).dtypes == (dtype("int64"), dtype("float64"))
 
+    def test_multi_index(self):
+        import numpypy as np
+
+        a = np.arange(6).reshape(2, 3)
+
+        it = np.nditer(a, flags=['multi_index'])
+
+        res = []
+        while not it.finished:
+            res.append((it[0], it.multi_index))
+            it.iternext()
+
+        assert res == [(0, (0, 0)), (1, (0, 1)), (2, (0, 2)), (3, (1, 0)), (4, (1, 1)), (5, (1, 2))]

From noreply at buildbot.pypy.org  Wed Jun 19 02:17:33 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 19 Jun 2013 02:17:33 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Implement the multi_index flag
Message-ID: <20130619001733.5D6FC1C0619@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64933:00c6b14ae653
Date: 2013-06-19 02:09 +0200
http://bitbucket.org/pypy/pypy/changeset/00c6b14ae653/

Log:	Implement the multi_index flag

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -274,7 +274,11 @@
         if self.tracked_index != "":
             if self.order == "K":
                 self.order = self.seq[0].implementation.order
-            self.index_iter = IndexIterator(iter_shape, backward=self.order != self.tracked_index)
+            if self.tracked_index == "multi":
+                backward = False
+            else:
+                backward = self.order != self.tracked_index
+            self.index_iter = IndexIterator(iter_shape, backward=backward)
         if self.external_loop:
             for i in range(len(self.seq)):
                 self.iters.append(ExternalLoopIterator(get_external_loop_iter(space, self.order,
@@ -300,8 +304,7 @@
             'not implemented yet'))
 
     def descr_len(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        space.wrap(len(self.iters))
 
     def descr_next(self, space):
         for it in self.iters:
@@ -369,8 +372,10 @@
             'not implemented yet'))
 
     def descr_get_dtypes(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        res = [None] * len(self.seq)
+        for i in range(len(self.seq)):
+            res[i] = self.seq[i].descr_get_dtype(space)
+        return space.newtuple(res)
 
     def descr_get_finished(self, space):
         return space.wrap(self.done)
@@ -380,22 +385,24 @@
             'not implemented yet'))
 
     def descr_get_has_index(self, space):
-        return space.wrap(not self.tracked_index == "")
+        return space.wrap(self.tracked_index in ["C", "F"])
 
     def descr_get_index(self, space):
-        if self.tracked_index == "":
+        if not self.tracked_index in ["C", "F"]:
             raise OperationError(space.w_ValueError, space.wrap("Iterator does not have an index"))
         if self.done:
             raise OperationError(space.w_ValueError, space.wrap("Iterator is past the end"))
         return space.wrap(self.index_iter.getvalue())
 
     def descr_get_has_multi_index(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        return space.wrap(self.tracked_index == "multi")
 
     def descr_get_multi_index(self, space):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            'not implemented yet'))
+        if not self.tracked_index == "multi":
+            raise OperationError(space.w_ValueError, space.wrap("Iterator is not tracking a multi-index"))
+        if self.done:
+            raise OperationError(space.w_ValueError, space.wrap("Iterator is past the end"))
+        return space.newtuple([space.wrap(x) for x in self.index_iter.index])
 
     def descr_get_iterationneedsapi(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(

From noreply at buildbot.pypy.org  Wed Jun 19 02:18:53 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 19 Jun 2013 02:18:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: special case nan containment
Message-ID: <20130619001853.4E6891C0619@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r64934:a052f153e311
Date: 2013-06-18 17:11 -0700
http://bitbucket.org/pypy/pypy/changeset/a052f153e311/

Log:	special case nan containment

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1222,8 +1222,7 @@
     def _safe_find(self, w_list, obj, start, stop):
         l = self.unerase(w_list.lstorage)
         for i in range(start, min(stop, len(l))):
-            val = l[i]
-            if val == obj:
+            if l[i] == obj:
                 return i
         raise ValueError
 
@@ -1543,6 +1542,18 @@
         if reverse:
             l.reverse()
 
+    def _safe_find(self, w_list, obj, start, stop):
+        from rpython.rlib.rfloat import isnan
+        if not isnan(obj):
+            return AbstractUnwrappedStrategy._safe_find(self, w_list, obj,
+                                                        start, stop)
+        # unwrapped nan != nan, finding it requires more effort
+        l = self.unerase(w_list.lstorage)
+        for i in range(start, min(stop, len(l))):
+            if isnan(l[i]):
+                return i
+        raise ValueError
+
 
 class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = None
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1314,6 +1314,10 @@
         non_list = NonList()
         assert [] != non_list
 
+    def test_nan_containment(self):
+        nan = float('nan')
+        assert nan in [nan]
+
 
 class AppTestForRangeLists(AppTestW_ListObject):
     spaceconfig = {"objspace.std.withrangelist": True}

From noreply at buildbot.pypy.org  Wed Jun 19 02:18:54 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 19 Jun 2013 02:18:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130619001854.A34001C0619@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64935:d854f8325e81
Date: 2013-06-18 17:18 -0700
http://bitbucket.org/pypy/pypy/changeset/d854f8325e81/

Log:	merge default

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1201,8 +1201,7 @@
     def _safe_find(self, w_list, obj, start, stop):
         l = self.unerase(w_list.lstorage)
         for i in range(start, min(stop, len(l))):
-            val = l[i]
-            if val == obj:
+            if l[i] == obj:
                 return i
         raise ValueError
 
@@ -1522,6 +1521,18 @@
         if reverse:
             l.reverse()
 
+    def _safe_find(self, w_list, obj, start, stop):
+        from rpython.rlib.rfloat import isnan
+        if not isnan(obj):
+            return AbstractUnwrappedStrategy._safe_find(self, w_list, obj,
+                                                        start, stop)
+        # unwrapped nan != nan, finding it requires more effort
+        l = self.unerase(w_list.lstorage)
+        for i in range(start, min(stop, len(l))):
+            if isnan(l[i]):
+                return i
+        raise ValueError
+
 
 class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = None
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1268,6 +1268,16 @@
     def test_use_method_for_wrong_object(self):
         raises(TypeError, list.append, 1, 2)
 
+    def test_ne_NotImplemented(self):
+        class NonList(object):
+            pass
+        non_list = NonList()
+        assert [] != non_list
+
+    def test_nan_containment(self):
+        nan = float('nan')
+        assert nan in [nan]
+
     def test_issue1266(self):
         l = list(range(1))
         l.pop()

From noreply at buildbot.pypy.org  Wed Jun 19 09:21:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 09:21:58 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Can't assert this
Message-ID: <20130619072158.4AFB91C07E1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r191:ba37dce6e637
Date: 2013-06-19 09:21 +0200
http://bitbucket.org/pypy/stmgc/changeset/ba37dce6e637/

Log:	Can't assert this

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -202,7 +202,9 @@
   gcptrlist_insert(&d->list_of_read_objects, P);
 
  add_in_recent_reads_cache:
-  assert(!(P->h_tid & GCFLAG_NURSERY_MOVED));
+  /* The risks are that the following assert fails, because the flag was
+     added just now by a parallel thread during stealing... */
+  /*assert(!(P->h_tid & GCFLAG_NURSERY_MOVED));*/
   fxcache_add(&d->recent_reads_cache, P);
   return P;
 

From noreply at buildbot.pypy.org  Wed Jun 19 09:49:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 09:49:45 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Remove a bogus assert: it's possible
 that older, protected objects were
Message-ID: <20130619074945.4E0E21C034A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r192:70b5ecf4fa2f
Date: 2013-06-19 09:49 +0200
http://bitbucket.org/pypy/stmgc/changeset/70b5ecf4fa2f/

Log:	Remove a bogus assert: it's possible that older, protected objects
	were stolen in parallel.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -847,7 +847,6 @@
   assert(d->num_private_from_protected_known_old == 0);
   assert(d->num_read_objects_known_old == 0);
   assert(!g2l_any_entry(&d->public_to_private));
-  assert(d->public_descriptor->stolen_objects.size == 0);
 
   d->count_reads = 1;
   fxcache_clear(&d->recent_reads_cache);

From noreply at buildbot.pypy.org  Wed Jun 19 10:17:34 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 10:17:34 +0200 (CEST)
Subject: [pypy-commit] cffi default: Attack issue 91; first step.
Message-ID: <20130619081734.98B571C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1271:3306e1a851ae
Date: 2013-06-19 09:45 +0200
http://bitbucket.org/cffi/cffi/changeset/3306e1a851ae/

Log:	Attack issue 91; first step.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3351,22 +3351,18 @@
     return (PyObject *)td;
 }
 
-static PyObject *_b_struct_or_union_type(const char *kind, const char *name,
-                                         int flag)
+static PyObject *_b_struct_or_union_type(const char *name, int flag)
 {
-    int kindlen = strlen(kind);
     int namelen = strlen(name);
-    CTypeDescrObject *td = ctypedescr_new(kindlen + 1 + namelen + 1);
+    CTypeDescrObject *td = ctypedescr_new(namelen + 1);
     if (td == NULL)
         return NULL;
 
     td->ct_size = -1;
     td->ct_length = -1;
     td->ct_flags = flag | CT_IS_OPAQUE;
-    memcpy(td->ct_name, kind, kindlen);
-    td->ct_name[kindlen] = ' ';
-    memcpy(td->ct_name + kindlen + 1, name, namelen + 1);
-    td->ct_name_position = kindlen + 1 + namelen;
+    memcpy(td->ct_name, name, namelen + 1);
+    td->ct_name_position = namelen;
     return (PyObject *)td;
 }
 
@@ -3378,9 +3374,9 @@
         return NULL;
 
     flag = CT_STRUCT;
-    if (strcmp(name, "_IO_FILE") == 0 || strcmp(name, "$FILE") == 0)
+    if (strcmp(name, "struct _IO_FILE") == 0 || strcmp(name, "FILE") == 0)
         flag |= CT_IS_FILE;
-    return _b_struct_or_union_type("struct", name, flag);
+    return _b_struct_or_union_type(name, flag);
 }
 
 static PyObject *b_new_union_type(PyObject *self, PyObject *args)
@@ -3388,7 +3384,7 @@
     char *name;
     if (!PyArg_ParseTuple(args, "s:new_union_type", &name))
         return NULL;
-    return _b_struct_or_union_type("union", name, CT_UNION);
+    return _b_struct_or_union_type(name, CT_UNION);
 }
 
 static CFieldObject *
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -652,6 +652,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -659,7 +661,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -668,7 +670,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -700,7 +702,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -719,7 +721,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -739,7 +741,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -755,7 +757,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -769,7 +771,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -803,7 +805,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -814,7 +816,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -843,7 +845,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -948,7 +950,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -964,7 +966,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -976,7 +978,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -996,7 +998,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1034,7 +1036,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1059,7 +1061,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1232,7 +1234,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1249,7 +1251,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1339,7 +1341,7 @@
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
     BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1443,7 +1445,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1466,7 +1468,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1496,14 +1498,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1572,7 +1574,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1612,7 +1614,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1630,7 +1632,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1708,7 +1710,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1725,7 +1727,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1736,7 +1738,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1746,7 +1748,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1756,7 +1758,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1765,7 +1767,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1777,7 +1779,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1786,7 +1788,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1796,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1806,7 +1808,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1829,7 +1831,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1856,7 +1858,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1968,7 +1970,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1988,7 +1990,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2175,7 +2177,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2194,7 +2196,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2203,7 +2205,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2319,7 +2321,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2331,8 +2333,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2364,12 +2366,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2377,7 +2379,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2435,7 +2437,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2451,14 +2453,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2519,7 +2521,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2551,7 +2553,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2574,7 +2576,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2773,7 +2775,7 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
@@ -2786,7 +2788,7 @@
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2798,7 +2800,7 @@
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),

From noreply at buildbot.pypy.org  Wed Jun 19 10:17:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 10:17:35 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fix the front-end to work (but still
	show $ signs)
Message-ID: <20130619081735.B8BAE1C07E1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1272:613cd8c50b9f
Date: 2013-06-19 09:50 +0200
http://bitbucket.org/cffi/cffi/changeset/613cd8c50b9f/

Log:	Fix the front-end to work (but still show $ signs)

diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -707,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -357,7 +357,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_struct_type',
+                            'struct ' + self.name, key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +366,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            'union ' + self.name, key=self)
 
 
 class EnumType(StructOrUnionOrEnum):

From noreply at buildbot.pypy.org  Wed Jun 19 10:17:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 10:17:36 +0200 (CEST)
Subject: [pypy-commit] cffi default: Document ffi.new_handle().
Message-ID: <20130619081736.E635D1C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1273:e98e87d25d7d
Date: 2013-06-19 10:15 +0200
http://bitbucket.org/cffi/cffi/changeset/e98e87d25d7d/

Log:	Document ffi.new_handle().

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -1186,6 +1186,18 @@
 
 .. "versionadded:: 0.3" --- inlined in the previous paragraph
 
+``ffi.new_handle(python_object)``: return a non-NULL cdata of type
+``void *`` that contains an opaque reference to ``python_object``.  You
+can pass it around to C functions or store it into C structures.  Later,
+you can use ``ffi.from_handle(p)`` to retrive the original
+``python_object`` from a value with the same ``void *`` pointer.  The
+cdata object returned by ``new_handle()`` has *ownership*, in the same
+sense as ``ffi.new()`` or ``ffi.gc()``: the association ``void * ->
+python_object`` is only valid as long as *this* exact cdata returned by
+``new_handle()`` is alive.  *New in version 0.7.*
+
+.. "versionadded:: 0.7" --- inlined in the previous paragraph
+
 ``ffi.addressof(cdata, field=None)``: from a cdata whose type is
 ``struct foo_s``, return its "address", as a cdata whose type is
 ``struct foo_s *``.  Also works on unions, but not on any other type.

From noreply at buildbot.pypy.org  Wed Jun 19 10:17:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 10:17:38 +0200 (CEST)
Subject: [pypy-commit] cffi default: merge heads
Message-ID: <20130619081738.03E851C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1274:52dcd2ee1de1
Date: 2013-06-19 10:17 +0200
http://bitbucket.org/cffi/cffi/changeset/52dcd2ee1de1/

Log:	merge heads

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3421,6 +3421,7 @@
 }
 
 #define SF_MSVC_BITFIELDS 1
+#define SF_GCC_ARM_BITFIELDS 2
 
 static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args)
 {
@@ -3435,7 +3436,11 @@
 #ifdef MS_WIN32
     int sflags = SF_MSVC_BITFIELDS;
 #else
+# ifdef __arm__
+    int sflags = SF_GCC_ARM_BITFIELDS;
+# else
     int sflags = 0;
+# endif
 #endif
 
     if (!PyArg_ParseTuple(args, "O!O!|Onii:complete_struct_or_union",
@@ -3499,8 +3504,7 @@
             goto error;
 
         do_align = 1;
-#ifndef __arm__
-        if (fbitsize >= 0) {
+        if (!(sflags & SF_GCC_ARM_BITFIELDS) && fbitsize >= 0) {
             if (!(sflags & SF_MSVC_BITFIELDS)) {
                 /* GCC: anonymous bitfields (of any size) don't cause alignment */
                 do_align = PyText_GetSize(fname) > 0;
@@ -3510,7 +3514,6 @@
                 do_align = fbitsize > 0;
             }
         }
-#endif
         if (alignment < falign && do_align)
             alignment = falign;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2780,10 +2780,10 @@
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
@@ -2796,7 +2796,10 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
@@ -2808,10 +2811,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2820,6 +2828,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy

From noreply at buildbot.pypy.org  Wed Jun 19 10:29:55 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 10:29:55 +0200 (CEST)
Subject: [pypy-commit] cffi default: Finish the fix for issue #91.
Message-ID: <20130619082955.1B2CF1C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1275:1d4ca4f4914a
Date: 2013-06-19 10:29 +0200
http://bitbucket.org/cffi/cffi/changeset/1d4ca4f4914a/

Log:	Finish the fix for issue #91.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4459,13 +4459,12 @@
     Py_CLEAR(dict2);
     Py_CLEAR(dict1);
 
-    name_size = strlen("enum ") + strlen(ename) + 1;
+    name_size = strlen(ename) + 1;
     td = ctypedescr_new(name_size);
     if (td == NULL)
         goto error;
 
-    memcpy(td->ct_name, "enum ", strlen("enum "));
-    memcpy(td->ct_name + strlen("enum "), ename, name_size - strlen("enum "));
+    memcpy(td->ct_name, ename, name_size);
     td->ct_stuff = combined;
     td->ct_size = basetd->ct_size;
     td->ct_length = basetd->ct_length;   /* alignment */
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1288,26 +1288,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1321,26 +1322,26 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -934,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,8 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
+        
         return global_cache(self, ffi, 'new_struct_type',
-                            'struct ' + self.name, key=self)
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -367,7 +372,7 @@
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
         return global_cache(self, ffi, 'new_union_type',
-                            'union ' + self.name, key=self)
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -390,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/testing/backend_tests.py b/testing/backend_tests.py
--- a/testing/backend_tests.py
+++ b/testing/backend_tests.py
@@ -989,7 +989,7 @@
         assert f.a == 12345
         assert b.b == b"B"
         assert b.c == b"C"
-        assert repr(b).startswith("<cdata 'struct $bar_t *'")
+        assert repr(b).startswith("<cdata 'bar_t *'")
 
     def test_struct_with_two_usages(self):
         for name in ['foo_s', '']:    # anonymous or not
@@ -1299,9 +1299,9 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("typedef enum { Value0 = 0 } e, *pe;\n"
                  "typedef enum { Value1 = 1 } e1;")
-        assert ffi.getctype("e*") == 'enum $e *'
-        assert ffi.getctype("pe") == 'enum $e *'
-        assert ffi.getctype("e1*") == 'enum $e1 *'
+        assert ffi.getctype("e*") == 'e *'
+        assert ffi.getctype("pe") == 'e *'
+        assert ffi.getctype("e1*") == 'e1 *'
 
     def test_new_ctype(self):
         ffi = FFI(backend=self.Backend())
diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py
--- a/testing/test_ffi_backend.py
+++ b/testing/test_ffi_backend.py
@@ -178,3 +178,9 @@
             'ffi.cdef("struct s2 { char x:0; };"); ffi.new("struct s2 *")')
         py.test.raises(TypeError,
             'ffi.cdef("struct s3 { char x:9; };"); ffi.new("struct s3 *")')
+
+    def test_struct_with_typedef(self):
+        ffi = FFI()
+        ffi.cdef("typedef struct { float x; } foo_t;")
+        p = ffi.new("foo_t *", [5.2])
+        assert repr(p).startswith("<cdata 'foo_t *' ")
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -1441,7 +1441,7 @@
                        "foo_t myfunc(void) { foo_t x = { 42 }; return x; }")
     assert str(e.value) in [
         "function myfunc: 'foo_t' is used as result type, but is opaque",
-        "function myfunc: result type 'struct $foo_t' is opaque"]
+        "function myfunc: result type 'foo_t' is opaque"]
 
 def test_include():
     ffi1 = FFI()

From noreply at buildbot.pypy.org  Wed Jun 19 10:35:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 10:35:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Import and fix cffi/1d4ca4f4914a.
Message-ID: <20130619083514.B9A3C1C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64936:094ca61ae02f
Date: 2013-06-19 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/094ca61ae02f/

Log:	Import and fix cffi/1d4ca4f4914a.

diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -13,7 +13,6 @@
     _mixin_ = True
 
     def __init__(self, space, name, size, align, enumerators, enumvalues):
-        name = "enum " + name
         self._super.__init__(self, space, size, name, len(name), align)
         self.enumerators2values = {}   # str -> int
         self.enumvalues2erators = {}   # int -> str
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         else:
             extra = " *"
         self.is_file = (ctitem.name == "struct _IO_FILE" or
-                        ctitem.name == "struct $FILE")
+                        ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -25,7 +25,6 @@
     custom_field_pos = False
 
     def __init__(self, space, name):
-        name = '%s %s' % (self.kind, name)
         W_CType.__init__(self, space, -1, name, len(name))
 
     def check_complete(self, w_errorcls=None):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -641,6 +641,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -648,7 +650,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -657,7 +659,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -689,7 +691,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -708,7 +710,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -728,7 +730,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -744,7 +746,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -758,7 +760,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -792,7 +794,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -803,7 +805,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -832,7 +834,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -937,7 +939,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -953,7 +955,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -965,7 +967,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -985,7 +987,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1023,7 +1025,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1048,7 +1050,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1221,7 +1223,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1238,7 +1240,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1275,26 +1277,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1308,27 +1311,27 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1432,7 +1435,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1455,7 +1458,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1485,14 +1488,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1561,7 +1564,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1601,7 +1604,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1619,7 +1622,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1697,7 +1700,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1714,7 +1717,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1725,7 +1728,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1735,7 +1738,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1745,7 +1748,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1754,7 +1757,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1766,7 +1769,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1775,7 +1778,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1785,7 +1788,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1795,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1818,7 +1821,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1845,7 +1848,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1957,7 +1960,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1977,7 +1980,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2164,7 +2167,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2183,7 +2186,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2192,7 +2195,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2308,7 +2311,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2320,8 +2323,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2353,12 +2356,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2366,7 +2369,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2424,7 +2427,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2440,14 +2443,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2508,7 +2511,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2540,7 +2543,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2563,7 +2566,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2762,7 +2765,7 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
@@ -2775,7 +2778,7 @@
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2790,7 +2793,7 @@
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),

From noreply at buildbot.pypy.org  Wed Jun 19 10:40:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 10:40:05 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130619084005.C30AC1C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r193:bb14332bb658
Date: 2013-06-19 10:39 +0200
http://bitbucket.org/pypy/stmgc/changeset/bb14332bb658/

Log:	Fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -776,7 +776,6 @@
   AbortPrivateFromProtected(d);
   gcptrlist_clear(&d->list_of_read_objects);
   g2l_clear(&d->public_to_private);
-  gcptrlist_clear(&d->public_descriptor->stolen_objects);
 
   /* release the lock */
   spinlock_release(d->public_descriptor->collection_lock);
@@ -1453,11 +1452,12 @@
     struct tx_descriptor *d = thread_descriptor;
     assert(d != NULL);
     assert(d->active == 0);
-    assert(d->public_descriptor->stolen_objects.size == 0);
-    gcptrlist_delete(&d->public_descriptor->stolen_objects);
+    stmgcpage_acquire_global_lock();
+
+    /* our nursery is empty at this point */
+    assert(d->public_descriptor->stolen_young_stubs.size == 0);
     gcptrlist_delete(&d->public_descriptor->stolen_young_stubs);
 
-    stmgcpage_acquire_global_lock();
     stmgcpage_done_tls();
     i = d->public_descriptor_index;
     assert(stm_descriptor_array[i] == d->public_descriptor);

From noreply at buildbot.pypy.org  Wed Jun 19 14:32:53 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 19 Jun 2013 14:32:53 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add demo2 (bubble sort)
Message-ID: <20130619123253.025821C308F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r194:716b7629c9cd
Date: 2013-06-19 11:20 +0200
http://bitbucket.org/pypy/stmgc/changeset/716b7629c9cd/

Log:	add demo2 (bubble sort)

diff --git a/c4/demo2.c b/c4/demo2.c
new file mode 100644
--- /dev/null
+++ b/c4/demo2.c
@@ -0,0 +1,218 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <pthread.h>
+#include <semaphore.h>
+
+#include "stmgc.h"
+#include "fprintcolor.h"
+
+
+#define LIST_LENGTH 500
+#define NUMTHREADS  4
+
+
+#define GCTID_STRUCT_NODE     123
+
+struct node {
+    struct stm_object_s hdr;
+    long value;
+    struct node *next;
+};
+typedef struct node * nodeptr;
+
+size_t stmcb_size(gcptr ob)
+{
+    assert(stm_get_tid(ob) == GCTID_STRUCT_NODE);
+    return sizeof(struct node);
+}
+
+void stmcb_trace(gcptr ob, void visit(gcptr *))
+{
+    nodeptr n;
+    assert(stm_get_tid(ob) == GCTID_STRUCT_NODE);
+    n = (nodeptr)ob;
+    visit((gcptr *)&n->next);
+}
+
+
+struct node global_chained_list = {
+    { GCTID_STRUCT_NODE | PREBUILT_FLAGS, PREBUILT_REVISION },
+    -1,
+    NULL,
+};
+
+
+long check_sorted()
+{
+    nodeptr r_n;
+    long prev, sum;
+    r_n = (nodeptr)stm_read_barrier((gcptr)&global_chained_list);
+    assert(r_n->value == -1);
+    
+    prev = -1;
+    sum = 0;
+    while (r_n->next) {
+        r_n = (nodeptr)stm_read_barrier((gcptr)r_n->next);
+        sum += r_n->value;
+        
+        if (prev >= r_n->value)
+            return -1;
+        
+        prev = r_n->value;
+    }
+    
+    return sum;
+}
+
+void swap_nodes(nodeptr prev, nodeptr current, nodeptr next)
+{
+    nodeptr w_prev, w_current, w_next;
+    w_prev = (nodeptr)stm_write_barrier((gcptr)prev);
+    w_current = (nodeptr)stm_write_barrier((gcptr)current);
+    w_next = (nodeptr)stm_write_barrier((gcptr)next);
+    
+    w_prev->next = w_next;
+    w_current->next = w_next->next;
+    w_next->next = w_current;
+}
+
+int bubble_run(gcptr arg1, int retry_counter)
+{
+    nodeptr r_prev, r_current, r_next, tmp;
+    
+    r_prev = (nodeptr)stm_read_barrier(arg1);
+    r_current = (nodeptr)stm_read_barrier((gcptr)r_prev->next);
+    r_next = (nodeptr)stm_read_barrier((gcptr)r_current->next);
+    
+    while (r_next) {
+        if (r_next->value < r_current->value) {
+            // swap current and next
+            swap_nodes(r_prev, r_current, r_next);	  
+            fprintf(stdout, "#");
+            
+            // needs read barriers, because of write barriers in swap_nodes
+            r_prev = (struct node*)stm_read_barrier((gcptr)r_prev);
+            tmp = r_current;
+            r_current = (struct node*)stm_read_barrier((gcptr)r_next);
+            r_next = (struct node*)stm_read_barrier((gcptr)tmp);
+        }
+        // results from consecutive read_barriers can differ. needs Ptr_Eq()
+        /* assert(stm_read_barrier((gcptr)r_prev->next) == r_current */
+        /* 	     && stm_read_barrier((gcptr)r_current->next) == r_next); */
+        // for now:
+        assert(((nodeptr)stm_read_barrier((gcptr)r_prev->next))->value 
+               == r_current->value
+               && 
+               ((nodeptr)stm_read_barrier((gcptr)r_current->next))->value
+               == r_next->value);
+        
+        r_prev = r_current;
+        r_current = r_next;
+        r_next = r_next->next;
+        if (r_next != NULL)
+            r_next = (nodeptr)stm_read_barrier((gcptr)r_next);
+    }
+    
+    
+    return 0;
+}
+
+
+static sem_t done;
+
+static int thr_mynum = 0;
+
+void *demo2(void *arg)
+{
+    
+    int status;
+    stm_initialize();
+    
+    thr_mynum++;   /* protected by being inevitable here */
+    fprintf(stderr, "THREAD STARTING\n");
+    
+    
+    while (check_sorted() == -1) {
+        stm_perform_transaction((gcptr)&global_chained_list, bubble_run);
+    }
+    
+    stm_finalize();
+    
+    status = sem_post(&done);
+    assert(status == 0);
+    return NULL;
+}
+
+void final_check(void)
+{
+    long sum;
+    
+    stm_initialize();
+    
+    sum = check_sorted();
+    
+    // little Gauss:
+    assert(sum == (1 + LIST_LENGTH) * (LIST_LENGTH / 2));
+    
+    stm_finalize();
+    printf("check ok\n");
+}
+
+
+void newthread(void*(*func)(void*), void *arg)
+{
+    pthread_t th;
+    int status = pthread_create(&th, NULL, func, arg);
+    assert(status == 0);
+    pthread_detach(th);
+    printf("started new thread\n");
+}
+
+
+/* initialize list with values in decreasing order */
+void setup_list()
+{
+    int i;
+    nodeptr w_newnode, w_prev;
+    stm_initialize();
+    
+    w_prev = &global_chained_list;
+    for (i = 0; i < LIST_LENGTH; i++) {
+        stm_push_root((gcptr)w_prev);
+        w_newnode = (nodeptr)stm_allocate(sizeof(struct node),
+                                          GCTID_STRUCT_NODE);
+        w_prev = (nodeptr)stm_pop_root();
+        w_newnode->value = LIST_LENGTH - i;
+        w_newnode->next = NULL;
+        
+        w_prev = (nodeptr)stm_write_barrier((gcptr)w_prev);
+        w_prev->next = w_newnode;
+        w_prev = w_newnode;
+    }
+    
+    stm_finalize();
+    printf("setup ok\n");
+}
+
+int main(void)
+{
+    int i, status;
+    
+    setup_list();
+    
+    status = sem_init(&done, 0, 0);
+    assert(status == 0);
+    
+    for (i = 0; i < NUMTHREADS; i++)
+        newthread(demo2, NULL);
+    
+    for (i=0; i < NUMTHREADS; i++) {
+        status = sem_wait(&done);
+        assert(status == 0);
+        printf("thread finished\n");
+    }
+    
+    final_check();
+    return 0;
+}

From noreply at buildbot.pypy.org  Wed Jun 19 14:32:54 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 19 Jun 2013 14:32:54 +0200 (CEST)
Subject: [pypy-commit] stmgc default: first random-test version in C. may
	have some bugs..
Message-ID: <20130619123254.A1D321C3146@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r195:bbf01c89fe52
Date: 2013-06-19 14:32 +0200
http://bitbucket.org/pypy/stmgc/changeset/bbf01c89fe52/

Log:	first random-test version in C. may have some bugs..

diff --git a/c4/demo_random.c b/c4/demo_random.c
new file mode 100644
--- /dev/null
+++ b/c4/demo_random.c
@@ -0,0 +1,316 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <time.h>
+
+#include "stmgc.h"
+#include "fprintcolor.h"
+
+
+#define NUMTHREADS 4
+#define STEPS 100000
+#define NUMROOTS 10
+#define PREBUILT 3
+#define MAXROOTS 1000
+#define SHARED_ROOTS 5
+
+
+// SUPPORT
+#define GCTID_STRUCT_ROOT     123
+
+struct root {
+    struct stm_object_s hdr;
+    long value;
+    struct root *next;
+};
+typedef struct root * rootptr;
+
+size_t stmcb_size(gcptr ob)
+{
+    assert(stm_get_tid(ob) == GCTID_STRUCT_ROOT);
+    return sizeof(struct root);
+}
+void stmcb_trace(gcptr ob, void visit(gcptr *))
+{
+    rootptr n;
+    assert(stm_get_tid(ob) == GCTID_STRUCT_ROOT);
+    n = (rootptr)ob;
+    visit((gcptr *)&n->next);
+}
+
+// global and per-thread-data
+time_t default_seed;
+gcptr shared_roots[SHARED_ROOTS];
+__thread unsigned int thread_seed = 0;
+__thread gcptr roots[MAXROOTS];
+__thread gcptr roots_outside_perform[MAXROOTS];
+__thread gcptr current_root = 0;
+__thread int num_roots = 0;
+__thread int num_roots_outside_perform = 0;
+__thread int steps_left;
+__thread int interruptible = 0;
+
+
+// helper functions
+int get_rand(int max)
+{
+    return (int)(rand_r(&thread_seed) % (unsigned int)max);
+}
+
+void copy_roots(gcptr *from, gcptr *to)
+{
+    int i;
+    for (i = 0; i < num_roots; i++)
+        *(to++) = *(from++);
+}
+
+gcptr allocate_pseudoprebuilt(size_t size, int tid)
+{
+    gcptr x = calloc(1, size);
+    x->h_tid = PREBUILT_FLAGS | tid;
+    x->h_revision = PREBUILT_REVISION;
+    return x;
+}
+
+void push_roots()
+{
+    int i;
+    for (i = 0; i < num_roots; i++)
+        stm_push_root(roots[i]);
+}
+
+void pop_roots()
+{
+    int i;
+    for (i = num_roots - 1; i >= 0; i--)
+        roots[i] = stm_pop_root();
+}
+
+void del_root(int idx)
+{
+    int i;
+    for (i = idx; i < num_roots - 1; i++)
+        roots[i] = roots[i + 1];
+}
+
+gcptr allocate_root(size_t size, int tid)
+{
+    gcptr r;
+    push_roots();
+    r = stm_allocate(size, tid);
+    pop_roots();
+    return r;
+}
+
+
+
+// THREAD TESTER
+int interruptible_callback(gcptr arg1, int retry_counter);
+int run_me();
+void transaction_break();
+
+void setup_thread()
+{
+    int i;
+    thread_seed = default_seed;
+    steps_left = STEPS;
+    interruptible = 0;
+    
+    num_roots = PREBUILT + NUMROOTS;
+    for (i = 0; i < PREBUILT; i++) {
+        roots[i] = allocate_pseudoprebuilt(sizeof(struct root), 
+                                           GCTID_STRUCT_ROOT);
+    }
+    for (i = PREBUILT; i < PREBUILT + NUMROOTS; i++) {
+        roots[i] = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
+    }
+
+}
+
+gcptr do_step(gcptr p)
+{
+    fprintf(stdout, "#");
+    
+    rootptr w_r, w_sr;
+    gcptr _r, _sr;
+    int num, k;
+
+    num = get_rand(num_roots);
+    _r = roots[num];
+
+    num = get_rand(SHARED_ROOTS);
+    _sr = shared_roots[num];
+
+    k = get_rand(14);
+
+    if (!p) // some parts expect it to be != 0
+        p = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
+
+    switch (k) {
+    case 0: // remove a root
+        if (num > 0)
+            del_root(num);
+        break;
+    case 1: // set 'p' to point to a root
+        if (_r)
+            p = _r;
+        break;
+    case 2: // add 'p' to roots
+        if (num_roots < MAXROOTS)
+            roots[num_roots++] = p;
+        break;
+    case 3: // allocate fresh 'p'
+        p = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
+        break;
+    case 4: // set 'p' as *next in one of the roots
+        w_r = (rootptr)stm_write_barrier(_r);
+        // XXX: do I have to read_barrier(p)?
+        w_r->next = (struct root*)p;
+        break;
+    case 5:  // read and validate 'p'
+        stm_read_barrier(p);
+        break;
+    case 6: // transaction break
+        if (interruptible)
+            return -1; // break current
+        transaction_break();
+        p = NULL;
+        break;
+    case 7: // only do a stm_write_barrier
+        p = stm_write_barrier(p);
+        break;
+    case 8:
+        p = (gcptr)(((rootptr)stm_read_barrier(p))->next);
+        break;
+    case 9: // XXX: rare events
+        break;
+    case 10: // only do a stm_read_barrier
+        p = stm_read_barrier(p);
+        break;
+    case 11:
+        stm_read_barrier(_sr);
+        break;
+    case 12:
+        stm_write_barrier(_sr);
+        break;
+    case 13:
+        w_sr = stm_write_barrier(_sr);
+        w_sr->next = shared_roots[get_rand(SHARED_ROOTS)];
+    default:
+        break;
+    }
+    return p;
+}
+
+
+void transaction_break()
+{
+    push_roots();
+    interruptible = 1;
+    
+    copy_roots(roots, roots_outside_perform);
+    num_roots_outside_perform = num_roots;
+    
+    stm_perform_transaction(NULL, interruptible_callback);
+    
+    num_roots = num_roots_outside_perform;
+    copy_roots(roots_outside_perform, roots);
+    
+    interruptible = 0;
+    pop_roots();
+}
+
+
+int interruptible_callback(gcptr arg1, int retry_counter)
+{
+    num_roots = num_roots_outside_perform;
+    copy_roots(roots_outside_perform, roots);
+
+    arg1 = stm_pop_root();
+    pop_roots();
+    push_roots();
+    stm_push_root(arg1);
+
+    gcptr p = run_me();
+    int restart = p == -1 ? get_rand(3) != 1 : 0;
+
+    return restart;
+}
+
+int run_me()
+{
+    gcptr p = NULL;
+    while (steps_left) {
+        steps_left--;
+        p = do_step(p);
+
+        if (p == -1)
+            return p;
+    }
+    return 0;
+}
+
+
+static sem_t done;
+static int thr_mynum = 0;
+
+void *demo(void *arg)
+{  
+    int status, i;
+    
+    stm_initialize();
+    setup_thread();
+    
+    thr_mynum++;   /* protected by being inevitable here */
+    fprintf(stderr, "THREAD STARTING\n");
+
+    run_me();
+    
+    stm_finalize();
+    
+    status = sem_post(&done);
+    assert(status == 0);
+    return NULL;
+}
+
+
+void newthread(void*(*func)(void*), void *arg)
+{
+    pthread_t th;
+    int status = pthread_create(&th, NULL, func, arg);
+    assert(status == 0);
+    pthread_detach(th);
+    printf("started new thread\n");
+}
+
+
+
+int main(void)
+{
+    int i, status;
+    
+    // seed changes daily
+    // a bit pointless for now..
+    default_seed = time(NULL) / 3600 / 24;
+    
+    for (i = 0; i < SHARED_ROOTS; i++) {
+        shared_roots[i] = allocate_pseudoprebuilt(sizeof(struct root), 
+                                                  GCTID_STRUCT_ROOT);
+    }    
+    
+    status = sem_init(&done, 0, 0);
+    assert(status == 0);
+    
+    for (i = 0; i < NUMTHREADS; i++)
+        newthread(demo, NULL);
+    
+    for (i=0; i < NUMTHREADS; i++) {
+        status = sem_wait(&done);
+        assert(status == 0);
+        printf("thread finished\n");
+    }
+    
+    return 0;
+}

From noreply at buildbot.pypy.org  Wed Jun 19 15:53:52 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 19 Jun 2013 15:53:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: demo_random: remove ambiguous naming
Message-ID: <20130619135352.A01901C101E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r197:661b7c5a26c9
Date: 2013-06-19 14:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/661b7c5a26c9/

Log:	demo_random: remove ambiguous naming

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -18,25 +18,25 @@
 
 
 // SUPPORT
-#define GCTID_STRUCT_ROOT     123
+#define GCTID_STRUCT_NODE     123
 
-struct root {
+struct node {
     struct stm_object_s hdr;
     long value;
-    struct root *next;
+    struct node *next;
 };
-typedef struct root * rootptr;
+typedef struct node * nodeptr;
 
 size_t stmcb_size(gcptr ob)
 {
-    assert(stm_get_tid(ob) == GCTID_STRUCT_ROOT);
-    return sizeof(struct root);
+    assert(stm_get_tid(ob) == GCTID_STRUCT_NODE);
+    return sizeof(struct node);
 }
 void stmcb_trace(gcptr ob, void visit(gcptr *))
 {
-    rootptr n;
-    assert(stm_get_tid(ob) == GCTID_STRUCT_ROOT);
-    n = (rootptr)ob;
+    nodeptr n;
+    assert(stm_get_tid(ob) == GCTID_STRUCT_NODE);
+    n = (nodeptr)ob;
     visit((gcptr *)&n->next);
 }
 
@@ -99,7 +99,7 @@
         td.roots[i] = td.roots[i + 1];
 }
 
-gcptr allocate_root(size_t size, int tid)
+gcptr allocate_node(size_t size, int tid)
 {
     gcptr r;
     push_roots();
@@ -124,11 +124,11 @@
     
     td.num_roots = PREBUILT + NUMROOTS;
     for (i = 0; i < PREBUILT; i++) {
-        td.roots[i] = allocate_pseudoprebuilt(sizeof(struct root), 
-                                              GCTID_STRUCT_ROOT);
+        td.roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
+                                              GCTID_STRUCT_NODE);
     }
     for (i = PREBUILT; i < PREBUILT + NUMROOTS; i++) {
-        td.roots[i] = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
+        td.roots[i] = allocate_node(sizeof(struct node), GCTID_STRUCT_NODE);
     }
 
 }
@@ -137,7 +137,7 @@
 {
     fprintf(stdout, "#");
     
-    rootptr w_r, w_sr;
+    nodeptr w_r, w_sr;
     gcptr _r, _sr;
     int num, k;
 
@@ -150,7 +150,7 @@
     k = get_rand(14);
 
     if (!p) // some parts expect it to be != 0
-        p = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
+        p = allocate_node(sizeof(struct node), GCTID_STRUCT_NODE);
 
     switch (k) {
     case 0: // remove a root
@@ -166,12 +166,12 @@
             td.roots[td.num_roots++] = p;
         break;
     case 3: // allocate fresh 'p'
-        p = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
+        p = allocate_node(sizeof(struct node), GCTID_STRUCT_NODE);
         break;
     case 4: // set 'p' as *next in one of the roots
-        w_r = (rootptr)stm_write_barrier(_r);
+        w_r = (nodeptr)stm_write_barrier(_r);
         // XXX: do I have to read_barrier(p)?
-        w_r->next = (struct root*)p;
+        w_r->next = (struct node*)p;
         break;
     case 5:  // read and validate 'p'
         stm_read_barrier(p);
@@ -186,7 +186,7 @@
         p = stm_write_barrier(p);
         break;
     case 8:
-        p = (gcptr)(((rootptr)stm_read_barrier(p))->next);
+        p = (gcptr)(((nodeptr)stm_read_barrier(p))->next);
         break;
     case 9: // XXX: rare events
         break;
@@ -200,8 +200,8 @@
         stm_write_barrier(_sr);
         break;
     case 13:
-        w_sr = (rootptr)stm_write_barrier(_sr);
-        w_sr->next = (rootptr)shared_roots[get_rand(SHARED_ROOTS)];
+        w_sr = (nodeptr)stm_write_barrier(_sr);
+        w_sr->next = (nodeptr)shared_roots[get_rand(SHARED_ROOTS)];
     default:
         break;
     }
@@ -300,8 +300,8 @@
     default_seed = time(NULL) / 3600 / 24;
     
     for (i = 0; i < SHARED_ROOTS; i++) {
-        shared_roots[i] = allocate_pseudoprebuilt(sizeof(struct root), 
-                                                  GCTID_STRUCT_ROOT);
+        shared_roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
+                                                  GCTID_STRUCT_NODE);
     }    
     
     status = sem_init(&done, 0, 0);

From noreply at buildbot.pypy.org  Wed Jun 19 15:53:51 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 19 Jun 2013 15:53:51 +0200 (CEST)
Subject: [pypy-commit] stmgc default: demo_random: thread locals -> thread
	descriptor
Message-ID: <20130619135351.857CA1C094A@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r196:8d3cf9f0c450
Date: 2013-06-19 14:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/8d3cf9f0c450/

Log:	demo_random: thread locals -> thread descriptor

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -40,29 +40,33 @@
     visit((gcptr *)&n->next);
 }
 
+
 // global and per-thread-data
 time_t default_seed;
 gcptr shared_roots[SHARED_ROOTS];
-__thread unsigned int thread_seed = 0;
-__thread gcptr roots[MAXROOTS];
-__thread gcptr roots_outside_perform[MAXROOTS];
-__thread gcptr current_root = 0;
-__thread int num_roots = 0;
-__thread int num_roots_outside_perform = 0;
-__thread int steps_left;
-__thread int interruptible = 0;
+struct thread_data {
+    unsigned int thread_seed;
+    gcptr roots[MAXROOTS];
+    gcptr roots_outside_perform[MAXROOTS];
+    gcptr current_root;
+    int num_roots;
+    int num_roots_outside_perform;
+    int steps_left;
+    int interruptible;
+};
+__thread struct thread_data td;
 
 
 // helper functions
 int get_rand(int max)
 {
-    return (int)(rand_r(&thread_seed) % (unsigned int)max);
+    return (int)(rand_r(&td.thread_seed) % (unsigned int)max);
 }
 
-void copy_roots(gcptr *from, gcptr *to)
+void copy_roots(gcptr *from, gcptr *to, int num)
 {
     int i;
-    for (i = 0; i < num_roots; i++)
+    for (i = 0; i < num; i++)
         *(to++) = *(from++);
 }
 
@@ -77,22 +81,22 @@
 void push_roots()
 {
     int i;
-    for (i = 0; i < num_roots; i++)
-        stm_push_root(roots[i]);
+    for (i = 0; i < td.num_roots; i++)
+        stm_push_root(td.roots[i]);
 }
 
 void pop_roots()
 {
     int i;
-    for (i = num_roots - 1; i >= 0; i--)
-        roots[i] = stm_pop_root();
+    for (i = td.num_roots - 1; i >= 0; i--)
+        td.roots[i] = stm_pop_root();
 }
 
 void del_root(int idx)
 {
     int i;
-    for (i = idx; i < num_roots - 1; i++)
-        roots[i] = roots[i + 1];
+    for (i = idx; i < td.num_roots - 1; i++)
+        td.roots[i] = td.roots[i + 1];
 }
 
 gcptr allocate_root(size_t size, int tid)
@@ -114,17 +118,17 @@
 void setup_thread()
 {
     int i;
-    thread_seed = default_seed;
-    steps_left = STEPS;
-    interruptible = 0;
+    td.thread_seed = default_seed;
+    td.steps_left = STEPS;
+    td.interruptible = 0;
     
-    num_roots = PREBUILT + NUMROOTS;
+    td.num_roots = PREBUILT + NUMROOTS;
     for (i = 0; i < PREBUILT; i++) {
-        roots[i] = allocate_pseudoprebuilt(sizeof(struct root), 
-                                           GCTID_STRUCT_ROOT);
+        td.roots[i] = allocate_pseudoprebuilt(sizeof(struct root), 
+                                              GCTID_STRUCT_ROOT);
     }
     for (i = PREBUILT; i < PREBUILT + NUMROOTS; i++) {
-        roots[i] = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
+        td.roots[i] = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
     }
 
 }
@@ -137,8 +141,8 @@
     gcptr _r, _sr;
     int num, k;
 
-    num = get_rand(num_roots);
-    _r = roots[num];
+    num = get_rand(td.num_roots);
+    _r = td.roots[num];
 
     num = get_rand(SHARED_ROOTS);
     _sr = shared_roots[num];
@@ -158,8 +162,8 @@
             p = _r;
         break;
     case 2: // add 'p' to roots
-        if (num_roots < MAXROOTS)
-            roots[num_roots++] = p;
+        if (td.num_roots < MAXROOTS)
+            td.roots[td.num_roots++] = p;
         break;
     case 3: // allocate fresh 'p'
         p = allocate_root(sizeof(struct root), GCTID_STRUCT_ROOT);
@@ -173,8 +177,8 @@
         stm_read_barrier(p);
         break;
     case 6: // transaction break
-        if (interruptible)
-            return -1; // break current
+        if (td.interruptible)
+            return (gcptr)-1; // break current
         transaction_break();
         p = NULL;
         break;
@@ -196,8 +200,8 @@
         stm_write_barrier(_sr);
         break;
     case 13:
-        w_sr = stm_write_barrier(_sr);
-        w_sr->next = shared_roots[get_rand(SHARED_ROOTS)];
+        w_sr = (rootptr)stm_write_barrier(_sr);
+        w_sr->next = (rootptr)shared_roots[get_rand(SHARED_ROOTS)];
     default:
         break;
     }
@@ -208,32 +212,32 @@
 void transaction_break()
 {
     push_roots();
-    interruptible = 1;
+    td.interruptible = 1;
     
-    copy_roots(roots, roots_outside_perform);
-    num_roots_outside_perform = num_roots;
+    copy_roots(td.roots, td.roots_outside_perform, td.num_roots);
+    td.num_roots_outside_perform = td.num_roots;
     
     stm_perform_transaction(NULL, interruptible_callback);
     
-    num_roots = num_roots_outside_perform;
-    copy_roots(roots_outside_perform, roots);
+    td.num_roots = td.num_roots_outside_perform;
+    copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
     
-    interruptible = 0;
+    td.interruptible = 0;
     pop_roots();
 }
 
 
 int interruptible_callback(gcptr arg1, int retry_counter)
 {
-    num_roots = num_roots_outside_perform;
-    copy_roots(roots_outside_perform, roots);
+    td.num_roots = td.num_roots_outside_perform;
+    copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
 
     arg1 = stm_pop_root();
     pop_roots();
     push_roots();
     stm_push_root(arg1);
 
-    gcptr p = run_me();
+    int p = run_me();
     int restart = p == -1 ? get_rand(3) != 1 : 0;
 
     return restart;
@@ -242,12 +246,12 @@
 int run_me()
 {
     gcptr p = NULL;
-    while (steps_left) {
-        steps_left--;
+    while (td.steps_left) {
+        td.steps_left--;
         p = do_step(p);
 
-        if (p == -1)
-            return p;
+        if (p == (gcptr)-1)
+            return -1;
     }
     return 0;
 }
@@ -258,7 +262,7 @@
 
 void *demo(void *arg)
 {  
-    int status, i;
+    int status;
     
     stm_initialize();
     setup_thread();

From noreply at buildbot.pypy.org  Wed Jun 19 17:47:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Jun 2013 17:47:00 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Check that the test passes if written
 this way, retrying up to 1000 times
Message-ID: <20130619154700.A2E481C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r198:8b48f8933856
Date: 2013-06-19 17:46 +0200
http://bitbucket.org/pypy/stmgc/changeset/8b48f8933856/

Log:	Check that the test passes if written this way, retrying up to 1000
	times for stm_read_barrier() to return the same pointer.

diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -98,8 +98,15 @@
             r_next = (struct node*)stm_read_barrier((gcptr)tmp);
         }
         // results from consecutive read_barriers can differ. needs Ptr_Eq()
-        /* assert(stm_read_barrier((gcptr)r_prev->next) == r_current */
-        /* 	     && stm_read_barrier((gcptr)r_current->next) == r_next); */
+        int i = 0;
+        while (!(stm_read_barrier((gcptr)r_prev->next) ==
+                 stm_read_barrier((gcptr)r_current) &&
+                 stm_read_barrier((gcptr)r_current->next) ==
+                 stm_read_barrier((gcptr)r_next))) {
+            asm volatile ("pause":::"memory");  /* smp_spinloop() */
+            i++;
+            assert(i < 1000);
+        }
         // for now:
         assert(((nodeptr)stm_read_barrier((gcptr)r_prev->next))->value 
                == r_current->value

From noreply at buildbot.pypy.org  Wed Jun 19 20:18:37 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 19 Jun 2013 20:18:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: typo
Message-ID: <20130619181837.B7AA01C094A@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64937:ec54480e12ef
Date: 2013-06-18 03:58 -0500
http://bitbucket.org/pypy/pypy/changeset/ec54480e12ef/

Log:	typo

diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 

From noreply at buildbot.pypy.org  Wed Jun 19 20:18:39 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 19 Jun 2013 20:18:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: add tests for edgecases of longlong
 bitfields that fail on 32bit
Message-ID: <20130619181839.216A21C094A@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64938:82cd3868d29c
Date: 2013-06-18 04:00 -0500
http://bitbucket.org/pypy/pypy/changeset/82cd3868d29c/

Log:	add tests for edgecases of longlong bitfields that fail on 32bit

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
@@ -66,6 +66,10 @@
         x.a, x.b, x.c = -1, 7, -1
         assert (x.a, x.b, x.c) == (-1, 7, -1)
 
+        x = X()
+        x.a, x.b, x.c = -1, -7, -1
+        assert (x.a, x.b, x.c) == (-1, -7, -1)
+
     def test_ulonglong(self):
         class X(Structure):
             _fields_ = [("a", c_ulonglong, 1),
@@ -75,8 +79,8 @@
         assert sizeof(X) == sizeof(c_longlong)
         x = X()
         assert (x.a, x.b, x.c) == (0, 0, 0)
-        x.a, x.b, x.c = 7, 7, 7
-        assert (x.a, x.b, x.c) == (1, 7, 1)
+        x.a, x.b, x.c = 7, 2305843009213693953, 7
+        assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1)
 
     def test_signed(self):
         for c_typ in signed_int_types:

From noreply at buildbot.pypy.org  Wed Jun 19 20:18:41 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 19 Jun 2013 20:18:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: more tests for _rawffi structures using
	bitfields
Message-ID: <20130619181841.397951C094A@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64939:0e209e181c89
Date: 2013-06-19 13:01 -0500
http://bitbucket.org/pypy/pypy/changeset/0e209e181c89/

Log:	more tests for _rawffi structures using bitfields

diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -48,12 +48,12 @@
             free(x1->next);
             free(x1);
         }
-        
+
         const char *static_str = "xxxxxx";
         long static_int = 42;
         double static_double = 42.42;
         long double static_longdouble = 42.42;
-        
+
         unsigned short add_shorts(short one, short two)
         {
            return one + two;
@@ -175,7 +175,7 @@
             inp.y = inp.x * 100;
             return inp;
         }
-        
+
         '''))
         symbols = """get_char char_check get_raw_pointer
                      add_shorts
@@ -194,7 +194,7 @@
         eci = ExternalCompilationInfo(export_symbols=symbols)
         return str(platform.compile([c_file], eci, 'x', standalone=False))
     prepare_c_example = staticmethod(prepare_c_example)
-    
+
     def setup_class(cls):
         space = cls.space
         from rpython.rlib.clibffi import get_libc_name
@@ -438,7 +438,7 @@
         arg.free()
         assert t.tm_year == 70
         assert t.tm_sec == 1
-        assert t.tm_min == 2      
+        assert t.tm_min == 2
         x.free()
 
     def test_nested_structures(self):
@@ -464,6 +464,20 @@
         free_double_struct = lib.ptr("free_double_struct", ['P'], None)
         free_double_struct(res)
 
+    def test_structure_bitfields_char(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'B', 1),
+                               ('B', 'B', 6),
+                               ('C', 'B', 1)])
+        x = X()
+        x.A = 0xf
+        x.B = 0xff
+        x.C = 0xf
+        assert x.A == 1
+        assert x.B == 63
+        assert x.C == 1
+        x.free()
+
     def test_structure_bitfields(self):
         import _rawffi
         X = _rawffi.Structure([('A', 'I', 1),
@@ -486,6 +500,60 @@
         assert (y.a, y.b, y.c) == (-1, -7, 0)
         y.free()
 
+    def test_structure_bitfields_longlong(self):
+        import _rawffi
+        Z = _rawffi.Structure([('a', 'Q', 1),
+                               ('b', 'Q', 62),
+                               ('c', 'Q', 1)])
+        z = Z()
+        z.a, z.b, z.c = 7, 0x1000000000000001, 7
+        assert (z.a, z.b, z.c) == (1, 0x1000000000000001, 1)
+        z.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
     def test_invalid_bitfields(self):
         import _rawffi
         raises(TypeError, _rawffi.Structure, [('A', 'c', 1)])
@@ -571,7 +639,7 @@
         res = pass_ll(arg1)
         assert res[0] == 1<<42
         arg1.free()
-    
+
     def test_callback(self):
         import _rawffi
         import struct
@@ -970,7 +1038,7 @@
         assert res.y == 33
         assert s2h.x == 7
         assert s2h.y == 11
-        
+
         s2h.free()
 
     def test_ret_struct_containing_array(self):
@@ -1029,7 +1097,7 @@
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
-    
+
     def setup_class(cls):
         cls.w_sizes_and_alignments = cls.space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))

From noreply at buildbot.pypy.org  Wed Jun 19 20:18:42 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 19 Jun 2013 20:18:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for the bitmask calculation of
 bitfields in _rawffi and ctypes
Message-ID: <20130619181842.74FB11C094A@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64940:fcd5bdf14412
Date: 2013-06-19 13:10 -0500
http://bitbucket.org/pypy/pypy/changeset/fcd5bdf14412/

Log:	fix for the bitmask calculation of bitfields in _rawffi and ctypes
	On 32 bit systems longlong bitfields need a bitmask that is larger
	than a word

diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -15,7 +15,8 @@
 from pypy.module._rawffi.interp_rawffi import size_alignment, LL_TYPEMAP
 from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers
 from rpython.rlib import clibffi
-from rpython.rlib.rarithmetic import intmask, r_uint, signedtype, widen
+from rpython.rlib.rarithmetic import intmask, signedtype, widen
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong
 
 def unpack_fields(space, w_fields):
     fields_w = space.unpackiterable(w_fields)
@@ -260,10 +261,17 @@
 
 def LOW_BIT(x):
     return x & 0xFFFF
+
 def NUM_BITS(x):
     return x >> 16
-def BIT_MASK(x):
-    return (1 << x) - 1
+
+def BIT_MASK(x, ll_t):
+    if ll_t is lltype.SignedLongLong:
+        return (r_longlong(1) << x) - 1
+    elif ll_t is lltype.UnsignedLongLong:
+        return (r_ulonglong(1) << x) - 1
+    return (1  << x) -1
+BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
@@ -279,8 +287,9 @@
             lowbit = LOW_BIT(bitsize)
             if numbits:
                 value = widen(value)
+                bitmask = BIT_MASK(numbits, TP)
+                #
                 current = widen(rffi.cast(T, ptr)[0])
-                bitmask = BIT_MASK(numbits)
                 current &= ~ (bitmask << lowbit)
                 current |= (value & bitmask) << lowbit
                 value = rffi.cast(TP, current)
@@ -301,13 +310,16 @@
             numbits = NUM_BITS(bitsize)
             lowbit = LOW_BIT(bitsize)
             if numbits:
-                value = widen(value)
+                value = widen(rffi.cast(ll_t, value))
+                bitmask = BIT_MASK(numbits, ll_t)
+                #
                 value >>= lowbit
-                value &= BIT_MASK(numbits)
+                value &= bitmask
                 if ll_t is lltype.Bool or signedtype(ll_t._type):
                     sign = (value >> (numbits - 1)) & 1
                     if sign:
-                        value = value - (1 << numbits)
+                        one = r_longlong(1) if ll_t is lltype.SignedLongLong else 1
+                        value = value - (one << numbits)
                 value = rffi.cast(ll_t, value)
             break
 

From noreply at buildbot.pypy.org  Wed Jun 19 20:18:44 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 19 Jun 2013 20:18:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20130619181844.895FC1C094A@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64941:b4cd93e60f86
Date: 2013-06-19 13:11 -0500
http://bitbucket.org/pypy/pypy/changeset/b4cd93e60f86/

Log:	merge heads

diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -13,7 +13,6 @@
     _mixin_ = True
 
     def __init__(self, space, name, size, align, enumerators, enumvalues):
-        name = "enum " + name
         self._super.__init__(self, space, size, name, len(name), align)
         self.enumerators2values = {}   # str -> int
         self.enumvalues2erators = {}   # int -> str
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         else:
             extra = " *"
         self.is_file = (ctitem.name == "struct _IO_FILE" or
-                        ctitem.name == "struct $FILE")
+                        ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -25,7 +25,6 @@
     custom_field_pos = False
 
     def __init__(self, space, name):
-        name = '%s %s' % (self.kind, name)
         W_CType.__init__(self, space, -1, name, len(name))
 
     def check_complete(self, w_errorcls=None):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -641,6 +641,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -648,7 +650,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -657,7 +659,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -689,7 +691,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -708,7 +710,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -728,7 +730,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -744,7 +746,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -758,7 +760,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -792,7 +794,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -803,7 +805,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -832,7 +834,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -937,7 +939,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -953,7 +955,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -965,7 +967,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -985,7 +987,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1023,7 +1025,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1048,7 +1050,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1221,7 +1223,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1238,7 +1240,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1275,26 +1277,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1308,27 +1311,27 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1432,7 +1435,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1455,7 +1458,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1485,14 +1488,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1561,7 +1564,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1601,7 +1604,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1619,7 +1622,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1697,7 +1700,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1714,7 +1717,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1725,7 +1728,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1735,7 +1738,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1745,7 +1748,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1754,7 +1757,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1766,7 +1769,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1775,7 +1778,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1785,7 +1788,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1795,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1818,7 +1821,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1845,7 +1848,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1957,7 +1960,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1977,7 +1980,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2164,7 +2167,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2183,7 +2186,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2192,7 +2195,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2308,7 +2311,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2320,8 +2323,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2353,12 +2356,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2366,7 +2369,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2424,7 +2427,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2440,14 +2443,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2508,7 +2511,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2540,7 +2543,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2563,7 +2566,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2762,7 +2765,7 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
@@ -2775,7 +2778,7 @@
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2790,7 +2793,7 @@
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1222,8 +1222,7 @@
     def _safe_find(self, w_list, obj, start, stop):
         l = self.unerase(w_list.lstorage)
         for i in range(start, min(stop, len(l))):
-            val = l[i]
-            if val == obj:
+            if l[i] == obj:
                 return i
         raise ValueError
 
@@ -1543,6 +1542,18 @@
         if reverse:
             l.reverse()
 
+    def _safe_find(self, w_list, obj, start, stop):
+        from rpython.rlib.rfloat import isnan
+        if not isnan(obj):
+            return AbstractUnwrappedStrategy._safe_find(self, w_list, obj,
+                                                        start, stop)
+        # unwrapped nan != nan, finding it requires more effort
+        l = self.unerase(w_list.lstorage)
+        for i in range(start, min(stop, len(l))):
+            if isnan(l[i]):
+                return i
+        raise ValueError
+
 
 class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = None
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1314,6 +1314,10 @@
         non_list = NonList()
         assert [] != non_list
 
+    def test_nan_containment(self):
+        nan = float('nan')
+        assert nan in [nan]
+
 
 class AppTestForRangeLists(AppTestW_ListObject):
     spaceconfig = {"objspace.std.withrangelist": True}

From noreply at buildbot.pypy.org  Wed Jun 19 23:00:29 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 19 Jun 2013 23:00:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Soem cleanups.
Message-ID: <20130619210029.4A49A1C0268@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64942:8148f32245b6
Date: 2013-06-19 13:59 -0700
http://bitbucket.org/pypy/pypy/changeset/8148f32245b6/

Log:	Soem cleanups.

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,22 +1,19 @@
 from rpython.jit.metainterp import resume
-from rpython.jit.metainterp.optimizeopt import virtualize
-from rpython.jit.metainterp.optimizeopt.optimizer import LEVEL_CONSTANT, \
-                                                      LEVEL_KNOWNCLASS, \
-                                                      LEVEL_NONNULL, \
-                                                      LEVEL_UNKNOWN, \
-                                                      MININT, MAXINT, OptValue
 from rpython.jit.metainterp.history import BoxInt, ConstInt, BoxPtr, Const
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded
+from rpython.jit.metainterp.optimizeopt import virtualize
+from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
+from rpython.jit.metainterp.optimizeopt.optimizer import (LEVEL_CONSTANT,
+    LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
-from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.objectmodel import we_are_translated
-import os
+
 
 class BadVirtualState(Exception):
     pass
 
+
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
 
@@ -62,7 +59,6 @@
         else:
             debug_print(indent + "    ...")
 
-
     def debug_header(self, indent):
         raise NotImplementedError
 
@@ -143,6 +139,7 @@
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
 
+
 class VStructStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, typedescr, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
@@ -215,7 +212,6 @@
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
 
 
-
 class VArrayStructStateInfo(AbstractVirtualStateInfo):
     def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
@@ -291,7 +287,6 @@
         debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
 
 
-
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value, is_opaque=False):
         self.is_opaque = is_opaque
@@ -382,21 +377,21 @@
             extra_guards.append(op)
             return
 
-        if self.level == LEVEL_NONNULL and \
-               other.level == LEVEL_UNKNOWN and \
-               isinstance(box, BoxPtr) and \
-               box.nonnull():
+        if (self.level == LEVEL_NONNULL and
+               other.level == LEVEL_UNKNOWN and
+               isinstance(box, BoxPtr) and
+               box.nonnull()):
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
             extra_guards.append(op)
             return
 
-        if self.level == LEVEL_UNKNOWN and \
-               other.level == LEVEL_UNKNOWN and \
-               isinstance(box, BoxInt) and \
-               self.intbound.contains(box.getint()):
+        if (self.level == LEVEL_UNKNOWN and
+               other.level == LEVEL_UNKNOWN and
+               isinstance(box, BoxInt) and
+               self.intbound.contains(box.getint())):
             if self.intbound.has_lower:
                 bound = self.intbound.lower
-                if not (other.intbound.has_lower and \
+                if not (other.intbound.has_lower and
                         other.intbound.lower >= bound):
                     res = BoxInt()
                     op = ResOperation(rop.INT_GE, [box, ConstInt(bound)], res)
@@ -405,7 +400,7 @@
                     extra_guards.append(op)
             if self.intbound.has_upper:
                 bound = self.intbound.upper
-                if not (other.intbound.has_upper and \
+                if not (other.intbound.has_upper and
                         other.intbound.upper <= bound):
                     res = BoxInt()
                     op = ResOperation(rop.INT_LE, [box, ConstInt(bound)], res)
@@ -417,9 +412,6 @@
         # Remaining cases are probably not interesting
         raise InvalidLoop('Generating guards for making the VirtualStates ' +
                           'at hand match have not been implemented')
-        if self.level == LEVEL_CONSTANT:
-            import pdb; pdb.set_trace()
-            raise NotImplementedError
 
     def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
@@ -463,6 +455,7 @@
         debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
                     ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
 
+
 class VirtualState(object):
     def __init__(self, state):
         self.state = state
@@ -521,6 +514,7 @@
         for s in self.state:
             s.debug_print("    ", seen, bad)
 
+
 class VirtualStateAdder(resume.ResumeDataVirtualAdder):
     def __init__(self, optimizer):
         self.fieldboxes = {}
@@ -587,9 +581,11 @@
     def make_vrawbuffer(self, size, offsets, descrs):
         raise NotImplementedError
 
+
 class BoxNotProducable(Exception):
     pass
 
+
 class ShortBoxes(object):
     def __init__(self, optimizer, surviving_boxes, availible_boxes=None):
         self.potential_ops = {}
@@ -672,7 +668,7 @@
         if self.availible_boxes is not None and box not in self.availible_boxes:
             raise BoxNotProducable
         self.short_boxes_in_production[box] = None
-        
+
         if box in self.potential_ops:
             ops = self.prioritized_alternatives(box)
             produced_one = False

From noreply at buildbot.pypy.org  Wed Jun 19 23:53:06 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 19 Jun 2013 23:53:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: some random pep8 cleanups
Message-ID: <20130619215306.55A1D1C0268@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64943:2024b065cf2e
Date: 2013-06-19 14:52 -0700
http://bitbucket.org/pypy/pypy/changeset/2024b065cf2e/

Log:	some random pep8 cleanups

diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ b/rpython/jit/metainterp/optimizeopt/generalize.py
@@ -1,5 +1,6 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
 
+
 class GeneralizationStrategy(object):
     def __init__(self, optimizer):
         self.optimizer = optimizer
@@ -7,13 +8,13 @@
     def apply(self):
         raise NotImplementedError
 
+
 class KillHugeIntBounds(GeneralizationStrategy):
     def apply(self):
         for v in self.optimizer.values.values():
             if v.is_constant():
                 continue
-            if v.intbound.lower < MININT/2:
+            if v.intbound.lower < MININT / 2:
                 v.intbound.lower = MININT
-            if v.intbound.upper > MAXINT/2:
+            if v.intbound.upper > MAXINT / 2:
                 v.intbound.upper = MAXINT
-          
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -1,8 +1,8 @@
 import os
 
+from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.jitexc import JitException
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS
-from rpython.jit.metainterp.history import ConstInt, Const
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.rlib.objectmodel import we_are_translated
@@ -32,14 +32,14 @@
     def do_setfield(self, optheap, op):
         # Update the state with the SETFIELD_GC/SETARRAYITEM_GC operation 'op'.
         structvalue = optheap.getvalue(op.getarg(0))
-        fieldvalue  = optheap.getvalue(op.getarglist()[-1])
+        fieldvalue = optheap.getvalue(op.getarglist()[-1])
         if self.possible_aliasing(optheap, structvalue):
             self.force_lazy_setfield(optheap)
             assert not self.possible_aliasing(optheap, structvalue)
         cached_fieldvalue = self._cached_fields.get(structvalue, None)
 
         # Hack to ensure constants are imported from the preamble
-        if cached_fieldvalue and fieldvalue.is_constant(): 
+        if cached_fieldvalue and fieldvalue.is_constant():
             optheap.optimizer.ensure_imported(cached_fieldvalue)
             cached_fieldvalue = self._cached_fields.get(structvalue, None)
 
@@ -50,7 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
-            
+
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -100,7 +100,7 @@
             # back in the cache: the value of this particular structure's
             # field.
             structvalue = optheap.getvalue(op.getarg(0))
-            fieldvalue  = optheap.getvalue(op.getarglist()[-1])
+            fieldvalue = optheap.getvalue(op.getarglist()[-1])
             self.remember_field_value(structvalue, fieldvalue, op)
         elif not can_cache:
             self.clear()
@@ -403,7 +403,6 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
-        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -1,13 +1,12 @@
 import sys
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, CONST_1, CONST_0, \
-                                                  MODE_ARRAY, MODE_STR, MODE_UNICODE
 from rpython.jit.metainterp.history import ConstInt
+from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import (IntBound, IntLowerBound,
     IntUpperBound)
+from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
+    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.rlib.rarithmetic import LONG_BIT
 
 
 class OptIntBounds(Optimization):
@@ -24,7 +23,6 @@
         assert not op.is_ovf()
         self.emit_operation(op)
 
-
     def propagate_bounds_backward(self, box):
         # FIXME: This takes care of the instruction where box is the reuslt
         #        but the bounds produced by all instructions where box is
@@ -68,11 +66,11 @@
         if v2.is_constant():
             val = v2.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0,val))
+                r.intbound.intersect(IntBound(0, val))
         elif v1.is_constant():
             val = v1.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0,val))
+                r.intbound.intersect(IntBound(0, val))
 
     def optimize_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -111,7 +109,7 @@
     def optimize_INT_MOD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and 
+        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and
                         v2.intbound.known_ge(IntBound(0, 0)))
         if known_nonneg and v2.is_constant():
             val = v2.box.getint()
@@ -197,7 +195,7 @@
         if opnum not in (rop.INT_ADD_OVF, rop.INT_SUB_OVF, rop.INT_MUL_OVF):
             raise InvalidLoop('An INT_xxx_OVF was proven not to overflow but' +
                               'guarded with GUARD_OVERFLOW')
-                             
+
         self.emit_operation(op)
 
     def optimize_INT_ADD_OVF(self, op):
@@ -299,7 +297,7 @@
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
-        array  = self.getvalue(op.getarg(0))
+        array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_ARRAY, op.getdescr(), -1)
         array.lenbound.bound.intersect(result.intbound)
@@ -307,7 +305,7 @@
 
     def optimize_STRLEN(self, op):
         self.emit_operation(op)
-        array  = self.getvalue(op.getarg(0))
+        array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_STR, op.getdescr(), -1)
         array.lenbound.bound.intersect(result.intbound)
@@ -315,7 +313,7 @@
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
-        array  = self.getvalue(op.getarg(0))
+        array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_UNICODE, op.getdescr(), -1)
         array.lenbound.bound.intersect(result.intbound)
@@ -470,9 +468,9 @@
         if v1.intbound.intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
 
-    propagate_bounds_INT_ADD_OVF  = propagate_bounds_INT_ADD
-    propagate_bounds_INT_SUB_OVF  = propagate_bounds_INT_SUB
-    propagate_bounds_INT_MUL_OVF  = propagate_bounds_INT_MUL
+    propagate_bounds_INT_ADD_OVF = propagate_bounds_INT_ADD
+    propagate_bounds_INT_SUB_OVF = propagate_bounds_INT_SUB
+    propagate_bounds_INT_MUL_OVF = propagate_bounds_INT_MUL
 
 
 dispatch_opt = make_dispatcher_method(OptIntBounds, 'optimize_',

From noreply at buildbot.pypy.org  Thu Jun 20 06:48:19 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Jun 2013 06:48:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: random pep8 cleanups
Message-ID: <20130620044819.700C81C101E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64944:34b8e7d584a8
Date: 2013-06-19 21:46 -0700
http://bitbucket.org/pypy/pypy/changeset/34b8e7d584a8/

Log:	random pep8 cleanups

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -9,19 +9,20 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 from rpython.jit.metainterp.resoperation import ResOperation, rop, get_deep_immutable_oplist
-from rpython.jit.metainterp.history import TreeLoop, Box, JitCellToken, TargetToken
-from rpython.jit.metainterp.history import AbstractFailDescr, BoxInt
-from rpython.jit.metainterp.history import BoxPtr, BoxFloat, ConstInt
-from rpython.jit.metainterp import history, resume, jitexc
+from rpython.jit.metainterp.history import (TreeLoop, Box, JitCellToken,
+    TargetToken, AbstractFailDescr, BoxInt, BoxPtr, BoxFloat, ConstInt)
+from rpython.jit.metainterp import history, jitexc
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.inliner import Inliner
 from rpython.jit.metainterp.resume import NUMBERING, PENDINGFIELDSP, ResumeDataDirectReader
 from rpython.jit.codewriter import heaptracker, longlong
 
+
 def giveup():
     from rpython.jit.metainterp.pyjitpl import SwitchToBlackhole
     raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
 
+
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
     if option and (option.view or option.viewloops):
@@ -657,17 +658,21 @@
 class AllVirtuals:
     llopaque = True
     cache = None
+
     def __init__(self, cache):
         self.cache = cache
+
     def hide(self, cpu):
         ptr = cpu.ts.cast_instance_to_base_ref(self)
         return cpu.ts.cast_to_ref(ptr)
+
     @staticmethod
     def show(cpu, gcref):
         from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
         ptr = cpu.ts.cast_to_baseclass(gcref)
         return cast_base_ptr_to_instance(AllVirtuals, ptr)
 
+
 class ResumeGuardForcedDescr(ResumeGuardDescr):
 
     def __init__(self, metainterp_sd, jitdriver_sd):
@@ -762,9 +767,12 @@
     b = c = -1
     for i in range(1, 5):
         if self.counters[i] > self.counters[a]:
-            c = b; b = a; a = i
+            c = b
+            b = a
+            a = i
         elif b < 0 or self.counters[i] > self.counters[b]:
-            c = b; b = i
+            c = b
+            b = i
         elif c < 0 or self.counters[i] > self.counters[c]:
             c = i
     self.counters[c] = 1
@@ -878,10 +886,14 @@
     assert len(redargtypes) == nb_red_args
     inputargs = []
     for kind in redargtypes:
-        if   kind == history.INT:   box = BoxInt()
-        elif kind == history.REF:   box = BoxPtr()
-        elif kind == history.FLOAT: box = BoxFloat()
-        else: raise AssertionError
+        if kind == history.INT:
+            box = BoxInt()
+        elif kind == history.REF:
+            box = BoxPtr()
+        elif kind == history.FLOAT:
+            box = BoxFloat()
+        else:
+            raise AssertionError
         inputargs.append(box)
     k = jitdriver_sd.portal_runner_adr
     funcbox = history.ConstInt(heaptracker.adr2int(k))
@@ -909,7 +921,7 @@
         ResOperation(rop.CALL, callargs, result, descr=jd.portal_calldescr),
         ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr),
         ResOperation(rop.FINISH, finishargs, None, descr=jd.portal_finishtoken)
-        ]
+    ]
     operations[1].setfailargs([])
     operations = get_deep_immutable_oplist(operations)
     cpu.compile_loop(inputargs, operations, jitcell_token, log=False)
diff --git a/rpython/jit/metainterp/inliner.py b/rpython/jit/metainterp/inliner.py
--- a/rpython/jit/metainterp/inliner.py
+++ b/rpython/jit/metainterp/inliner.py
@@ -1,6 +1,7 @@
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.resume import Snapshot
 
+
 class Inliner(object):
     def __init__(self, inputargs, jump_args):
         assert len(inputargs) == len(jump_args)
@@ -54,4 +55,3 @@
         new_snapshot = Snapshot(self.inline_snapshot(snapshot.prev), boxes)
         self.snapshot_map[snapshot] = new_snapshot
         return new_snapshot
-
diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py
--- a/rpython/jit/metainterp/jitprof.py
+++ b/rpython/jit/metainterp/jitprof.py
@@ -18,7 +18,7 @@
 
 class EmptyProfiler(BaseProfiler):
     initialized = True
-    
+
     def start(self):
         pass
 
@@ -96,7 +96,7 @@
     def end_backend(self):     self._end  (Counters.BACKEND)
 
     def count(self, kind, inc=1):
-        self.counters[kind] += inc        
+        self.counters[kind] += inc
 
     def get_counter(self, num):
         if num == Counters.TOTAL_COMPILED_LOOPS:
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -1,14 +1,13 @@
-
-from rpython.rlib.debug import have_debug_prints
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
+from rpython.jit.metainterp.history import (ConstInt, BoxInt, ConstFloat,
+    BoxFloat, TargetToken)
+from rpython.jit.metainterp.resoperation import rop
+from rpython.rlib.debug import (have_debug_prints, debug_start, debug_stop,
+    debug_print)
 from rpython.rlib.objectmodel import we_are_translated, compute_unique_id
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.history import ConstInt, \
-     BoxInt, ConstFloat, BoxFloat, TargetToken
+
 
 class Logger(object):
-
     def __init__(self, metainterp_sd, guard_number=False):
         self.metainterp_sd = metainterp_sd
         self.guard_number = guard_number
@@ -24,7 +23,7 @@
             debug_stop("jit-log-compiling-loop")
         else:
             debug_start("jit-log-opt-loop")
-            debug_print("# Loop", number, '(%s)' % name , ":", type,
+            debug_print("# Loop", number, '(%s)' % name, ":", type,
                         "with", len(operations), "ops")
             logops = self._log_operations(inputargs, operations, ops_offset)
             debug_stop("jit-log-opt-loop")
@@ -138,7 +137,7 @@
             else:
                 r = self.repr_of_descr(descr)
             if args:
-                args += ', descr=' +  r
+                args += ', descr=' + r
             else:
                 args = "descr=" + r
         if is_guard and op.getfailargs() is not None:
diff --git a/rpython/jit/metainterp/optimize.py b/rpython/jit/metainterp/optimize.py
--- a/rpython/jit/metainterp/optimize.py
+++ b/rpython/jit/metainterp/optimize.py
@@ -1,6 +1,7 @@
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.jit.metainterp.jitexc import JitException
 
+
 class InvalidLoop(JitException):
     """Raised when the optimize*.py detect that the loop that
     we are trying to build cannot possibly make sense as a
diff --git a/rpython/jit/metainterp/optimizeopt/earlyforce.py b/rpython/jit/metainterp/optimizeopt/earlyforce.py
--- a/rpython/jit/metainterp/optimizeopt/earlyforce.py
+++ b/rpython/jit/metainterp/optimizeopt/earlyforce.py
@@ -1,7 +1,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization
-from rpython.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop
+
 
 def is_raw_free(op, opnum):
     if opnum != rop.CALL:
@@ -14,14 +14,14 @@
     def propagate_forward(self, op):
         opnum = op.getopnum()
 
-        if (opnum != rop.SETFIELD_GC and 
+        if (opnum != rop.SETFIELD_GC and
             opnum != rop.SETARRAYITEM_GC and
             opnum != rop.SETARRAYITEM_RAW and
             opnum != rop.QUASIIMMUT_FIELD and
             opnum != rop.SAME_AS and
             opnum != rop.MARK_OPAQUE_PTR and
             not is_raw_free(op, opnum)):
-               
+
             for arg in op.getarglist():
                 if arg in self.optimizer.values:
                     value = self.getvalue(arg)
@@ -33,5 +33,3 @@
 
     def setup(self):
         self.optimizer.optearlyforce = self
-
-    
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -9,6 +9,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 
+
 class BaseTestMultiLabel(BaseTest):
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
@@ -25,7 +26,7 @@
         optimized = TreeLoop('optimized')
         optimized.inputargs = loop.inputargs
         optimized.operations = []
-        
+
         labels = [i for i, op in enumerate(loop.operations) \
                   if op.getopnum()==rop.LABEL]
         prv = 0
@@ -46,10 +47,10 @@
                 last_label = [part.operations.pop()]
             else:
                 last_label = []
-            
+
             optimized.operations.extend(part.operations)
             prv = nxt + 1
-        
+
         #
         print
         print "Optimized:"
@@ -84,7 +85,7 @@
                 self.assert_equal(short_preamble, expected_short,
                                   text_right='expected short preamble')
 
-        
+
         return optimized
 
 class OptimizeoptTestMultiLabel(BaseTestMultiLabel):
@@ -138,12 +139,12 @@
         [p1]
         p2 = new_array(3, descr=arraydescr)
         label(p2)
-        p4 = new_array(2, descr=arraydescr)        
+        p4 = new_array(2, descr=arraydescr)
         jump(p4)
         """
         with raises(InvalidLoop):
             self.optimize_loop(ops, ops)
-        
+
     def test_nonmatching_arraystruct_1(self):
         ops = """
         [p1, f0]
@@ -156,7 +157,7 @@
         """
         with raises(InvalidLoop):
             self.optimize_loop(ops, ops)
-        
+
     def test_nonmatching_arraystruct_2(self):
         ops = """
         [p1, f0]
@@ -164,7 +165,7 @@
         setinteriorfield_gc(p2, 2, f0, descr=complexrealdescr)
         label(p2, f0)
         p4 = new_array(2, descr=complexarraydescr)
-        setinteriorfield_gc(p4, 0, f0, descr=complexrealdescr)        
+        setinteriorfield_gc(p4, 0, f0, descr=complexrealdescr)
         jump(p4, f0)
         """
         with raises(InvalidLoop):
@@ -213,7 +214,7 @@
         """
         with raises(InvalidLoop):
             self.optimize_loop(ops, ops)
-        
+
     def test_virtuals_turns_not_equal(self):
         ops = """
         [p1, p2]
@@ -317,7 +318,7 @@
         short = """
         [p1, i1]
         label(p1, i1)
-        i2 = getfield_gc(p1, descr=valuedescr)        
+        i2 = getfield_gc(p1, descr=valuedescr)
         jump(p1, i1, i2)
         """
         self.optimize_loop(ops, expected, expected_shorts=[short, short])
@@ -360,7 +361,7 @@
         jump(p1, i5)
         """
         self.optimize_loop(ops, exported)
-    
+
     def test_import_virtual_across_multiple_labels(self):
         ops = """
         [p0, i1]
@@ -435,25 +436,25 @@
     def test_boxed_opaque_unknown_class(self):
         ops = """
         [p1]
-        p2 = getfield_gc(p1, descr=nextdescr) 
-        mark_opaque_ptr(p2)        
+        p2 = getfield_gc(p1, descr=nextdescr)
+        mark_opaque_ptr(p2)
         i3 = getfield_gc(p2, descr=otherdescr)
         label(p1)
         i4 = getfield_gc(p1, descr=otherdescr)
         label(p1)
-        p5 = getfield_gc(p1, descr=nextdescr) 
-        mark_opaque_ptr(p5)        
+        p5 = getfield_gc(p1, descr=nextdescr)
+        mark_opaque_ptr(p5)
         i6 = getfield_gc(p5, descr=otherdescr)
         i7 = call(i6, descr=nonwritedescr)
         """
         expected = """
         [p1]
-        p2 = getfield_gc(p1, descr=nextdescr) 
+        p2 = getfield_gc(p1, descr=nextdescr)
         i3 = getfield_gc(p2, descr=otherdescr)
         label(p1)
         i4 = getfield_gc(p1, descr=otherdescr)
         label(p1)
-        p5 = getfield_gc(p1, descr=nextdescr) 
+        p5 = getfield_gc(p1, descr=nextdescr)
         i6 = getfield_gc(p5, descr=otherdescr)
         i7 = call(i6, descr=nonwritedescr)
         """
@@ -462,20 +463,20 @@
     def test_opaque_pointer_fails_to_close_loop(self):
         ops = """
         [p1, p11]
-        p2 = getfield_gc(p1, descr=nextdescr) 
+        p2 = getfield_gc(p1, descr=nextdescr)
         guard_class(p2, ConstClass(node_vtable)) []
-        mark_opaque_ptr(p2)        
+        mark_opaque_ptr(p2)
         i3 = getfield_gc(p2, descr=otherdescr)
         label(p1, p11)
-        p12 = getfield_gc(p1, descr=nextdescr) 
+        p12 = getfield_gc(p1, descr=nextdescr)
         i13 = getfield_gc(p2, descr=otherdescr)
-        i14 = call(i13, descr=nonwritedescr)        
+        i14 = call(i13, descr=nonwritedescr)
         jump(p11, p1)
         """
         with raises(InvalidLoop):
             self.optimize_loop(ops, ops)
 
-            
+
 
 
 class OptRenameStrlen(Optimization):
@@ -487,7 +488,7 @@
         newop.result = op.result.clonebox()
         self.emit_operation(newop)
         self.make_equal_to(op.result, self.getvalue(newop.result))
-    
+
 dispatch_opt = make_dispatcher_method(OptRenameStrlen, 'optimize_',
                                       default=OptRenameStrlen.emit_operation)
 
@@ -537,7 +538,7 @@
         jump(p1, i11)
         """
         self.optimize_loop(ops, expected)
-        
+
 
 class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -51,7 +51,7 @@
 class QuasiImmut(object):
     llopaque = True
     compress_limit = 30
-    
+
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,5 +1,6 @@
 from rpython.rlib.objectmodel import we_are_translated
 
+
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
     op = cls(result)
@@ -50,7 +51,6 @@
     def numargs(self):
         raise NotImplementedError
 
-
     # methods implemented by GuardResOp
     # ---------------------------------
 
@@ -183,6 +183,7 @@
 class PlainResOp(AbstractResOp):
     pass
 
+
 class ResOpWithDescr(AbstractResOp):
 
     _descr = None
@@ -347,6 +348,7 @@
         else:
             raise IndexError
 
+
 class N_aryOp(object):
     _mixin_ = True
     _args = None
@@ -577,7 +579,7 @@
         opclasses.append(cls)
         oparity.append(arity)
         opwithdescr.append(withdescr)
-    assert len(opclasses)==len(oparity)==len(opwithdescr)==len(_oplist)
+    assert len(opclasses) == len(oparity) == len(opwithdescr) == len(_oplist)
 
 def get_base_class(mixin, base):
     try:
@@ -597,7 +599,7 @@
         1: UnaryOp,
         2: BinaryOp,
         3: TernaryOp
-        }
+    }
 
     is_guard = name.startswith('GUARD')
     if is_guard:
@@ -639,7 +641,7 @@
 
     rop.PTR_EQ: rop.PTR_NE,
     rop.PTR_NE: rop.PTR_EQ,
-    }
+}
 
 opboolreflex = {
     rop.INT_EQ: rop.INT_EQ,
@@ -663,7 +665,7 @@
 
     rop.PTR_EQ: rop.PTR_EQ,
     rop.PTR_NE: rop.PTR_NE,
-    }
+}
 
 
 def get_deep_immutable_oplist(operations):
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -1,8 +1,11 @@
-import sys, py
+import sys
+
+import py
+
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator,\
-     cast_base_ptr_to_instance, hlstr
+from rpython.rtyper.annlowlevel import (llhelper, MixLevelHelperAnnotator,
+    cast_base_ptr_to_instance, hlstr)
 from rpython.annotator import model as annmodel
 from rpython.rtyper.llinterp import LLException
 from rpython.rtyper.test.test_llinterp import get_interpreter, clear_tcache
@@ -20,11 +23,12 @@
 from rpython.jit.metainterp.pyjitpl import MetaInterpStaticData
 from rpython.jit.metainterp.jitprof import Profiler, EmptyProfiler
 from rpython.jit.metainterp.jitdriver import JitDriverStaticData
-from rpython.jit.codewriter import support, codewriter, longlong
+from rpython.jit.codewriter import support, codewriter
 from rpython.jit.codewriter.policy import JitPolicy
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.optimizeopt import ALL_OPTS_NAMES
 
+
 # ____________________________________________________________
 # Bootstrapping
 
@@ -39,8 +43,8 @@
     warmrunnerdesc = WarmRunnerDesc(translator,
                                     translate_support_code=True,
                                     listops=True,
-                                    no_stats = True,
-                                    ProfilerClass = ProfilerClass,
+                                    no_stats=True,
+                                    ProfilerClass=ProfilerClass,
                                     **kwds)
     for jd in warmrunnerdesc.jitdrivers_sd:
         jd.warmstate.set_param_inlining(inline)
@@ -336,7 +340,6 @@
             c_new_driver = Constant(new_driver, v_driver.concretetype)
             op.args[1] = c_new_driver
 
-
     def find_portals(self):
         self.jitdrivers_sd = []
         graphs = self.translator.graphs
@@ -419,9 +422,12 @@
             self.annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
         cpu = CPUClass(self.translator.rtyper, self.stats, self.opt,
                        translate_support_code, gcdescr=self.gcdescr)
-        if not supports_floats:       cpu.supports_floats       = False
-        if not supports_longlong:     cpu.supports_longlong     = False
-        if not supports_singlefloats: cpu.supports_singlefloats = False
+        if not supports_floats:
+            cpu.supports_floats = False
+        if not supports_longlong:
+            cpu.supports_longlong = False
+        if not supports_singlefloats:
+            cpu.supports_singlefloats = False
         self.cpu = cpu
 
     def build_meta_interp(self, ProfilerClass):
@@ -677,6 +683,7 @@
             # get special treatment since we rewrite it to a call that accepts
             # jit driver
             func = func_with_new_name(func, func.func_name + '_compiled')
+
             def new_func(ignored, *args):
                 return func(self, *args)
             ARGS = [lltype.Void] + [arg.concretetype for arg in op.args[3:]]
@@ -734,7 +741,7 @@
         from rpython.jit.metainterp.warmstate import specialize_value
         from rpython.jit.metainterp.warmstate import unspecialize_value
         portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal',
-                                         graph = portalgraph)
+                                         graph=portalgraph)
         jd._portal_ptr = portal_ptr
         #
         portalfunc_ARGS = []
@@ -758,7 +765,6 @@
         ts = self.cpu.ts
         state = jd.warmstate
         maybe_compile_and_run = jd._maybe_compile_and_run_fn
-        cpu = jd.warmstate.cpu
 
         def ll_portal_runner(*args):
             start = True
@@ -920,7 +926,7 @@
 
         if self.cpu.translate_support_code:
             call_final_function(self.translator, finish,
-                                annhelper = self.annhelper)
+                                annhelper=self.annhelper)
 
     def rewrite_set_param_and_get_stats(self):
         from rpython.rtyper.lltypesystem.rstr import STR
@@ -952,7 +958,7 @@
             return Constant(funcptr, TP)
         #
         for graph, block, i in find_set_param(graphs):
-            
+
             op = block.operations[i]
             if op.args[1].value is not None:
                 for jd in self.jitdrivers_sd:
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -1,17 +1,18 @@
-import sys, weakref
+import sys
+import weakref
+
+from rpython.jit.codewriter import support, heaptracker, longlong
+from rpython.jit.metainterp import history
+from rpython.rlib.debug import debug_start, debug_stop, debug_print
+from rpython.rlib.jit import PARAMETERS, BaseJitCell
+from rpython.rlib.nonconst import NonConstant
+from rpython.rlib.objectmodel import specialize, we_are_translated, r_dict
+from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.unroll import unrolling_iterable
+from rpython.rtyper.annlowlevel import (hlstr, cast_base_ptr_to_instance,
+    cast_object_to_ptr)
 from rpython.rtyper.lltypesystem import lltype, llmemory, rstr, rffi
 from rpython.rtyper.ootypesystem import ootype
-from rpython.rtyper.annlowlevel import hlstr, cast_base_ptr_to_instance
-from rpython.rtyper.annlowlevel import cast_object_to_ptr
-from rpython.rlib.objectmodel import specialize, we_are_translated, r_dict
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.nonconst import NonConstant
-from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.jit import PARAMETERS
-from rpython.rlib.jit import BaseJitCell
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
-from rpython.jit.metainterp import history
-from rpython.jit.codewriter import support, heaptracker, longlong
 
 # ____________________________________________________________
 
@@ -601,7 +602,7 @@
                 fn = support.maybe_on_top_of_llinterp(rtyper, inline_ptr)
                 return fn(*greenargs)
         self.should_unroll_one_iteration = should_unroll_one_iteration
-        
+
         redargtypes = ''.join([kind[0] for kind in jd.red_args_types])
 
         def get_assembler_token(greenkey):

From noreply at buildbot.pypy.org  Thu Jun 20 08:33:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 08:33:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Write a more reasonable buffer
	implementation for mmap.
Message-ID: <20130620063354.21F881C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64945:cfbfd1251d6f
Date: 2013-06-20 08:33 +0200
http://bitbucket.org/pypy/pypy/changeset/cfbfd1251d6f/

Log:	Write a more reasonable buffer implementation for mmap.

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.buffer import RWBuffer
 from rpython.rlib import rmmap, rarithmetic
 from rpython.rlib.rmmap import RValueError, RTypeError
 
@@ -133,6 +134,8 @@
         return self.space.wrap(self.mmap.size)
 
     def check_valid(self):
+        # XXX the check_xxx() are inconsistent in this file.  Please review!
+        # For example readline() might raise an interp-level RValueError.
         try:
             self.mmap.check_valid()
         except RValueError, v:
@@ -166,6 +169,8 @@
         start, stop, step = space.decode_index(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(self.mmap.getitem(start))
+        elif step == 1:
+            return space.wrap(self.mmap.getslice(start, stop - start))
         else:
             res = "".join([self.mmap.getitem(i)
                            for i in range(start, stop, step)])
@@ -189,15 +194,16 @@
             if len(value) != length:
                 raise OperationError(space.w_ValueError,
                           space.wrap("mmap slice assignment is wrong size"))
-            for i in range(length):
-                self.mmap.setitem(start, value[i])
-                start += step
+            if step == 1:
+                self.mmap.setslice(start, value)
+            else:
+                for i in range(length):
+                    self.mmap.setitem(start, value[i])
+                    start += step
 
     def descr_buffer(self):
-        # XXX improve to work directly on the low-level address
-        from pypy.interpreter.buffer import StringLikeBuffer
-        space = self.space
-        return space.wrap(StringLikeBuffer(space, space.wrap(self)))
+        self.check_valid()
+        return self.space.wrap(MMapBuffer(self.space, self.mmap))
 
 if rmmap._POSIX:
 
@@ -277,3 +283,45 @@
 def mmap_error(space, e):
     w_error = space.fromcache(Cache).w_error
     return wrap_oserror(space, e, w_exception_class=w_error)
+
+
+class MMapBuffer(RWBuffer):
+    def __init__(self, space, mmap):
+        self.space = space
+        self.mmap = mmap
+
+    def get_raw_address(self):
+        return self.mmap.data
+
+    def getlength(self):
+        return self.mmap.size
+
+    def getitem(self, index):
+        return self.mmap.data[index]
+
+    def getslice(self, start, stop, step, size):
+        if step == 1:
+            return self.mmap._getslice(start, size)
+        else:
+            return RWBuffer.getslice(self, start, stop, step, size)
+
+    def check_writeable(self):
+        try:
+            self.mmap.check_valid()
+            self.mmap.check_writeable()
+        except RValueError, v:
+            raise OperationError(self.space.w_ValueError,
+                                 self.space.wrap(v.message))
+        except RTypeError, v:
+            raise OperationError(self.space.w_TypeError,
+                                 self.space.wrap(v.message))
+
+    def setitem(self, index, char):
+        self.mmap.check_writeable()
+        self.mmap.data[index] = char
+
+    def setslice(self, start, string):
+        self.mmap.setslice(start, string)
+
+    def get_raw_address(self):
+        return self.mmap.data
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -550,6 +550,24 @@
         m.close()
         f.close()
 
+    def test_buffer_write(self):
+        from mmap import mmap
+        f = open(self.tmpname + "y", "w+")
+        f.write("foobar")
+        f.flush()
+        m = mmap(f.fileno(), 6)
+        m[5] = '?'
+        b = buffer(m)
+        try:
+            b[:3] = "FOO"
+        except TypeError:     # on CPython: "buffer is read-only" :-/
+            skip("on CPython: buffer is read-only")
+        m.close()
+        f.seek(0)
+        got = f.read()
+        assert got == "FOOba?"
+        f.close()
+
     def test_offset(self):
         from mmap import mmap, ALLOCATIONGRANULARITY
         f = open(self.tmpname + "y", "w+")
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -3,6 +3,7 @@
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.nonconst import NonConstant
+from rpython.rlib.rarithmetic import intmask
 
 import sys
 import os
@@ -349,13 +350,12 @@
         else: # no '\n' found
             eol = self.size
 
-        res = "".join([data[i] for i in range(self.pos, eol)])
+        res = self._getslice(self.pos, eol - self.pos)
         self.pos += len(res)
         return res
 
     def read(self, num=-1):
         self.check_valid()
-
         if num < 0:
             # read all
             eol = self.size
@@ -365,8 +365,7 @@
             if eol > self.size:
                 eol = self.size
 
-        res = [self.data[i] for i in range(self.pos, eol)]
-        res = "".join(res)
+        res = self._getslice(self.pos, eol - self.pos)
         self.pos += len(res)
         return res
 
@@ -430,7 +429,7 @@
         if not (0 <= where <= self.size):
             raise RValueError("seek out of range")
 
-        self.pos = where
+        self.pos = intmask(where)
 
     def tell(self):
         self.check_valid()
@@ -458,15 +457,12 @@
     def write(self, data):
         self.check_valid()
         self.check_writeable()
-
         data_len = len(data)
-        if self.pos + data_len > self.size:
+        start = self.pos
+        if start + data_len > self.size:
             raise RValueError("data out of range")
 
-        internaldata = self.data
-        start = self.pos
-        for i in range(data_len):
-            internaldata[start+i] = data[i]
+        self._setslice(start, data)
         self.pos = start + data_len
 
     def write_byte(self, byte):
@@ -485,6 +481,23 @@
     def getptr(self, offset):
         return rffi.ptradd(self.data, offset)
 
+    def getslice(self, start, length):
+        self.check_valid()
+        return self._getslice(start, length)
+
+    def setslice(self, start, data):
+        self.check_valid()
+        self.check_writeable()
+        self._setslice(start, data)
+
+    def _getslice(self, start, length):
+        return rffi.charpsize2str(self.getptr(start), length)
+
+    def _setslice(self, start, newdata):
+        internaldata = self.data
+        for i in range(len(newdata)):
+            internaldata[start+i] = newdata[i]
+
     def flush(self, offset=0, size=0):
         self.check_valid()
 

From noreply at buildbot.pypy.org  Thu Jun 20 08:36:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 08:36:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Forgot to import the app-level parts of
 cffi/1d4ca4f4914a
Message-ID: <20130620063632.30EF41C0541@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64946:ce2b1ea66d55
Date: 2013-06-20 08:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ce2b1ea66d55/

Log:	Forgot to import the app-level parts of cffi/1d4ca4f4914a

diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -707,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -934,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
@@ -990,7 +990,7 @@
         assert f.a == 12345
         assert b.b == b"B"
         assert b.c == b"C"
-        assert repr(b).startswith("<cdata 'struct $bar_t *'")
+        assert repr(b).startswith("<cdata 'bar_t *'")
 
     def test_struct_with_two_usages(self):
         for name in ['foo_s', '']:    # anonymous or not
@@ -1300,9 +1300,9 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("typedef enum { Value0 = 0 } e, *pe;\n"
                  "typedef enum { Value1 = 1 } e1;")
-        assert ffi.getctype("e*") == 'enum $e *'
-        assert ffi.getctype("pe") == 'enum $e *'
-        assert ffi.getctype("e1*") == 'enum $e1 *'
+        assert ffi.getctype("e*") == 'e *'
+        assert ffi.getctype("pe") == 'e *'
+        assert ffi.getctype("e1*") == 'e1 *'
 
     def test_new_ctype(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -179,3 +179,9 @@
             'ffi.cdef("struct s2 { char x:0; };"); ffi.new("struct s2 *")')
         py.test.raises(TypeError,
             'ffi.cdef("struct s3 { char x:9; };"); ffi.new("struct s3 *")')
+
+    def test_struct_with_typedef(self):
+        ffi = FFI()
+        ffi.cdef("typedef struct { float x; } foo_t;")
+        p = ffi.new("foo_t *", [5.2])
+        assert repr(p).startswith("<cdata 'foo_t *' ")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
@@ -1442,7 +1442,7 @@
                        "foo_t myfunc(void) { foo_t x = { 42 }; return x; }")
     assert str(e.value) in [
         "function myfunc: 'foo_t' is used as result type, but is opaque",
-        "function myfunc: result type 'struct $foo_t' is opaque"]
+        "function myfunc: result type 'foo_t' is opaque"]
 
 def test_include():
     ffi1 = FFI()

From noreply at buildbot.pypy.org  Thu Jun 20 08:40:49 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 20 Jun 2013 08:40:49 +0200 (CEST)
Subject: [pypy-commit] stmgc default: demo_random: more checks
Message-ID: <20130620064049.BF36A1C0541@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r199:b2ce4052626c
Date: 2013-06-20 08:40 +0200
http://bitbucket.org/pypy/stmgc/changeset/b2ce4052626c/

Log:	demo_random: more checks

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -6,15 +6,19 @@
 #include <time.h>
 
 #include "stmgc.h"
+#include "stmimpl.h"
 #include "fprintcolor.h"
 
+extern revision_t get_private_rev_num(void);
+
 
 #define NUMTHREADS 4
 #define STEPS 100000
-#define NUMROOTS 10
-#define PREBUILT 3
+#define NUMROOTS 10 // per thread
+#define PREBUILT 3 // per thread
 #define MAXROOTS 1000
-#define SHARED_ROOTS 5
+#define SHARED_ROOTS 5 // shared by threads
+
 
 
 // SUPPORT
@@ -58,6 +62,9 @@
 
 
 // helper functions
+int classify(gcptr p);
+void check(gcptr p);
+
 int get_rand(int max)
 {
     return (int)(rand_r(&td.thread_seed) % (unsigned int)max);
@@ -81,15 +88,19 @@
 void push_roots()
 {
     int i;
-    for (i = 0; i < td.num_roots; i++)
+    for (i = 0; i < td.num_roots; i++) {
+        check(td.roots[i]);
         stm_push_root(td.roots[i]);
+    }
 }
 
 void pop_roots()
 {
     int i;
-    for (i = td.num_roots - 1; i >= 0; i--)
+    for (i = td.num_roots - 1; i >= 0; i--) {
         td.roots[i] = stm_pop_root();
+        check(td.roots[i]);
+    }
 }
 
 void del_root(int idx)
@@ -99,15 +110,117 @@
         td.roots[i] = td.roots[i + 1];
 }
 
-gcptr allocate_node(size_t size, int tid)
+nodeptr allocate_node()
 {
-    gcptr r;
+    nodeptr r;
     push_roots();
-    r = stm_allocate(size, tid);
+    r = (nodeptr)stm_allocate(sizeof(struct node), GCTID_STRUCT_NODE);
     pop_roots();
     return r;
 }
 
+int is_shared_prebuilt(gcptr p)
+{
+    int i;
+    for (i = 0; i < SHARED_ROOTS; i++)
+        if (shared_roots[i] == p)
+            return 1;
+    return 0;
+}
+
+void check_not_free(gcptr p)
+{
+    assert(p != NULL);
+    assert((p->h_tid & 0xFFFF) == GCTID_STRUCT_NODE);
+    if (is_shared_prebuilt(p))
+        assert(p->h_tid & GCFLAG_PREBUILT_ORIGINAL);
+}
+
+void check(gcptr p)
+{
+    if (p != NULL) {
+        check_not_free(p);
+        classify(p); // additional asserts
+    }
+}
+
+gcptr read_barrier(gcptr p)
+{
+    gcptr r = p;
+    if (p != NULL) {
+        check(p);
+        r = stm_read_barrier(p);
+        check(r);
+    }
+    return r;
+}
+
+gcptr write_barrier(gcptr p)
+{
+    gcptr w = p;
+    if (p != NULL) {
+        check(p);
+        w = stm_write_barrier(p);
+        check(w);
+    }
+    return w;
+}
+
+int in_nursery(gcptr obj)
+{
+    struct tx_descriptor *d = thread_descriptor;
+    int result1 = (d->nursery_base <= (char*)obj &&
+                   ((char*)obj) < d->nursery_end);
+    if (obj->h_tid & GCFLAG_OLD) {
+        assert(result1 == 0);
+    }
+    else {
+        /* this assert() also fails if "obj" is in another nursery than
+           the one of the current thread.  This is ok, because we
+           should not see such pointers. */
+        assert(result1 == 1);
+    }
+    return result1;
+}
+
+static const revision_t C_PRIVATE_FROM_PROTECTED = 1;
+static const revision_t C_PRIVATE                = 2;
+static const revision_t C_STUB                   = 3;
+static const revision_t C_PUBLIC                 = 4;
+static const revision_t C_BACKUP                 = 5;
+static const revision_t C_PROTECTED              = 6;
+int classify(gcptr p)
+{
+    int priv_from_prot = (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) != 0;
+    int private_other = p->h_revision == get_private_rev_num();
+    int public = (p->h_tid & GCFLAG_PUBLIC) != 0;
+    int backup = (p->h_tid & GCFLAG_BACKUP_COPY) != 0;
+    int stub = (p->h_tid & GCFLAG_STUB) != 0;
+    assert(priv_from_prot + private_other + public + backup <= 1);
+    assert(public || !stub);
+    
+    if (priv_from_prot)
+        return C_PRIVATE_FROM_PROTECTED;
+    if (private_other)
+        return C_PRIVATE;
+    if (public) {
+        if (stub) {
+            assert(!in_nursery(p));
+        }
+        else {
+            if (in_nursery(p)) {
+                assert(p->h_tid & GCFLAG_NURSERY_MOVED);
+                assert(!(p->h_revision & 1));
+            }
+            return C_PUBLIC;
+        }
+    }
+    if (backup)
+        return C_BACKUP;
+    return C_PROTECTED;
+}
+
+
 
 
 // THREAD TESTER
@@ -128,7 +241,7 @@
                                               GCTID_STRUCT_NODE);
     }
     for (i = PREBUILT; i < PREBUILT + NUMROOTS; i++) {
-        td.roots[i] = allocate_node(sizeof(struct node), GCTID_STRUCT_NODE);
+        td.roots[i] = (gcptr)allocate_node();
     }
 
 }
@@ -149,9 +262,6 @@
 
     k = get_rand(14);
 
-    if (!p) // some parts expect it to be != 0
-        p = allocate_node(sizeof(struct node), GCTID_STRUCT_NODE);
-
     switch (k) {
     case 0: // remove a root
         if (num > 0)
@@ -162,19 +272,21 @@
             p = _r;
         break;
     case 2: // add 'p' to roots
-        if (td.num_roots < MAXROOTS)
+        if (p && td.num_roots < MAXROOTS)
             td.roots[td.num_roots++] = p;
         break;
     case 3: // allocate fresh 'p'
-        p = allocate_node(sizeof(struct node), GCTID_STRUCT_NODE);
+        p = (gcptr)allocate_node();
         break;
     case 4: // set 'p' as *next in one of the roots
-        w_r = (nodeptr)stm_write_barrier(_r);
-        // XXX: do I have to read_barrier(p)?
+        check(_r);
+        w_r = (nodeptr)write_barrier(_r);
+        check((gcptr)w_r);
+        check(p);
         w_r->next = (struct node*)p;
         break;
     case 5:  // read and validate 'p'
-        stm_read_barrier(p);
+        read_barrier(p);
         break;
     case 6: // transaction break
         if (td.interruptible)
@@ -183,25 +295,27 @@
         p = NULL;
         break;
     case 7: // only do a stm_write_barrier
-        p = stm_write_barrier(p);
+        p = write_barrier(p);
         break;
     case 8:
-        p = (gcptr)(((nodeptr)stm_read_barrier(p))->next);
+        if (p)
+            p = (gcptr)(((nodeptr)read_barrier(p))->next);
         break;
     case 9: // XXX: rare events
         break;
     case 10: // only do a stm_read_barrier
-        p = stm_read_barrier(p);
+        p = read_barrier(p);
         break;
     case 11:
-        stm_read_barrier(_sr);
+        read_barrier(_sr);
         break;
     case 12:
-        stm_write_barrier(_sr);
+        write_barrier(_sr);
         break;
     case 13:
-        w_sr = (nodeptr)stm_write_barrier(_sr);
+        w_sr = (nodeptr)write_barrier(_sr);
         w_sr->next = (nodeptr)shared_roots[get_rand(SHARED_ROOTS)];
+        break;
     default:
         break;
     }

From noreply at buildbot.pypy.org  Thu Jun 20 09:31:07 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 20 Jun 2013 09:31:07 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add more stm_normalize_stolen_objects
	where needed
Message-ID: <20130620073107.328D11C0268@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r200:0efe1499373f
Date: 2013-06-20 09:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/0efe1499373f/

Log:	add more stm_normalize_stolen_objects where needed

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -698,8 +698,12 @@
   long long elapsed_time;
 
   /* acquire the lock, but don't double-acquire it if already committing */
-  if (d->public_descriptor->collection_lock != 'C')
+  if (d->public_descriptor->collection_lock != 'C') {
     spinlock_acquire(d->public_descriptor->collection_lock, 'C');
+    if (d->public_descriptor->stolen_objects.size != 0)
+      stm_normalize_stolen_objects(d);
+  }
+
 
   assert(d->active != 0);
   assert(!is_inevitable(d));
@@ -870,6 +874,8 @@
   revision_t my_lock = d->my_lock;
   wlog_t *item;
 
+  assert(d->public_descriptor->stolen_objects.size == 0);
+
   if (!g2l_any_entry(&d->public_to_private))
     return;
 
@@ -1025,6 +1031,7 @@
   gcptr *items = d->private_from_protected.items;
   revision_t new_revision = cur_time + 1;     // make an odd number
   assert(new_revision & 1);
+  assert(d->public_descriptor->stolen_objects.size == 0);
 
   for (i = 0; i < size; i++)
     {
@@ -1125,7 +1132,6 @@
   spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
   if (d->public_descriptor->stolen_objects.size != 0)
     stm_normalize_stolen_objects(d);
-
   AcquireLocks(d);
 
   if (is_inevitable(d))
@@ -1151,6 +1157,9 @@
               inev_mutex_acquire();   // wait until released
               inev_mutex_release();
               spinlock_acquire(d->public_descriptor->collection_lock, 'C');
+              if (d->public_descriptor->stolen_objects.size != 0)
+                stm_normalize_stolen_objects(d);
+
               AcquireLocks(d);
               continue;
             }
@@ -1162,6 +1171,7 @@
         if (!ValidateDuringTransaction(d, 1))
           AbortTransaction(ABRT_VALIDATE_COMMIT);
     }
+
   CommitPrivateFromProtected(d, cur_time);
 
   /* we cannot abort any more from here */

From noreply at buildbot.pypy.org  Thu Jun 20 09:36:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 09:36:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Change check_valid() to make its usage
 explicit at the level of RPython:
Message-ID: <20130620073635.319BF1C0268@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64947:c924be84e77b
Date: 2013-06-20 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/c924be84e77b/

Log:	Change check_valid() to make its usage explicit at the level of
	RPython: users of rmmap.py have to call the check_*() methods
	explicitly if they want that behavior. The result is a bit more
	consistent.

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.buffer import RWBuffer
 from rpython.rlib import rmmap, rarithmetic
-from rpython.rlib.rmmap import RValueError, RTypeError
+from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -21,13 +21,14 @@
         self.mmap.close()
 
     def read_byte(self):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.read_byte())
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def readline(self):
+        self.check_valid()
         return self.space.wrap(self.mmap.readline())
 
     @unwrap_spec(num=int)
@@ -37,6 +38,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def find(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -50,6 +52,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def rfind(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -63,16 +66,18 @@
 
     @unwrap_spec(pos=OFF_T, whence=int)
     def seek(self, pos, whence=0):
+        self.check_valid()
         try:
             self.mmap.seek(pos, whence)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def tell(self):
+        self.check_valid()
         return self.space.wrap(self.mmap.tell())
 
     def descr_size(self):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.file_size())
         except OSError, e:
@@ -80,41 +85,40 @@
 
     @unwrap_spec(data='bufferstr')
     def write(self, data):
+        self.check_valid()
         self.check_writeable()
         try:
             self.mmap.write(data)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(byte=str)
     def write_byte(self, byte):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.write_byte(byte)
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(offset=int, size=int)
     def flush(self, offset=0, size=0):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.flush(offset, size))
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
         except OSError, e:
             raise mmap_error(self.space, e)
 
     @unwrap_spec(dest=int, src=int, count=int)
     def move(self, dest, src, count):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.move(dest, src, count)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(newsize=int)
     def resize(self, newsize):
@@ -134,33 +138,22 @@
         return self.space.wrap(self.mmap.size)
 
     def check_valid(self):
-        # XXX the check_xxx() are inconsistent in this file.  Please review!
-        # For example readline() might raise an interp-level RValueError.
         try:
             self.mmap.check_valid()
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def check_writeable(self):
         try:
             self.mmap.check_writeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def check_resizeable(self):
         try:
             self.mmap.check_resizeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def descr_getitem(self, w_index):
         self.check_valid()
@@ -219,10 +212,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 elif rmmap._MS_WINDOWS:
@@ -238,10 +229,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 W_MMap.typedef = TypeDef("mmap",
@@ -281,8 +270,19 @@
                                                  space.w_EnvironmentError)
 
 def mmap_error(space, e):
-    w_error = space.fromcache(Cache).w_error
-    return wrap_oserror(space, e, w_exception_class=w_error)
+    if isinstance(e, RValueError):
+        return OperationError(space.w_ValueError,
+                              space.wrap(e.message))
+    elif isinstance(e, RTypeError):
+        return OperationError(space.w_TypeError,
+                              space.wrap(e.message))
+    elif isinstance(e, OSError):
+        w_error = space.fromcache(Cache).w_error
+        return wrap_oserror(space, e, w_exception_class=w_error)
+    else:
+        # bogus 'e'?
+        return OperationError(space.w_SystemError, space.wrap(repr(e)))
+mmap_error._dont_inline_ = True
 
 
 class MMapBuffer(RWBuffer):
@@ -297,31 +297,37 @@
         return self.mmap.size
 
     def getitem(self, index):
+        self.check_valid()
         return self.mmap.data[index]
 
     def getslice(self, start, stop, step, size):
+        self.check_valid()
         if step == 1:
-            return self.mmap._getslice(start, size)
+            return self.mmap.getslice(start, size)
         else:
             return RWBuffer.getslice(self, start, stop, step, size)
 
-    def check_writeable(self):
+    def setitem(self, index, char):
+        self.check_valid_writeable()
+        self.mmap.data[index] = char
+
+    def setslice(self, start, string):
+        self.check_valid_writeable()
+        self.mmap.setslice(start, string)
+
+    def get_raw_address(self):
+        self.check_valid()
+        return self.mmap.data
+
+    def check_valid(self):
+        try:
+            self.mmap.check_valid()
+        except RValueError, v:
+            raise mmap_error(self.space, v)
+
+    def check_valid_writeable(self):
         try:
             self.mmap.check_valid()
             self.mmap.check_writeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
-
-    def setitem(self, index, char):
-        self.mmap.check_writeable()
-        self.mmap.data[index] = char
-
-    def setslice(self, start, string):
-        self.mmap.setslice(start, string)
-
-    def get_raw_address(self):
-        return self.mmap.data
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -1,3 +1,11 @@
+"""Interp-level mmap-like object.
+
+Note that all the methods assume that the mmap is valid (or writable, for
+writing methods).  You have to call check_valid() from the higher-level API,
+as well as maybe check_writeable().  In the case of PyPy, this is done from
+pypy/module/mmap/.
+"""
+
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rposix
@@ -15,13 +23,15 @@
 _64BIT = "64bit" in platform.architecture()[0]
 _CYGWIN = "cygwin" == sys.platform
 
-class RValueError(Exception):
+class RMMapError(Exception):
     def __init__(self, message):
         self.message = message
 
-class RTypeError(Exception):
-    def __init__(self, message):
-        self.message = message
+class RValueError(RMMapError):
+    pass
+
+class RTypeError(RMMapError):
+    pass
 
 includes = ["sys/types.h"]
 if _POSIX:
@@ -330,8 +340,6 @@
         self.close()
 
     def read_byte(self):
-        self.check_valid()
-
         if self.pos < self.size:
             value = self.data[self.pos]
             self.pos += 1
@@ -340,8 +348,6 @@
             raise RValueError("read byte out of range")
 
     def readline(self):
-        self.check_valid()
-
         data = self.data
         for pos in xrange(self.pos, self.size):
             if data[pos] == '\n':
@@ -350,12 +356,11 @@
         else: # no '\n' found
             eol = self.size
 
-        res = self._getslice(self.pos, eol - self.pos)
+        res = self.getslice(self.pos, eol - self.pos)
         self.pos += len(res)
         return res
 
     def read(self, num=-1):
-        self.check_valid()
         if num < 0:
             # read all
             eol = self.size
@@ -365,13 +370,11 @@
             if eol > self.size:
                 eol = self.size
 
-        res = self._getslice(self.pos, eol - self.pos)
+        res = self.getslice(self.pos, eol - self.pos)
         self.pos += len(res)
         return res
 
     def find(self, tofind, start, end, reverse=False):
-        self.check_valid()
-
         # XXX naive! how can we reuse the rstr algorithm?
         if start < 0:
             start += self.size
@@ -412,8 +415,6 @@
             p += step
 
     def seek(self, pos, whence=0):
-        self.check_valid()
-
         dist = pos
         how = whence
 
@@ -432,12 +433,9 @@
         self.pos = intmask(where)
 
     def tell(self):
-        self.check_valid()
         return self.pos
 
     def file_size(self):
-        self.check_valid()
-
         size = self.size
         if _MS_WINDOWS:
             if self.file_handle != INVALID_HANDLE:
@@ -455,23 +453,18 @@
         return size
 
     def write(self, data):
-        self.check_valid()
-        self.check_writeable()
         data_len = len(data)
         start = self.pos
         if start + data_len > self.size:
             raise RValueError("data out of range")
 
-        self._setslice(start, data)
+        self.setslice(start, data)
         self.pos = start + data_len
 
     def write_byte(self, byte):
-        self.check_valid()
-
         if len(byte) != 1:
             raise RTypeError("write_byte() argument must be char")
 
-        self.check_writeable()
         if self.pos >= self.size:
             raise RValueError("write byte out of range")
 
@@ -482,25 +475,14 @@
         return rffi.ptradd(self.data, offset)
 
     def getslice(self, start, length):
-        self.check_valid()
-        return self._getslice(start, length)
-
-    def setslice(self, start, data):
-        self.check_valid()
-        self.check_writeable()
-        self._setslice(start, data)
-
-    def _getslice(self, start, length):
         return rffi.charpsize2str(self.getptr(start), length)
 
-    def _setslice(self, start, newdata):
+    def setslice(self, start, newdata):
         internaldata = self.data
         for i in range(len(newdata)):
             internaldata[start+i] = newdata[i]
 
     def flush(self, offset=0, size=0):
-        self.check_valid()
-
         if size == 0:
             size = self.size
         if offset < 0 or size < 0 or offset + size > self.size:
@@ -521,10 +503,6 @@
         return 0
 
     def move(self, dest, src, count):
-        self.check_valid()
-
-        self.check_writeable()
-
         # check boundings
         if (src < 0 or dest < 0 or count < 0 or
                 src + count > self.size or dest + count > self.size):
@@ -535,10 +513,6 @@
         c_memmove(datadest, datasrc, count)
 
     def resize(self, newsize):
-        self.check_valid()
-
-        self.check_resizeable()
-
         if _POSIX:
             if not has_mremap:
                 raise RValueError("mmap: resizing not available--no mremap()")
@@ -597,21 +571,15 @@
             raise winerror
 
     def len(self):
-        self.check_valid()
-
         return self.size
 
     def getitem(self, index):
-        self.check_valid()
         # simplified version, for rpython
         if index < 0:
             index += self.size
         return self.data[index]
 
     def setitem(self, index, value):
-        self.check_valid()
-        self.check_writeable()
-
         if len(value) != 1:
             raise RValueError("mmap assignment must be "
                               "single-character string")
diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py
--- a/rpython/rlib/test/test_rmmap.py
+++ b/rpython/rlib/test/test_rmmap.py
@@ -99,7 +99,7 @@
             m = mmap.mmap(no, 1)
             m.close()
             try:
-                m.read(1)
+                m.check_valid()
             except RValueError:
                 pass
             else:
@@ -214,13 +214,13 @@
         def func(no):
             m = mmap.mmap(no, 6, access=mmap.ACCESS_READ)
             try:
-                m.write('x')
+                m.check_writeable()
             except RTypeError:
                 pass
             else:
                 assert False
             try:
-                m.resize(7)
+                m.check_resizeable()
             except RTypeError:
                 pass
             else:
@@ -286,7 +286,7 @@
         f.write("foobar")
         f.flush()
         m = mmap.mmap(f.fileno(), 6, prot=mmap.PROT_READ)
-        py.test.raises(RTypeError, m.write, "foo")
+        py.test.raises(RTypeError, m.check_writeable)
         m.close()
         f.close()
 
@@ -297,8 +297,8 @@
         f.write("foobar")
         f.flush()
         m = mmap.mmap(f.fileno(), 6, prot=~mmap.PROT_WRITE)
-        py.test.raises(RTypeError, m.write_byte, 'a')
-        py.test.raises(RTypeError, m.write, "foo")
+        py.test.raises(RTypeError, m.check_writeable)
+        py.test.raises(RTypeError, m.check_writeable)
         m.close()
         f.close()
 

From noreply at buildbot.pypy.org  Thu Jun 20 10:14:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 10:14:51 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Revert a part of 9507e1b05a3c: it's
 not the first time I mess this up.
Message-ID: <20130620081451.91BE31C10EF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r201:6b3036be4bb8
Date: 2013-06-20 10:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/6b3036be4bb8/

Log:	Revert a part of 9507e1b05a3c: it's not the first time I mess this
	up. Add a comment.

diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -100,9 +100,10 @@
         // results from consecutive read_barriers can differ. needs Ptr_Eq()
         int i = 0;
         while (!(stm_read_barrier((gcptr)r_prev->next) ==
-                 stm_read_barrier((gcptr)r_current) &&
+                 ((gcptr)r_current) &&
                  stm_read_barrier((gcptr)r_current->next) ==
-                 stm_read_barrier((gcptr)r_next))) {
+                 ((gcptr)r_next))) {
+            abort();
             asm volatile ("pause":::"memory");  /* smp_spinloop() */
             i++;
             assert(i < 1000);
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -640,15 +640,34 @@
       if (v >= LOCKED)            // locked
         {
           if (!during_commit)
-            assert(v != d->my_lock);    // we don't hold any lock
-
-          if (v != d->my_lock)
             {
-              /* not locked by me: spinloop until the other thread releases
-                 its lock */
+              assert(v != d->my_lock);    // we don't hold any lock
+              /* spinloop until the other thread releases its lock */
               SpinLoop(SPLP_LOCKED_VALIDATE);
               goto retry;
             }
+          else
+            {
+              if (v != d->my_lock)         // not locked by me: conflict
+                {
+                  /* A case that can occur: two threads A and B are both
+                     committing, thread A locked object a, thread B
+                     locked object b, and then thread A tries to
+                     validate the reads it did on object b and
+                     vice-versa.  In this case both threads cannot
+                     commit, but if they both enter the SpinLoop()
+                     above, then they will livelock.
+
+                     XXX This might lead both threads to cancel by
+                     reaching this point.  It might be possible to be
+                     more clever and let one of the threads commit
+                     anyway.
+                  */
+                  fprintf(stderr, "validation failed: "
+                          "%p is locked by another thread\n", R);
+                  return 0;
+                }
+            }
         }
     }
   return 1;

From noreply at buildbot.pypy.org  Thu Jun 20 10:16:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 10:16:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Revert changes, checked in by accident
Message-ID: <20130620081602.87B361C3013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r202:490f279910d8
Date: 2013-06-20 10:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/490f279910d8/

Log:	Revert changes, checked in by accident

diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -100,10 +100,9 @@
         // results from consecutive read_barriers can differ. needs Ptr_Eq()
         int i = 0;
         while (!(stm_read_barrier((gcptr)r_prev->next) ==
-                 ((gcptr)r_current) &&
+                 stm_read_barrier((gcptr)r_current) &&
                  stm_read_barrier((gcptr)r_current->next) ==
-                 ((gcptr)r_next))) {
-            abort();
+                 stm_read_barrier((gcptr)r_next))) {
             asm volatile ("pause":::"memory");  /* smp_spinloop() */
             i++;
             assert(i < 1000);

From noreply at buildbot.pypy.org  Thu Jun 20 10:54:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 10:54:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: Uh, fix
Message-ID: <20130620085423.AE8411C331F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64948:388909aea951
Date: 2013-06-20 10:53 +0200
http://bitbucket.org/pypy/pypy/changeset/388909aea951/

Log:	Uh, fix

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -281,7 +281,7 @@
         return wrap_oserror(space, e, w_exception_class=w_error)
     else:
         # bogus 'e'?
-        return OperationError(space.w_SystemError, space.wrap(repr(e)))
+        return OperationError(space.w_SystemError, space.wrap('%s' % e))
 mmap_error._dont_inline_ = True
 
 

From noreply at buildbot.pypy.org  Thu Jun 20 13:05:26 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 20 Jun 2013 13:05:26 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: implemented interpreter proxy
 function loadFunctionFrom
Message-ID: <20130620110526.7BD2E1C138E@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r466:6809cec55ee1
Date: 2013-06-19 16:35 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/6809cec55ee1/

Log:	implemented interpreter proxy function loadFunctionFrom implemented
	primitiveFail without raising ProxyFunctionFailedError

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -57,6 +57,8 @@
                     c_arg = c_arguments[i]
                     if spec is oop:
                         args += (IProxy.oop_to_object(c_arg), )
+                    elif spec is str:
+                        args += (rffi.charp2str(c_arg), )
                     else:
                         args += (c_arg, )
                 result = func(*args)
@@ -64,11 +66,6 @@
                 if result_type is oop:
                     assert isinstance(result, model.W_Object)
                     return IProxy.object_to_oop(result)
-                elif result_type is list:
-                    if isinstance(result, list):
-                        return IProxy.list_to_carray(result)
-                    else:
-                        return result
                 elif result_type in (int, float):
                     assert isinstance(result, result_type)
                     return result
@@ -544,7 +541,8 @@
 
 @expose_on_virtual_machine_proxy([], int)
 def primitiveFail():
-    raise ProxyFunctionFailed
+    IProxy.failed()
+    return 0
 
 @expose_on_virtual_machine_proxy([oop, int, int, int, int], int)
 def showDisplayBitsLeftTopRightBottom(w_dest_form, l, t, r, b):
@@ -558,8 +556,6 @@
         w_bitmap = w_dest_form.fetch(space, 0)
         assert isinstance(w_bitmap, model.W_DisplayBitmap)
         w_bitmap.flush_to_screen()
-    else:
-        print 'Drawn, but not flashed',
     return 0
 
 @expose_on_virtual_machine_proxy([int], int)
@@ -668,9 +664,9 @@
 # #endif
 
 # #if VM_PROXY_MINOR > 3
- at expose_on_virtual_machine_proxy([str, str], bool, minor=3)
+ at expose_on_virtual_machine_proxy([str, str], int, minor=3)
 def ioLoadFunctionFrom(fnName, modName):
-    raise ProxyFunctionFailed
+    return rffi.cast(sqInt, IProxy.loadFunctionFrom(modName, fnName))
 
 @expose_on_virtual_machine_proxy([], bool, minor=3)
 def ioMicroMSecs():
@@ -998,29 +994,34 @@
         self.fail_reason = 0
 
     def call(self, signature, interp, s_frame, argcount, s_method):
-        from rpython.rlib.rdynload import dlsym
         self.initialize_from_call(signature, interp, s_frame, argcount, s_method)
         try:
-            if signature[0] not in self.loaded_modules:
-                module = self.load_and_initialize(signature[0])
-            else:
-                module = self.loaded_modules[signature[0]]
-
-            # call the correct function in it...
-            try:
-                _external_function = dlsym(module, signature[1])
-            except KeyError:
-                self.failed()
-            else:
-                external_function = rffi.cast(func_bool_void, _external_function)
-                print "Calling %s >> %s" % signature
-                external_function()
+            # eventual errors are caught by the calling function (EXTERNAL_CALL)
+            external_function = rffi.cast(func_bool_void,
+                            self.loadFunctionFrom(signature[0], signature[1]))
+            print "Calling %s >> %s" % signature
+            external_function()
 
             if not self.fail_reason == 0:
                 raise error.PrimitiveFailedError
         finally:
             self.reset()
 
+    def loadFunctionFrom(self, module_name, function_name):
+        from rpython.rlib.rdynload import dlsym
+        if module_name not in self.loaded_modules:
+            module = self.load_and_initialize(module_name)
+        else:
+            module = self.loaded_modules[module_name]
+
+        try:
+            _external_function = dlsym(module, function_name)
+        except KeyError:
+            raise ProxyFunctionFailed
+        else:
+            return _external_function
+
+
     def initialize_from_call(self, signature, interp, s_frame, argcount, s_method):
         self.interp = interp
         self.s_frame = s_frame
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -801,6 +801,7 @@
 
 class W_WordsObject(W_AbstractObjectWithClassReference):
     _attrs_ = ['words', 'c_words', '_size']
+    _immutable_fields_ = ['_size']
 
     def __init__(self, space, w_class, size):
         W_AbstractObjectWithClassReference.__init__(self, space, w_class)

From noreply at buildbot.pypy.org  Thu Jun 20 13:05:25 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 20 Jun 2013 13:05:25 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added firstIndexableField
Message-ID: <20130620110525.2A2301C10EF@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r465:299db1bfe2dc
Date: 2013-06-19 12:53 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/299db1bfe2dc/

Log:	added firstIndexableField added secondary layout to
	WordsObjects which reads/writes from/into a heap-c-array

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -65,8 +65,10 @@
                     assert isinstance(result, model.W_Object)
                     return IProxy.object_to_oop(result)
                 elif result_type is list:
-                    assert isinstance(result, list)
-                    return IProxy.list_to_carray(result)
+                    if isinstance(result, list):
+                        return IProxy.list_to_carray(result)
+                    else:
+                        return result
                 elif result_type in (int, float):
                     assert isinstance(result, result_type)
                     return result
@@ -239,7 +241,10 @@
 @expose_on_virtual_machine_proxy([oop], list)
 def firstIndexableField(w_object):
     # return a list with values (?) of w_objects variable-parts
-    raise ProxyFunctionFailed
+    if isinstance(w_object, model.W_WordsObject):
+        return w_object.convert_to_c_layout()
+    else:
+        raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([int, oop], oop)
 def literalofMethod(offset, w_method):
@@ -542,8 +547,20 @@
     raise ProxyFunctionFailed
 
 @expose_on_virtual_machine_proxy([oop, int, int, int, int], int)
-def showDisplayBitsLeftTopRightBottom(w_form, l, t, r, b):
-    raise ProxyFunctionFailed
+def showDisplayBitsLeftTopRightBottom(w_dest_form, l, t, r, b):
+    # "Repaint the portion of the Smalltalk screen bounded by the affected
+    # rectangle. Used to synchronize the screen after a Bitblt to the Smalltalk
+    # Display object."
+    # We don't need to copy, because we let the stuf directly write into the
+    # display memory
+    space = IProxy.space
+    if w_dest_form.is_same_object(space.objtable['w_display']):
+        w_bitmap = w_dest_form.fetch(space, 0)
+        assert isinstance(w_bitmap, model.W_DisplayBitmap)
+        w_bitmap.flush_to_screen()
+    else:
+        print 'Drawn, but not flashed',
+    return 0
 
 @expose_on_virtual_machine_proxy([int], int)
 def signalSemaphoreWithIndex(n):
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -800,14 +800,16 @@
         return True
 
 class W_WordsObject(W_AbstractObjectWithClassReference):
-    _attrs_ = ['words']
+    _attrs_ = ['words', 'c_words', '_size']
 
     def __init__(self, space, w_class, size):
         W_AbstractObjectWithClassReference.__init__(self, space, w_class)
         self.words = [r_uint(0)] * size
+        self._size = size
 
     def fillin(self, space, g_self):
         self.words = g_self.get_ruints()
+        self._size = len(self.words)
         self.s_class = g_self.get_class().as_class_get_penumbra(space)
         self.hash = g_self.get_hash()
         self.space = space
@@ -821,10 +823,16 @@
         self.setword(index0, word)
 
     def getword(self, n):
-        return self.words[n]
+        if self.words is not None:
+            return self.words[n]
+        else:
+            return self.c_words[n]
 
     def setword(self, n, word):
-        self.words[n] = r_uint(word)
+        if self.words is not None:
+            self.words[n] = r_uint(word)
+        else:
+            self.c_words[n] = intmask(word)
 
     def short_at0(self, space, index0):
         word = intmask(self.getword(index0 / 2))
@@ -851,15 +859,19 @@
         self.setword(word_index0, value)
 
     def size(self):
-        return len(self.words)
+        return self._size
 
     def invariant(self):
         return (W_AbstractObjectWithClassReference.invariant(self) and
                 isinstance(self.words, list))
 
     def clone(self, space):
-        w_result = W_WordsObject(self.space, self.getclass(space), len(self.words))
-        w_result.words = list(self.words)
+        size = self.size()
+        w_result = W_WordsObject(self.space, self.getclass(space), size)
+        if self.words is not None:
+            w_result.words = list(self.words)
+        else:
+            w_result.words = [self.c_words[i] for i in range(size)]
         return w_result
 
     def as_repr_string(self):
@@ -869,6 +881,24 @@
     def is_array_object(self):
         return True
 
+    def convert_to_c_layout(self):
+        if self.words is None:
+            return self.c_words
+        else:
+            from spyvm.interpreter_proxy import sqIntArrayPtr
+            size = self.size()
+            old_words = self.words
+            c_words = self.c_words = lltype.malloc(sqIntArrayPtr.TO, size, flavor='raw')
+            for i in range(size):
+                c_words[i] = intmask(old_words[i])
+            self.words = None
+            return c_words
+
+    def __del__(self):
+        if self.words is None:
+            lltype.free(self.c_words, 'raw')
+
+
 NATIVE_DEPTH = 32
 
 class W_DisplayBitmap(W_AbstractObjectWithClassReference):
diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -61,6 +61,19 @@
     assert w_bytes.getword(0) == 0
     py.test.raises(IndexError, lambda: w_bytes.getword(20))
 
+def test_c_word_object():
+    w_class = mockclass(space, 0, format=shadow.WORDS)
+    w_bytes = w_class.as_class_get_shadow(space).new(20)
+    w_bytes.convert_to_c_layout()
+    assert w_bytes.getclass(space).is_same_object(w_class)
+    assert w_bytes.size() == 20
+    assert w_class.as_class_get_shadow(space).instsize() == 0
+    assert w_bytes.getword(3) == 0
+    w_bytes.setword(3, 42)
+    assert w_bytes.getword(3) == 42
+    assert w_bytes.getword(0) == 0
+    py.test.raises(IndexError, lambda: w_bytes.getword(20))
+
 def test_method_lookup():
     class mockmethod(object):
         def __init__(self, val):

From noreply at buildbot.pypy.org  Thu Jun 20 13:05:28 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 20 Jun 2013 13:05:28 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: implemented c_layout for
	BytesObject
Message-ID: <20130620110528.0FCAC1C10EF@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r467:469aa663812c
Date: 2013-06-19 16:36 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/469aa663812c/

Log:	implemented c_layout for BytesObject added immutable hint for _size

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -240,6 +240,8 @@
     # return a list with values (?) of w_objects variable-parts
     if isinstance(w_object, model.W_WordsObject):
         return w_object.convert_to_c_layout()
+    elif isinstance(w_object, model.W_BytesObject):
+        return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
     else:
         raise ProxyFunctionFailed
 
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -709,16 +709,19 @@
         return w_result
 
 class W_BytesObject(W_AbstractObjectWithClassReference):
-    _attrs_ = ['bytes']
+    _attrs_ = ['bytes', 'c_bytes', '_size']
+    _immutable_fields_ = ['_size']
 
     def __init__(self, space, w_class, size):
         W_AbstractObjectWithClassReference.__init__(self, space, w_class)
         assert isinstance(size, int)
         self.bytes = ['\x00'] * size
+        self._size = size
 
     def fillin(self, space, g_self):
         self.s_class = g_self.get_class().as_class_get_penumbra(space)
         self.bytes = g_self.get_bytes()
+        self._size = len(self.bytes)
         self.hash = g_self.get_hash()
         self.space = space
 
@@ -729,11 +732,19 @@
         self.setchar(index0, chr(space.unwrap_int(w_value)))
 
     def getchar(self, n0):
-        return self.bytes[n0]
+        if self.bytes is not None:
+            return self.bytes[n0]
+        else:
+            if n0 >= self._size:
+                raise IndexError
+            return self.c_bytes[n0]
 
     def setchar(self, n0, character):
         assert len(character) == 1
-        self.bytes[n0] = character
+        if self.bytes is not None:
+            self.bytes[n0] = character
+        else:
+            self.c_bytes[n0] = character
 
     def short_at0(self, space, index0):
         byte_index0 = index0 * 2
@@ -755,7 +766,7 @@
         self.setchar(byte_index0 + 1, chr(byte1))
 
     def size(self):
-        return len(self.bytes)
+        return self._size
 
     def __str__(self):
         return self.as_string()
@@ -765,7 +776,10 @@
             className='W_BytesObject', additionalInformation=self.as_string())
 
     def as_string(self):
-        return "".join(self.bytes)
+        if self.bytes is not None:
+            return "".join(self.bytes)
+        else:
+            return "".join([self.c_bytes[i] for i in range(self.size())])
 
     def invariant(self):
         if not W_AbstractObjectWithClassReference.invariant(self):
@@ -779,11 +793,24 @@
         # XXX this sounds very wrong to me
         if not isinstance(other, W_BytesObject):
             return False
-        return self.bytes == other.bytes
+        if self.bytes is not None and other.bytes is not None:
+            return self.bytes == other.bytes
+        else:
+            size = self.size()
+            if size != other.size():
+                return False
+            for i in range(size):
+                if self.getchar(i) != other.getchar(i):
+                    return False
+            return True
 
     def clone(self, space):
-        w_result = W_BytesObject(self.space, self.getclass(space), len(self.bytes))
-        w_result.bytes = list(self.bytes)
+        size = self.size()
+        w_result = W_BytesObject(self.space, self.getclass(space), size)
+        if self.bytes is not None:
+            w_result.bytes = list(self.bytes)
+        else:
+            w_result.bytes = [self.c_bytes[i] for i in range(size)]
         return w_result
 
     def unwrap_uint(self, space):
@@ -799,6 +826,19 @@
     def is_array_object(self):
         return True
 
+    def convert_to_c_layout(self):
+        if self.bytes is None:
+            return self.c_bytes
+        else:
+            size = self.size()
+            c_bytes = self.c_bytes = rffi.str2charp(self.bytes)
+            self.bytes = None
+            return c_bytes
+
+    def __del__(self):
+        if self.bytes is None:
+            rffi.free_charp(self.c_bytes)
+
 class W_WordsObject(W_AbstractObjectWithClassReference):
     _attrs_ = ['words', 'c_words', '_size']
     _immutable_fields_ = ['_size']
diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -49,6 +49,19 @@
     assert w_bytes.getchar(0) == "\x00"
     py.test.raises(IndexError, lambda: w_bytes.getchar(20))
 
+def test_c_bytes_object():
+    w_class = mockclass(space, 0, format=shadow.BYTES)
+    w_bytes = w_class.as_class_get_shadow(space).new(20)
+    w_bytes.convert_to_c_layout()
+    assert w_bytes.getclass(space).is_same_object(w_class)
+    assert w_bytes.size() == 20
+    assert w_class.as_class_get_shadow(space).instsize() == 0
+    assert w_bytes.getchar(3) == "\x00"
+    w_bytes.setchar(3, "\xAA")
+    assert w_bytes.getchar(3) == "\xAA"
+    assert w_bytes.getchar(0) == "\x00"
+    py.test.raises(IndexError, lambda: w_bytes.getchar(20))
+
 def test_word_object():
     w_class = mockclass(space, 0, format=shadow.WORDS)
     w_bytes = w_class.as_class_get_shadow(space).new(20)

From noreply at buildbot.pypy.org  Thu Jun 20 13:05:29 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 20 Jun 2013 13:05:29 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: masked print statements in
 interpreter_proxy with trace-checks
Message-ID: <20130620110529.369DC1C10EF@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r468:b149a1a7719f
Date: 2013-06-20 13:02 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/b149a1a7719f/

Log:	masked print statements in interpreter_proxy with trace-checks

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -51,7 +51,8 @@
         def wrapped(*c_arguments):
             assert len_unwrap_spec == len(c_arguments)
             args = ()
-            print 'Called InterpreterProxy >> %s' % func.func_name,
+            if IProxy.interp.trace:
+                print 'Called InterpreterProxy >> %s' % func.func_name,
             try:
                 for i, spec in unrolling_unwrap_spec:
                     c_arg = c_arguments[i]
@@ -62,7 +63,8 @@
                     else:
                         args += (c_arg, )
                 result = func(*args)
-                print '\t-> %s' % result
+                if IProxy.interp.trace:
+                    print '\t-> %s' % result
                 if result_type is oop:
                     assert isinstance(result, model.W_Object)
                     return IProxy.object_to_oop(result)
@@ -80,6 +82,9 @@
             except error.PrimitiveFailedError:
                 print '\t-> failed'
                 IProxy.failed()
+                from rpython.rlib.objectmodel import we_are_translated
+                if not we_are_translated():
+                    import pdb; pdb.set_trace()
                 if mapping[result_type] is sqInt:
                     return 0
                 elif mapping[result_type] is sqDouble:
@@ -1001,7 +1006,8 @@
             # eventual errors are caught by the calling function (EXTERNAL_CALL)
             external_function = rffi.cast(func_bool_void,
                             self.loadFunctionFrom(signature[0], signature[1]))
-            print "Calling %s >> %s" % signature
+            if interp.trace:
+                print "Calling %s >> %s" % signature
             external_function()
 
             if not self.fail_reason == 0:
@@ -1048,7 +1054,7 @@
             return self.object_map[w_object]
         except KeyError:
             new_index = self.next_oop()
-            print "Mapping new Object: %d -> %s" % (new_index, w_object)
+            # print "Mapping new Object: %d -> %s" % (new_index, w_object)
             self.oop_map[new_index] = w_object
             self.object_map[w_object] = new_index
             return new_index

From noreply at buildbot.pypy.org  Thu Jun 20 13:05:30 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 20 Jun 2013 13:05:30 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added convert_to_c_layout to
	W_DisplayBitmap
Message-ID: <20130620110530.4282D1C10EF@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r469:d5040bb62aaf
Date: 2013-06-20 13:04 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/d5040bb62aaf/

Log:	added convert_to_c_layout to W_DisplayBitmap removing the
	(now wrongful) caching is still pending

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -247,6 +247,8 @@
         return w_object.convert_to_c_layout()
     elif isinstance(w_object, model.W_BytesObject):
         return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
+    elif isinstance(w_object, model.W_DisplayBitmap):
+        return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
     else:
         raise ProxyFunctionFailed
 
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -998,6 +998,9 @@
     def is_array_object(self):
         return True
 
+    def convert_to_c_layout(self):
+        return self.pixelbuffer
+
 class W_DisplayBitmap1Bit(W_DisplayBitmap):
     def getword(self, n):
         return self._real_depth_buffer[n]

From noreply at buildbot.pypy.org  Thu Jun 20 13:05:31 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 20 Jun 2013 13:05:31 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed translation and type
 problems with the new c_layout changes to bytes and words objects
Message-ID: <20130620110531.59AA81C10EF@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r470:046f02416b4f
Date: 2013-06-20 13:05 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/046f02416b4f/

Log:	fixed translation and type problems with the new c_layout changes to
	bytes and words objects

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -523,7 +523,7 @@
     def size(self):
         if self.has_shadow():
             return self._shadow.size()
-        return self._size()
+        return self.basic_size()
 
     def store_shadow(self, shadow):
         assert self._shadow is None or self._shadow is shadow
@@ -643,7 +643,7 @@
         fieldtypes = jit.promote(self.fieldtypes)
         return fieldtypes.store(self, n0, w_value)
 
-    def _size(self):
+    def basic_size(self):
         return len(self._vars)
 
     def invariant(self):
@@ -685,7 +685,7 @@
         assert w_value is not None
         self._weakvars[n0] = weakref.ref(w_value)
 
-    def _size(self):
+    def basic_size(self):
         return len(self._weakvars)
 
     def invariant(self):
@@ -831,7 +831,7 @@
             return self.c_bytes
         else:
             size = self.size()
-            c_bytes = self.c_bytes = rffi.str2charp(self.bytes)
+            c_bytes = self.c_bytes = rffi.str2charp(self.as_string())
             self.bytes = None
             return c_bytes
 
@@ -867,7 +867,7 @@
         if self.words is not None:
             return self.words[n]
         else:
-            return self.c_words[n]
+            return r_uint(self.c_words[n])
 
     def setword(self, n, word):
         if self.words is not None:
@@ -912,7 +912,7 @@
         if self.words is not None:
             w_result.words = list(self.words)
         else:
-            w_result.words = [self.c_words[i] for i in range(size)]
+            w_result.words = [r_uint(self.c_words[i]) for i in range(size)]
         return w_result
 
     def as_repr_string(self):
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -27,7 +27,7 @@
     def store(self, n0, w_value):
         return self.w_self()._store(n0, w_value)
     def size(self):
-        return self.w_self()._size()
+        return self.w_self().basic_size()
     def w_self(self):
         return self._w_self
     def getname(self):

From noreply at buildbot.pypy.org  Thu Jun 20 13:27:09 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 20 Jun 2013 13:27:09 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed weird translation bug
 where keyword arguments are required by translator,
 instead of relying on sequence
Message-ID: <20130620112709.E025D1C10EF@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r471:064a045efe5b
Date: 2013-06-20 13:26 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/064a045efe5b/

Log:	fixed weird translation bug where keyword arguments are required by
	translator, instead of relying on sequence

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -937,7 +937,7 @@
 
     def __del__(self):
         if self.words is None:
-            lltype.free(self.c_words, 'raw')
+            lltype.free(self.c_words, flavor='raw')
 
 
 NATIVE_DEPTH = 32

From noreply at buildbot.pypy.org  Thu Jun 20 13:52:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 13:52:03 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Unsure here
Message-ID: <20130620115203.11A6F1C034A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r203:1b81e35c24e6
Date: 2013-06-20 13:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/1b81e35c24e6/

Log:	Unsure here

diff --git a/c4/doc-stmgc.txt b/c4/doc-stmgc.txt
--- a/c4/doc-stmgc.txt
+++ b/c4/doc-stmgc.txt
@@ -291,6 +291,9 @@
 is simplified to a regular complete garbage collection.  Additionally,
 as hinted above, we can compact chains of public object copies.
 
+[why? write a justification for the next paragraph or drop it.  I'm not
+sure but it may be the case that we no longer rely on this.]
+
 Note that our design relies on the following property: in a given copy
 of an object which was committed at revision N, all pointers point to
 copies of objects which were committed at or before revision N.  This

From noreply at buildbot.pypy.org  Thu Jun 20 15:59:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 15:59:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: gcpage: private_from_protected
Message-ID: <20130620135938.822361C034A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r209:1156c468afdd
Date: 2013-06-20 15:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/1156c468afdd/

Log:	gcpage: private_from_protected

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -199,12 +199,12 @@
         return;    /* already seen */
 
     if (obj->h_revision & 1) {
+        assert(!(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
         obj->h_tid &= ~GCFLAG_PUBLIC_TO_PRIVATE;  /* see also fix_outdated() */
     }
-    else {
+    else if (obj->h_tid & GCFLAG_PUBLIC) {
         /* h_revision is a ptr: we have a more recent version */
         gcptr prev_obj = obj;
-        assert(obj->h_tid & GCFLAG_PUBLIC);
 
         if (!(obj->h_revision & 2)) {
             /* go visit the more recent version */
@@ -236,6 +236,19 @@
         *pobj = obj;
         goto restart;
     }
+    else {
+        assert(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        gcptr B = (gcptr)obj->h_revision;
+        if (!(B->h_tid & GCFLAG_PUBLIC)) {
+            /* a regular private_from_protected object with a backup copy B */
+            assert(B->h_tid & GCFLAG_BACKUP_COPY);
+            B->h_tid |= GCFLAG_VISITED;
+        }
+        else {
+            assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
+            abort();  // XXX
+        }
+    }
     obj->h_tid |= GCFLAG_VISITED;
     gcptrlist_insert(&objects_to_trace, obj);
 }
@@ -335,7 +348,8 @@
            just removing it is very wrong --- we want 'd' to abort.
         */
         revision_t v = obj->h_revision;
-        if (IS_POINTER(v)) {    /* has a more recent revision.  Oups. */
+        if (IS_POINTER(v) && !(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)) {
+            /* has a more recent revision.  Oups. */
             fprintf(stderr,
                     "ABRT_COLLECT_MAJOR: %p was read but modified already\n",
                     obj);
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -295,3 +295,21 @@
     p1 = lib.stm_pop_root()
     assert not lib.in_nursery(p1)
     check_not_free(p1)
+
+def test_private_from_protected_young():
+    p1 = nalloc(HDR)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b == p1
+    check_not_free(follow_revision(p1))
+    assert follow_revision(p1).h_tid & GCFLAG_BACKUP_COPY
+    lib.stm_push_root(p1)
+    major_collect()
+    p1 = lib.stm_pop_root()
+    assert not lib.in_nursery(p1)
+    check_not_free(p1)
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b == p1
+    check_not_free(follow_revision(p1))
+    assert follow_revision(p1).h_tid & GCFLAG_BACKUP_COPY

From noreply at buildbot.pypy.org  Thu Jun 20 16:11:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 16:11:30 +0200 (CEST)
Subject: [pypy-commit] stmgc default: In inevitable transactions,
 clear the list_of_read_objects when
Message-ID: <20130620141130.E917D1C034A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r210:8065a37d6daa
Date: 2013-06-20 16:11 +0200
http://bitbucket.org/pypy/stmgc/changeset/8065a37d6daa/

Log:	In inevitable transactions, clear the list_of_read_objects when
	collecting, rather than fixing it.

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -339,6 +339,11 @@
     if (d->active < 0)
         return;       /* already "aborted" during forced minor collection */
 
+    if (d->active == 2) {
+        /* inevitable transaction: clear the list of read objects */
+        gcptrlist_clear(&d->list_of_read_objects);
+    }
+
     for (i = d->list_of_read_objects.size - 1; i >= 0; --i) {
         gcptr obj = items[i];
 
@@ -357,16 +362,13 @@
             return;
         }
 
-        /* on the other hand, if we see a non-visited object in the read
-           list, then we need to remove it --- it's wrong to just abort.
-           Consider the following case: the transaction is inevitable,
-           and since it started, it popped objects out of its shadow
-           stack.  Some popped objects might become free even if they
-           have been read from.  We must not abort such transactions
-           (and cannot anyway: they are inevitable!). */
-        if (!(obj->h_tid & GCFLAG_VISITED)) {
-            items[i] = items[--d->list_of_read_objects.size];
-        }
+        /* It should not be possible to see a non-visited object in the
+           read list.  I think the only case is: the transaction is
+           inevitable, and since it started, it popped objects out of
+           its shadow stack.  Some popped objects might become free even
+           if they have been read from.  But for inevitable transactions,
+           we clear the 'list_of_read_objects' above anyway. */
+        assert(obj->h_tid & GCFLAG_VISITED);
     }
 
     d->num_read_objects_known_old = d->list_of_read_objects.size;
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -281,6 +281,11 @@
     gcptr *items = d->list_of_read_objects.items;
     assert(d->list_of_read_objects.size >= limit);
 
+    if (d->active == 2) {
+        /* inevitable transaction: clear the list of read objects */
+        gcptrlist_clear(&d->list_of_read_objects);
+    }
+
     for (i = d->list_of_read_objects.size - 1; i >= limit; --i) {
         gcptr obj = items[i];
 

From noreply at buildbot.pypy.org  Thu Jun 20 16:16:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 16:16:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Run the spinloop up to 1'000'000 times,
 it seems that 1'000 times is not enough...
Message-ID: <20130620141602.5C3411C034A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r211:754c9140a2c0
Date: 2013-06-20 16:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/754c9140a2c0/

Log:	Run the spinloop up to 1'000'000 times, it seems that 1'000 times is
	not enough...

diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -105,7 +105,7 @@
                  stm_read_barrier((gcptr)r_next))) {
             asm volatile ("pause":::"memory");  /* smp_spinloop() */
             i++;
-            assert(i < 1000);
+            assert(i < 1000000);
         }
         // for now:
         assert(((nodeptr)stm_read_barrier((gcptr)r_prev->next))->value 

From noreply at buildbot.pypy.org  Thu Jun 20 17:22:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Jun 2013 17:22:20 +0200 (CEST)
Subject: [pypy-commit] stmgc default: The next test is complete I think,
 but leads to unexpected aborts
Message-ID: <20130620152220.232B71C034A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r212:2e72ea3bca12
Date: 2013-06-20 17:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/2e72ea3bca12/

Log:	The next test is complete I think, but leads to unexpected aborts

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -582,7 +582,7 @@
     countdown_next_major_coll = next;
 }
 
-void stm_major_collect(void)
+static void major_collect(void)
 {
     stmgcpage_acquire_global_lock();
     fprintf(stderr, ",-----\n| running major collection...\n");
@@ -628,7 +628,7 @@
        threads will also acquire the RW lock in exclusive mode, but won't
        do anything. */
     if (countdown_next_major_coll == 0)
-        stm_major_collect();
+        major_collect();
 
     stm_stop_single_thread();
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -385,6 +385,14 @@
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
         assert(gcptrlist_size(&d->public_with_young_copy) == 0);
+        assert(gcptrlist_size(&d->list_of_read_objects) >=
+               d->num_read_objects_known_old);
+        assert(gcptrlist_size(&d->private_from_protected) >=
+               d->num_private_from_protected_known_old);
+        d->num_read_objects_known_old =
+            gcptrlist_size(&d->list_of_read_objects);
+        d->num_private_from_protected_known_old =
+            gcptrlist_size(&d->private_from_protected);
         return 0;
     }
     else {
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -407,8 +407,10 @@
         assert classify(follow_revision(p)) == "stub"
         assert p1.h_revision & 1
         r.set(2)
+        r.wait(3)     # wait until the other thread really started
     def f2(r):
         r.wait(2)
+        r.set(3)
         p2 = lib.stm_read_barrier(p)    # steals
         assert classify(p2) == "public"
         q2 = lib.getptr(p2, 0)
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -313,3 +313,38 @@
     assert p1b == p1
     check_not_free(follow_revision(p1))
     assert follow_revision(p1).h_tid & GCFLAG_BACKUP_COPY
+
+def test_backup_stolen():
+    py.test.skip("unexpected abort")
+    p = palloc(HDR)
+    def f1(r):
+        p1 = lib.stm_write_barrier(p)   # private copy
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        assert classify(p) == "public"
+        assert classify(p1) == "protected"
+        assert classify(follow_revision(p)) == "stub"
+        assert p1.h_revision & 1
+        def cb(c):
+            assert c == 0
+            p1b = lib.stm_write_barrier(p1)
+            assert p1b == p1
+            assert classify(p1) == "private_from_protected"
+            assert classify(follow_revision(p1)) == "backup"
+            r.wait_while_in_parallel()
+            check_not_free(p1)
+            assert classify(p1) == "private_from_protected"
+            assert classify(follow_revision(p1)) == "public"  # has been stolen
+        perform_transaction(cb)
+    def f2(r):
+        def cb(c):
+            assert c == 0
+            r.enter_in_parallel()
+            p2 = lib.stm_read_barrier(p)    # steals
+            assert classify(p2) == "public"
+            r.leave_in_parallel()
+            major_collect()
+            check_not_free(p2)
+            assert classify(p2) == "public"
+        perform_transaction(cb)
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Thu Jun 20 20:18:04 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 20 Jun 2013 20:18:04 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.4: merge default
Message-ID: <20130620181804.BE8921C034A@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: stdlib-2.7.4
Changeset: r64949:ffcba4b6a2e8
Date: 2013-06-20 11:17 -0700
http://bitbucket.org/pypy/pypy/changeset/ffcba4b6a2e8/

Log:	merge default

diff too long, truncating to 2000 out of 49489 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/Makefile b/Makefile
new file mode 100644
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,7 @@
+
+all: pypy-c
+
+pypy-c:
+	@echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM"
+	@sleep 3
+	rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -22,3 +22,16 @@
 and send us feedback!
 
     the pypy-dev team <pypy-dev at python.org>
+
+Building
+========
+
+build with::
+
+  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+
+This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
+to use virtualenv with the resulting pypy-c as the interpreter, you can
+find more details about various installation schemes here:
+
+http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/command/install.py b/lib-python/2.7/distutils/command/install.py
--- a/lib-python/2.7/distutils/command/install.py
+++ b/lib-python/2.7/distutils/command/install.py
@@ -474,8 +474,8 @@
 
     def select_scheme (self, name):
         # it's the caller's problem if they supply a bad name!
-        if hasattr(sys, 'pypy_version_info') and not (
-                name.endswith('_user') or name.endswith('_home')):
+        if (hasattr(sys, 'pypy_version_info') and
+            not name.endswith(('_user', '_home'))):
             name = 'pypy'
         scheme = INSTALL_SCHEMES[name]
         for key in SCHEME_KEYS:
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -1,27 +1,16 @@
-"""Provide access to Python's configuration information.  The specific
-configuration variables available depend heavily on the platform and
-configuration.  The values may be retrieved using
-get_config_var(name), and the list of variables is available via
-get_config_vars().keys().  Additional convenience functions are also
-available.
-
-Written by:   Fred L. Drake, Jr.
-Email:        <fdrake at acm.org>
-"""
-
-__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
-
-import sys
-
 
 # The content of this file is redirected from
 # sysconfig_cpython or sysconfig_pypy.
+# All underscore names are imported too, because
+# people like to use undocumented sysconfig._xxx
+# directly.
 
+import sys
 if '__pypy__' in sys.builtin_module_names:
-    from distutils.sysconfig_pypy import *
-    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
-    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+    from distutils import sysconfig_pypy as _sysconfig_module
 else:
-    from distutils.sysconfig_cpython import *
-    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
-    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
+    from distutils import sysconfig_cpython as _sysconfig_module
+globals().update(_sysconfig_module.__dict__)
+
+_USE_CLANG = None
+
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -9,7 +9,7 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
 import os
 import re
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -1,9 +1,17 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -49,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -71,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
@@ -119,13 +122,13 @@
     optional C speedup components.
     """
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-fPIC', '-Wimplicit'])
+        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CFLAGS" in os.environ:
-            cflags = os.environ["CFLAGS"]
-            compiler.compiler.append(cflags)
-            compiler.compiler_so.append(cflags)
-            compiler.linker_so.append(cflags)
+            cflags = os.environ["CFLAGS"].split()
+            compiler.compiler.extend(cflags)
+            compiler.compiler_so.extend(cflags)
+            compiler.linker_so.extend(cflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/2.7/json/tests/test_decode.py b/lib-python/2.7/json/tests/test_decode.py
--- a/lib-python/2.7/json/tests/test_decode.py
+++ b/lib-python/2.7/json/tests/test_decode.py
@@ -49,6 +49,7 @@
         self.assertEqual(self.loads('{"empty": {}}',
                                     object_pairs_hook=OrderedDict),
                          OrderedDict([('empty', OrderedDict())]))
+        self.assertEqual(self.loads("{}", object_pairs_hook=list), [])
 
     def test_extra_data(self):
         s = '[1, 2, 3]5'
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,21 +134,30 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
 }
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
+}
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
@@ -164,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
@@ -174,8 +187,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -183,9 +196,9 @@
     if isinstance(level, (int, long)):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
@@ -277,7 +290,7 @@
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - long(ct)) * 1000
+        self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads and thread:
             self.thread = thread.get_ident()
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -156,7 +156,7 @@
         h = klass(*args)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -187,7 +187,7 @@
     opts = cp.options(sectname)
     if "level" in opts:
         level = cp.get(sectname, "level")
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = cp.get(sectname, "handlers")
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -1954,7 +1954,11 @@
                 if key is None:
                     callback(None, modname, '')
                 else:
-                    desc = split(__import__(modname).__doc__ or '', '\n')[0]
+                    try:
+                        module_doc = __import__(modname).__doc__
+                    except ImportError:
+                        module_doc = None
+                    desc = split(module_doc or '', '\n')[0]
                     if find(lower(modname + ' - ' + desc), key) >= 0:
                         callback(None, modname, desc)
 
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,11 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    self._sock._decref_socketios()
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_codecs.py b/lib-python/2.7/test/test_codecs.py
--- a/lib-python/2.7/test/test_codecs.py
+++ b/lib-python/2.7/test/test_codecs.py
@@ -2,7 +2,11 @@
 import unittest
 import codecs
 import locale
-import sys, StringIO, _testcapi
+import sys, StringIO
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 def coding_checker(self, coder):
     def check(input, expect):
@@ -1496,7 +1500,7 @@
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
-            if encoding not in broken_incremental_coders:
+            if encoding not in broken_incremental_coders and _testcapi:
                 # check incremental decoder/encoder (fetched via the Python
                 # and C API) and iterencode()/iterdecode()
                 try:
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -3609,6 +3609,9 @@
         list.__init__(a, sequence=[0, 1, 2])
         self.assertEqual(a, [0, 1, 2])
 
+    @unittest.skipIf(test_support.check_impl_detail(pypy=True) and
+                     sys.platform == 'win32',
+                     "XXX: https://bugs.pypy.org/issue1461")
     def test_recursive_call(self):
         # Testing recursive __call__() by setting to instance of class...
         class A(object):
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_fileio.py b/lib-python/2.7/test/test_fileio.py
--- a/lib-python/2.7/test/test_fileio.py
+++ b/lib-python/2.7/test/test_fileio.py
@@ -348,7 +348,6 @@
         self.assertRaises(ValueError, _FileIO, -10)
         self.assertRaises(OSError, _FileIO, make_bad_fd())
         if sys.platform == 'win32':
-            raise unittest.SkipTest('Set _invalid_parameter_handler for low level io')
             import msvcrt
             self.assertRaises(IOError, msvcrt.get_osfhandle, make_bad_fd())
         # Issue 15989
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -67,7 +67,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
         finally:
             logging._releaseLock()
 
@@ -99,8 +100,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
@@ -277,6 +280,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -7,7 +7,8 @@
 import subprocess
 from copy import copy, deepcopy
 
-from test.test_support import run_unittest, TESTFN, unlink, get_attribute
+from test.test_support import (run_unittest, TESTFN, unlink, get_attribute,
+                               import_module)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -245,7 +246,10 @@
 
     def test_get_config_h_filename(self):
         config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
+        # import_module skips the test when the CPython C Extension API
+        # appears to not be supported
+        self.assertTrue(os.path.isfile(config_h) or
+                        not import_module('_testcapi'), config_h)
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -1,6 +1,9 @@
 """Test cases for traceback module"""
 
-from _testcapi import traceback_print
+try:
+    from _testcapi import traceback_print
+except ImportError:
+    traceback_print = None
 from StringIO import StringIO
 import sys
 import unittest
@@ -176,6 +179,8 @@
 class TracebackFormatTests(unittest.TestCase):
 
     def test_traceback_format(self):
+        if traceback_print is None:
+            raise unittest.SkipTest('Requires _testcapi')
         try:
             raise KeyError('blah')
         except KeyError:
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -1636,7 +1636,10 @@
         self.assertEqual("{}".format(u), '__unicode__ overridden')
 
     def test_encode_decimal(self):
-        from _testcapi import unicode_encodedecimal
+        try:
+            from _testcapi import unicode_encodedecimal
+        except ImportError:
+            raise unittest.SkipTest('Requires _testcapi')
         self.assertEqual(unicode_encodedecimal(u'123'),
                          b'123')
         self.assertEqual(unicode_encodedecimal(u'\u0663.\u0661\u0664'),
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -131,7 +131,7 @@
     RegrTest('test_bz2.py', usemodules='bz2'),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
-    RegrTest('test_capi.py'),
+    RegrTest('test_capi.py', usemodules='cpyext'),
     RegrTest('test_cd.py'),
     RegrTest('test_cfgparser.py'),
     RegrTest('test_cgi.py'),
@@ -178,7 +178,7 @@
     RegrTest('test_cprofile.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
     RegrTest('test_csv.py', usemodules='_csv'),
-    RegrTest('test_ctypes.py', usemodules="_rawffi thread"),
+    RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
     RegrTest('test_curses.py'),
     RegrTest('test_datetime.py', usemodules='binascii struct'),
     RegrTest('test_dbm.py'),
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
@@ -166,9 +166,7 @@
         if self is StructOrUnion:
             return
         if '_fields_' not in self.__dict__:
-            self._fields_ = []
-            self._names = []
-            _set_shape(self, [], self._is_union)
+            self._fields_ = []  # As a side-effet, this also sets the ffishape.
 
     __setattr__ = struct_setattr
 
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -44,7 +44,7 @@
     import code
     if mainmodule is None:
         import __main__ as mainmodule
-    console = code.InteractiveConsole(mainmodule.__dict__)
+    console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
     # some parts of code.py are copied here because it seems to be impossible
     # to start an interactive console without printing at least one line
     # of banner
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_pypy_testcapi.py
copy from lib_pypy/_testcapi.py
copy to lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,14 +1,20 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
     """
     thisdir = os.path.dirname(__file__)
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -19,13 +25,13 @@
         ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
+    res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
                            extra_preargs=ccflags)
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
+    output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
@@ -37,7 +43,7 @@
             library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
+                        '/EXPORT:init' + modulename]
     else:
         libraries = []
         extra_ldargs = []
@@ -49,9 +55,7 @@
         libraries=libraries,
         extra_preargs=extra_ldargs)
 
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1007,17 +1007,18 @@
 
                 # Actually execute the SQL statement
                 ret = _lib.sqlite3_step(self.__statement._statement)
-                if ret not in (_lib.SQLITE_DONE, _lib.SQLITE_ROW):
-                    self.__statement._reset()
-                    raise self.__connection._get_exception(ret)
 
                 if ret == _lib.SQLITE_ROW:
                     if multiple:
                         raise ProgrammingError("executemany() can only execute DML statements.")
                     self.__build_row_cast_map()
                     self.__next_row = self.__fetch_one_row()
-                elif ret == _lib.SQLITE_DONE and not multiple:
+                elif ret == _lib.SQLITE_DONE:
+                    if not multiple:
+                        self.__statement._reset()
+                else:
                     self.__statement._reset()
+                    raise self.__connection._get_exception(ret)
 
                 if self.__statement._type in ("UPDATE", "DELETE", "INSERT", "REPLACE"):
                     if self.__rowcount == -1:
@@ -1098,7 +1099,6 @@
         try:
             next_row = self.__next_row
         except AttributeError:
-            self.__statement._reset()
             raise StopIteration
         del self.__next_row
 
@@ -1106,11 +1106,12 @@
             next_row = self.row_factory(self, next_row)
 
         ret = _lib.sqlite3_step(self.__statement._statement)
-        if ret not in (_lib.SQLITE_DONE, _lib.SQLITE_ROW):
+        if ret == _lib.SQLITE_ROW:
+            self.__next_row = self.__fetch_one_row()
+        else:
             self.__statement._reset()
-            raise self.__connection._get_exception(ret)
-        elif ret == _lib.SQLITE_ROW:
-            self.__next_row = self.__fetch_one_row()
+            if ret != _lib.SQLITE_DONE:
+                raise self.__connection._get_exception(ret)
         return next_row
 
     if sys.version_info[0] < 3:
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,57 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testcapi'")
+else:
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,48 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.6"
-__version_info__ = (0, 6)
+__version__ = "0.7"
+__version_info__ = (0, 7)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -73,15 +73,15 @@
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        BVoidP = self._get_cached_btype(model.voidp_type)
+        self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(BVoidP, 0)
+                FFI.NULL = self.cast(self.BVoidP, 0)
                 FFI.CData, FFI.CType = backend._get_types()
         else:
             # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(BVoidP, 0)
+            self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False):
@@ -346,6 +346,12 @@
         self._cdefsources.extend(ffi_to_include._cdefsources)
         self._cdefsources.append(']')
 
+    def new_handle(self, x):
+        return self._backend.newp_handle(self.BVoidP, x)
+
+    def from_handle(self, x):
+        return self._backend.from_handle(x)
+
 
 def _make_ffi_library(ffi, libname, flags):
     import os
@@ -355,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
@@ -370,7 +376,10 @@
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
             BType = ffi._get_cached_btype(tp)
-            value = backendlib.load_function(BType, name)
+            try:
+                value = backendlib.load_function(BType, name)
+            except KeyError as e:
+                raise AttributeError('%s: %s' % (name, e))
             library.__dict__[name] = value
             return
         #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -16,6 +16,7 @@
 class CTypesData(object):
     __metaclass__ = CTypesType
     __slots__ = ['__weakref__']
+    __name__ = '<cdata>'
 
     def __init__(self, *args):
         raise TypeError("cannot instantiate %r" % (self.__class__,))
@@ -491,6 +492,8 @@
         elif BItem in (getbtype(model.PrimitiveType('signed char')),
                        getbtype(model.PrimitiveType('unsigned char'))):
             kind = 'bytep'
+        elif BItem is getbtype(model.void_type):
+            kind = 'voidp'
         else:
             kind = 'generic'
         #
@@ -546,13 +549,13 @@
             def __setitem__(self, index, value):
                 self._as_ctype_ptr[index] = BItem._to_ctypes(value)
 
-            if kind == 'charp':
+            if kind == 'charp' or kind == 'voidp':
                 @classmethod
-                def _arg_to_ctypes(cls, value):
-                    if isinstance(value, bytes):
-                        return ctypes.c_char_p(value)
+                def _arg_to_ctypes(cls, *value):
+                    if value and isinstance(value[0], bytes):
+                        return ctypes.c_char_p(value[0])
                     else:
-                        return super(CTypesPtr, cls)._arg_to_ctypes(value)
+                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
 
             if kind == 'charp' or kind == 'bytep':
                 def _to_string(self, maxlen):
@@ -704,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -931,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),

From noreply at buildbot.pypy.org  Fri Jun 21 11:24:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Jun 2013 11:24:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Pass this test
Message-ID: <20130621092425.163181C02D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r214:ae4b3bacf4c0
Date: 2013-06-21 11:24 +0200
http://bitbucket.org/pypy/stmgc/changeset/ae4b3bacf4c0/

Log:	Pass this test

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -484,6 +484,8 @@
                 GCFLAG_WRITE_BARRIER     |
                 0);
   L->h_revision = stm_private_rev_num;
+  assert(stm_private_rev_num < 0);
+  assert(stm_private_rev_num & 1);
   g2l_insert(&d->public_to_private, R, L);
   fprintf(stderr, "write_barrier: adding %p -> %p to public_to_private\n",
           R, L);
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -195,8 +195,11 @@
         return;
 
  restart:
-    if (obj->h_tid & GCFLAG_VISITED)
+    if (obj->h_tid & GCFLAG_VISITED) {
+        fprintf(stderr, "[already visited: %p]\n", obj);
+        assert(obj == *pobj);
         return;    /* already seen */
+    }
 
     if (obj->h_revision & 1) {
         assert(!(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
@@ -246,8 +249,11 @@
             B->h_tid |= GCFLAG_VISITED;
         }
         else {
+            /* a private_from_protected with a stolen backup copy B */
             assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
-            abort();  // XXX
+            gcptr obj1 = B;
+            visit(&obj1);     /* xxx recursion? */
+            obj->h_revision = (revision_t)obj1;
         }
     }
     obj->h_tid |= GCFLAG_VISITED;
@@ -287,8 +293,12 @@
 {
     //assert(*root == END_MARKER);
     //root++;
-    while (root != end)
-        visit(root++);
+    while (root != end) {
+        gcptr o = *root;
+        visit(root);
+        fprintf(stderr, "visit stack root: %p -> %p\n", o, *root);
+        root++;
+    }
 }
 
 static void mark_all_stack_roots(void)
@@ -354,8 +364,13 @@
            case that obj->h_revision doesn't have GCFLAG_VISITED, but
            just removing it is very wrong --- we want 'd' to abort.
         */
+        if (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+            assert(IS_POINTER(obj->h_revision));
+            obj = (gcptr)obj->h_revision;
+        }
+
         revision_t v = obj->h_revision;
-        if (IS_POINTER(v) && !(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)) {
+        if (IS_POINTER(v)) {
             /* has a more recent revision.  Oups. */
             fprintf(stderr,
                     "ABRT_COLLECT_MAJOR: %p was read but modified already\n",
@@ -539,7 +554,8 @@
            collection was not preceeded by a minor collection if the
            thread is busy in a system call for example.
         */
-        if (stmgc_minor_collect_anything_to_do(d)) {
+        if (stmgc_minor_collect_anything_to_do(d) ||
+            (d->public_descriptor->stolen_objects.size != 0)) {
             /* Hack: temporarily pretend that we "are" the other thread...
              */
             thread_descriptor = d;
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -235,12 +235,13 @@
         if (L == NULL)
             continue;
         assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        assert(IS_POINTER(L->h_revision));
 
         g2l_insert(&d->public_to_private, B, L);
 
         /* this is definitely needed: all keys in public_to_private
            must appear in list_of_read_objects */
-        fprintf(stderr, "n.readobj: %p\n", B);
+        fprintf(stderr, "n.readobj: %p -> %p\n", B, L);
         assert(!(B->h_tid & GCFLAG_STUB));
         gcptrlist_insert(&d->list_of_read_objects, B);
 
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -487,7 +487,9 @@
     return p.h_tid & GCFLAG_STUB
 
 def check_not_free(p):
+    print >> sys.stderr, "[checking %r..." % p,
     assert 42 < (p.h_tid & 0xFFFF) < 521
+    print >> sys.stderr, "ok]"
 
 def check_nursery_free(p):
     #assert p.h_tid == p.h_revision == 0
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -315,10 +315,12 @@
     assert follow_revision(p1).h_tid & GCFLAG_BACKUP_COPY
 
 def test_backup_stolen():
-    py.test.skip("in-progress")
     p = palloc(HDR)
     def f1(r):
         p1 = lib.stm_write_barrier(p)   # private copy
+        lib.stm_push_root(p1)
+        minor_collect()
+        p1 = lib.stm_pop_root()
         lib.stm_commit_transaction()
         lib.stm_begin_inevitable_transaction()
         assert classify(p) == "public"
@@ -329,9 +331,13 @@
             assert c == 0
             p1b = lib.stm_write_barrier(p1)
             assert p1b == p1
+            assert not lib.in_nursery(p1)
             assert classify(p1) == "private_from_protected"
             assert classify(follow_revision(p1)) == "backup"
+            lib.stm_push_root(p1)
             r.wait_while_in_parallel()
+            p1b = lib.stm_pop_root()
+            assert p1b == p1
             check_not_free(p1)
             assert classify(p1) == "private_from_protected"
             assert classify(follow_revision(p1)) == "public"  # has been stolen

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:49 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:49 +0200 (CEST)
Subject: [pypy-commit] stmgc default: minor changes
Message-ID: <20130621114649.4BECE1C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r215:a295fa0455e1
Date: 2013-06-20 14:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/a295fa0455e1/

Log:	minor changes

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -183,12 +183,12 @@
     return result1;
 }
 
-static const revision_t C_PRIVATE_FROM_PROTECTED = 1;
-static const revision_t C_PRIVATE                = 2;
-static const revision_t C_STUB                   = 3;
-static const revision_t C_PUBLIC                 = 4;
-static const revision_t C_BACKUP                 = 5;
-static const revision_t C_PROTECTED              = 6;
+static const int C_PRIVATE_FROM_PROTECTED = 1;
+static const int C_PRIVATE                = 2;
+static const int C_STUB                   = 3;
+static const int C_PUBLIC                 = 4;
+static const int C_BACKUP                 = 5;
+static const int C_PROTECTED              = 6;
 int classify(gcptr p)
 {
     int priv_from_prot = (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) != 0;
@@ -260,7 +260,7 @@
     num = get_rand(SHARED_ROOTS);
     _sr = shared_roots[num];
 
-    k = get_rand(14);
+    k = get_rand(15);
 
     switch (k) {
     case 0: // remove a root
@@ -316,7 +316,11 @@
         w_sr = (nodeptr)write_barrier(_sr);
         w_sr->next = (nodeptr)shared_roots[get_rand(SHARED_ROOTS)];
         break;
-    default:
+    case 14:
+        push_roots();
+        stmgc_minor_collect();
+        pop_roots();
+        p = NULL;
         break;
     }
     return p;
@@ -344,17 +348,21 @@
 int interruptible_callback(gcptr arg1, int retry_counter)
 {
     td.num_roots = td.num_roots_outside_perform;
-    copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
+    // done by the following pop_roots():
+    //copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
 
+    // refresh td.roots:
     arg1 = stm_pop_root();
+    assert(arg1 == NULL);
     pop_roots();
     push_roots();
     stm_push_root(arg1);
 
     int p = run_me();
-    int restart = p == -1 ? get_rand(3) != 1 : 0;
+    if (p == -1) // maybe restart transaction
+        return get_rand(3) != 1;
 
-    return restart;
+    return 0;
 }
 
 int run_me()

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:50 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:50 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: work in progress
Message-ID: <20130621114650.6A8931C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r216:04f0776e142a
Date: 2013-06-20 16:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/04f0776e142a/

Log:	work in progress

diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -57,6 +57,9 @@
  * but converted from a protected.  These are precisely the objects
  * that have a backup copy (in h_revision), which gives a copy of the
  * original protected object.
+ * 
+ * GCFLAG_HAS_ID is set on young objects that have an old reserved
+ * memory to be copied to in minor collections (obj->h_original)
  */
 static const revision_t GCFLAG_OLD                    = STM_FIRST_GCFLAG << 0;
 static const revision_t GCFLAG_VISITED                = STM_FIRST_GCFLAG << 1;
@@ -68,6 +71,7 @@
 static const revision_t GCFLAG_BACKUP_COPY  /*debug*/ = STM_FIRST_GCFLAG << 7;
 static const revision_t GCFLAG_STUB         /*debug*/ = STM_FIRST_GCFLAG << 8;
 static const revision_t GCFLAG_PRIVATE_FROM_PROTECTED = STM_FIRST_GCFLAG << 9;
+static const revision_t GCFLAG_HAS_ID                 = STM_FIRST_GCFLAG << 10;
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -59,6 +59,7 @@
     assert(tid == (tid & STM_USER_TID_MASK));
     P->h_tid = tid;
     P->h_revision = stm_private_rev_num;
+    P->h_original = NULL;
     return P;
 }
 
@@ -71,6 +72,13 @@
 
     memcpy(L, P, size);
     L->h_tid &= ~GCFLAG_OLD;
+    L->h_tid &= ~GCFLAG_HAS_ID;
+
+    if (P->h_original)
+        L->h_original = P->h_original;
+    else
+        L->h_original = P;
+
     return L;
 }
 
@@ -80,11 +88,64 @@
     gcptr L = (gcptr)stmgcpage_malloc(size);
     memcpy(L, P, size);
     L->h_tid |= GCFLAG_OLD;
+    L->h_tid &= ~GCFLAG_HAS_ID;
+
+    if (P->h_original)
+        L->h_original = P->h_original;
+    else
+        L->h_original = P;
+
     return L;
 }
 
 /************************************************************/
 
+
+revision_t stm_hash(gcptr p)
+{
+    return stm_id(p);
+}
+
+revision_t stm_id(gcptr p)
+{
+    if (p->h_original) {
+        return p->h_original;
+    }
+    
+    //p->h_original == NULL
+    if (!(p->h_tid & GCFLAG_OLD)) {//(is_in_nursery(p)) {
+        // preallocate old "original" outside
+        // like stealing
+        gcptr O = stmgc_duplicate_old(L);
+        L->h_revision = (revision_t)O;
+        L->h_original = O;
+        L->h_tid |= GCFLAG_HAS_ID;
+        
+        
+        // could be stolen
+        if (p->h_tid & GCFLAG_NURSERY_MOVED) {
+            
+        }
+    }
+    else if (p->h_tid & GCFLAG_NURSERY_MOVED) {
+        if (p->h_tid & GCFLAG_PUBLIC) {
+            // moved by stealing
+            
+        }
+       
+    }
+    else {
+        assert(0);
+    }
+}
+
+revision_t stm_pointer_equal(gcptr p1, gcptr p2)
+{
+    return stm_id(p1) == stm_id(p2);
+}
+
+/************************************************************/
+
 static inline gcptr create_old_object_copy(gcptr obj)
 {
     assert(!(obj->h_tid & GCFLAG_PUBLIC));
@@ -100,6 +161,13 @@
     return fresh_old_copy;
 }
 
+static inline void copy_to_old_id_copy(gcptr obj, gcptr id)
+{
+    size_t size = stmcb_size(obj);
+    memcpy(id, obj, size);
+    id->h_tid &= ~GCFLAG_HAS_ID;
+}
+
 static void visit_if_young(gcptr *root)
 {
     gcptr obj = *root;
@@ -111,17 +179,25 @@
     }
     else {
         /* it's a nursery object.  Was it already moved? */
-
         if (UNLIKELY(obj->h_tid & GCFLAG_NURSERY_MOVED)) {
             /* yes.  Such an object can be a public object in the nursery
                too (such objects are always NURSERY_MOVED).  For all cases,
-               we can just fix the ref. */
+               we can just fix the ref. 
+               Can be stolen objects or those we already moved.
+            */
             *root = (gcptr)obj->h_revision;
             return;
         }
 
-        /* make a copy of it outside */
-        fresh_old_copy = create_old_object_copy(obj);
+        if (obj->h_tid & GCFLAG_HAS_ID) {
+            /* already has a place to go to */
+            fresh_old_copy = copy_to_old_id_copy(obj, obj->h_original);
+        } 
+        else {
+            /* make a copy of it outside */
+            fresh_old_copy = create_old_object_copy(obj);
+        }
+        
         obj->h_tid |= GCFLAG_NURSERY_MOVED;
         obj->h_revision = (revision_t)fresh_old_copy;
 
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -149,10 +149,20 @@
 
         fprintf(stderr, "stolen: %p -> %p\n", P, L);
 
-        /* Copy the object out of the other thread's nursery, if needed */
-        if (!(L->h_tid & GCFLAG_OLD)) {
-            gcptr O = stmgc_duplicate_old(L);
-            L->h_revision = (revision_t)O;
+        if (!(L->h_tid & GCFLAG_OLD)) { 
+            gcptr O;
+            if (L->h_tid & GCFLAG_HAS_ID) {
+                L->h_tid &= ~GCFLAG_HAS_ID;
+                L->h_revision = (revision_t)L->h_original;
+                copy_to_old_id_copy(L, L->h_original);
+                O = L->h_original;
+            } else {
+                /* Copy the object out of the other thread's nursery, 
+                   if needed */
+                O = stmgc_duplicate_old(L);
+                L->h_revision = (revision_t)O;
+                L->h_original = O;
+            }
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
             /* subtle: we need to remove L from the fxcache of the target
                thread, otherwise its read barrier might not trigger on it.
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -10,6 +10,7 @@
 typedef struct stm_object_s {
     revision_t h_tid;
     revision_t h_revision;
+    gcptr h_original;
 } *gcptr;
 
 
@@ -28,6 +29,14 @@
 /* allocate an object out of the local nursery */
 gcptr stm_allocate(size_t size, unsigned long tid);
 
+/* returns a never changing hash for the object */
+revision_t stm_hash(gcptr);
+/* returns an for the object which is unique during its lifetime */
+revision_t stm_id(gcptr);
+/* returns nonzero if the two object-copy pointers belong to the
+same original object */
+revision_t stm_pointer_equal(gcptr, gcptr);
+
 /* to push/pop objects into the local shadowstack */
 /* (could be turned into macros or something later) */
 void stm_push_root(gcptr);

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:51 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:51 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: merge
Message-ID: <20130621114651.76DA91C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r217:74a48c97a3f6
Date: 2013-06-20 16:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/74a48c97a3f6/

Log:	merge

diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -105,7 +105,7 @@
                  stm_read_barrier((gcptr)r_next))) {
             asm volatile ("pause":::"memory");  /* smp_spinloop() */
             i++;
-            assert(i < 1000);
+            assert(i < 1000000);
         }
         // for now:
         assert(((nodeptr)stm_read_barrier((gcptr)r_prev->next))->value 
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -972,7 +972,7 @@
     } G2L_LOOP_END;
 }
 
-//static pthread_mutex_t mutex_prebuilt_gcroots = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t mutex_prebuilt_gcroots = PTHREAD_MUTEX_INITIALIZER;
 
 static void UpdateChainHeads(struct tx_descriptor *d, revision_t cur_time,
                              revision_t localrev)
@@ -1028,16 +1028,13 @@
 #endif
       ACCESS_ONCE(R->h_revision) = v;
 
-#if 0
       if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
         {
           /* cannot possibly get here more than once for a given value of R */
           pthread_mutex_lock(&mutex_prebuilt_gcroots);
           gcptrlist_insert(&stm_prebuilt_gcroots, R);
           pthread_mutex_unlock(&mutex_prebuilt_gcroots);
-          /*mark*/
         }
-#endif
 
     } G2L_LOOP_END;
 
@@ -1433,6 +1430,8 @@
           /* we are reusing 'pd' */
           descriptor_array_free_list = pd->free_list_next;
           assert(descriptor_array_free_list >= 0);
+          assert(pd->stolen_objects.size == 0);
+          assert(pd->stolen_young_stubs.size == 0);
           assert(pd->collection_lock == 0 || pd->collection_lock == -1);
           pd->collection_lock = 0;
       }
@@ -1483,8 +1482,12 @@
     assert(d->active == 0);
     stmgcpage_acquire_global_lock();
 
-    /* our nursery is empty at this point */
+    /* our nursery is empty at this point.  The list 'stolen_objects'
+       should have been emptied at the previous minor collection and
+       should remain empty because we don't have any young object. */
+    assert(d->public_descriptor->stolen_objects.size == 0);
     assert(d->public_descriptor->stolen_young_stubs.size == 0);
+    gcptrlist_delete(&d->public_descriptor->stolen_objects);
     gcptrlist_delete(&d->public_descriptor->stolen_young_stubs);
 
     stmgcpage_done_tls();
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -199,21 +199,56 @@
         return;    /* already seen */
 
     if (obj->h_revision & 1) {
+        assert(!(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
         obj->h_tid &= ~GCFLAG_PUBLIC_TO_PRIVATE;  /* see also fix_outdated() */
     }
-    else {
+    else if (obj->h_tid & GCFLAG_PUBLIC) {
         /* h_revision is a ptr: we have a more recent version */
-        assert(!(obj->h_tid & GCFLAG_STUB));
         gcptr prev_obj = obj;
-        obj = (gcptr)obj->h_revision;   /* go visit the more recent version */
 
-        if (!(obj->h_tid & GCFLAG_VISITED) && IS_POINTER(obj->h_revision)) {
+        if (!(obj->h_revision & 2)) {
+            /* go visit the more recent version */
             obj = (gcptr)obj->h_revision;
+        }
+        else {
+            /* it's a stub: keep it if it points to a protected version,
+               because we need to keep the effect of stealing if it is
+               later accessed by the wrong thread.  If it points to a
+               public object (possibly outdated), we can ignore the stub.
+            */
+            assert(obj->h_tid & GCFLAG_STUB);
+            obj = (gcptr)(obj->h_revision - 2);
+            if (!(obj->h_tid & GCFLAG_PUBLIC)) {
+                prev_obj->h_tid |= GCFLAG_VISITED;
+                assert(*pobj == prev_obj);
+                gcptr obj1 = obj;
+                visit(&obj1);       /* recursion, but should be only once */
+                prev_obj->h_revision = ((revision_t)obj1) + 2;
+                return;
+            }
+        }
+
+        if (!(obj->h_revision & 3)) {
+            obj = (gcptr)obj->h_revision;
+            assert(obj->h_tid & GCFLAG_PUBLIC);
             prev_obj->h_revision = (revision_t)obj;
         }
         *pobj = obj;
         goto restart;
     }
+    else {
+        assert(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        gcptr B = (gcptr)obj->h_revision;
+        if (!(B->h_tid & GCFLAG_PUBLIC)) {
+            /* a regular private_from_protected object with a backup copy B */
+            assert(B->h_tid & GCFLAG_BACKUP_COPY);
+            B->h_tid |= GCFLAG_VISITED;
+        }
+        else {
+            assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
+            abort();  // XXX
+        }
+    }
     obj->h_tid |= GCFLAG_VISITED;
     gcptrlist_insert(&objects_to_trace, obj);
 }
@@ -304,6 +339,11 @@
     if (d->active < 0)
         return;       /* already "aborted" during forced minor collection */
 
+    if (d->active == 2) {
+        /* inevitable transaction: clear the list of read objects */
+        gcptrlist_clear(&d->list_of_read_objects);
+    }
+
     for (i = d->list_of_read_objects.size - 1; i >= 0; --i) {
         gcptr obj = items[i];
 
@@ -313,7 +353,8 @@
            just removing it is very wrong --- we want 'd' to abort.
         */
         revision_t v = obj->h_revision;
-        if (IS_POINTER(v)) {    /* has a more recent revision.  Oups. */
+        if (IS_POINTER(v) && !(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)) {
+            /* has a more recent revision.  Oups. */
             fprintf(stderr,
                     "ABRT_COLLECT_MAJOR: %p was read but modified already\n",
                     obj);
@@ -321,16 +362,13 @@
             return;
         }
 
-        /* on the other hand, if we see a non-visited object in the read
-           list, then we need to remove it --- it's wrong to just abort.
-           Consider the following case: the transaction is inevitable,
-           and since it started, it popped objects out of its shadow
-           stack.  Some popped objects might become free even if they
-           have been read from.  We must not abort such transactions
-           (and cannot anyway: they are inevitable!). */
-        if (!(obj->h_tid & GCFLAG_VISITED)) {
-            items[i] = items[--d->list_of_read_objects.size];
-        }
+        /* It should not be possible to see a non-visited object in the
+           read list.  I think the only case is: the transaction is
+           inevitable, and since it started, it popped objects out of
+           its shadow stack.  Some popped objects might become free even
+           if they have been read from.  But for inevitable transactions,
+           we clear the 'list_of_read_objects' above anyway. */
+        assert(obj->h_tid & GCFLAG_VISITED);
     }
 
     d->num_read_objects_known_old = d->list_of_read_objects.size;
@@ -483,6 +521,36 @@
 }
 
 
+/***** Major collections: forcing minor collections *****/
+
+void force_minor_collections(void)
+{
+    struct tx_descriptor *d;
+    struct tx_descriptor *saved = thread_descriptor;
+    revision_t saved_private_rev = stm_private_rev_num;
+    assert(saved_private_rev == *saved->private_revision_ref);
+
+    for (d = stm_tx_head; d; d = d->tx_next) {
+        /* Force a minor collection to run in the thread 'd'.
+           Usually not needed, but it may be the case that this major
+           collection was not preceeded by a minor collection if the
+           thread is busy in a system call for example.
+        */
+        if (stmgc_minor_collect_anything_to_do(d)) {
+            /* Hack: temporarily pretend that we "are" the other thread...
+             */
+            thread_descriptor = d;
+            stm_private_rev_num = *d->private_revision_ref;
+            //assert(stmgc_nursery_hiding(d, 0));
+            stmgc_minor_collect_no_abort();
+            //assert(stmgc_nursery_hiding(d, 1));
+            thread_descriptor = saved;
+            stm_private_rev_num = saved_private_rev;
+        }
+    }
+}
+
+
 /***** Major collections: main *****/
 
 void update_next_threshold(void)
@@ -519,11 +587,9 @@
     stmgcpage_acquire_global_lock();
     fprintf(stderr, ",-----\n| running major collection...\n");
 
-#if 0
     force_minor_collections();
 
     assert(gcptrlist_size(&objects_to_trace) == 0);
-#endif
     mark_prebuilt_roots();
     mark_all_stack_roots();
     visit_all_objects();
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -357,6 +357,11 @@
     gcptr *items = d->list_of_read_objects.items;
     assert(d->list_of_read_objects.size >= limit);
 
+    if (d->active == 2) {
+        /* inevitable transaction: clear the list of read objects */
+        gcptrlist_clear(&d->list_of_read_objects);
+    }
+
     for (i = d->list_of_read_objects.size - 1; i >= limit; --i) {
         gcptr obj = items[i];
 
@@ -444,6 +449,12 @@
     AbortNowIfDelayed();
 }
 
+void stmgc_minor_collect_no_abort(void)
+{
+    struct tx_descriptor *d = thread_descriptor;
+    minor_collect(d);
+}
+
 int stmgc_minor_collect_anything_to_do(struct tx_descriptor *d)
 {
     if (d->nursery_current == d->nursery_base /*&&
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -40,6 +40,7 @@
 void stmgc_init_nursery(void);
 void stmgc_done_nursery(void);
 void stmgc_minor_collect(void);
+void stmgc_minor_collect_no_abort(void);
 int stmgc_minor_collect_anything_to_do(struct tx_descriptor *);
 gcptr stmgc_duplicate(gcptr);
 gcptr stmgc_duplicate_old(gcptr);
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -265,3 +265,51 @@
     check_prebuilt(p1)
     check_free_old(p2)
     check_not_free(p3)     # XXX replace with p1
+
+def test_prebuilt_version_to_protected():
+    p1 = lib.pseudoprebuilt(HDR, 42 + HDR)
+    p2 = lib.stm_write_barrier(p1)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    minor_collect()
+    p2 = lib.stm_read_barrier(p1)
+    assert p2 != p1
+    minor_collect()
+    major_collect()
+    check_prebuilt(p1)
+    check_not_free(p2)     # XXX replace with p1
+
+def test_private():
+    p1 = nalloc(HDR)
+    lib.stm_push_root(p1)
+    minor_collect()
+    major_collect()
+    p1 = lib.stm_pop_root()
+    assert not lib.in_nursery(p1)
+    check_not_free(p1)
+
+def test_major_collect_first_does_minor_collect():
+    p1 = nalloc(HDR)
+    lib.stm_push_root(p1)
+    major_collect()
+    p1 = lib.stm_pop_root()
+    assert not lib.in_nursery(p1)
+    check_not_free(p1)
+
+def test_private_from_protected_young():
+    p1 = nalloc(HDR)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b == p1
+    check_not_free(follow_revision(p1))
+    assert follow_revision(p1).h_tid & GCFLAG_BACKUP_COPY
+    lib.stm_push_root(p1)
+    major_collect()
+    p1 = lib.stm_pop_root()
+    assert not lib.in_nursery(p1)
+    check_not_free(p1)
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b == p1
+    check_not_free(follow_revision(p1))
+    assert follow_revision(p1).h_tid & GCFLAG_BACKUP_COPY

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:52 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:52 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: doesn't crash
Message-ID: <20130621114652.8D3991C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r218:7ed5c33f287e
Date: 2013-06-20 21:09 +0200
http://bitbucket.org/pypy/stmgc/changeset/7ed5c33f287e/

Log:	doesn't crash

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -59,7 +59,7 @@
     assert(tid == (tid & STM_USER_TID_MASK));
     P->h_tid = tid;
     P->h_revision = stm_private_rev_num;
-    P->h_original = NULL;
+    P->h_original = 0;
     return P;
 }
 
@@ -77,7 +77,7 @@
     if (P->h_original)
         L->h_original = P->h_original;
     else
-        L->h_original = P;
+        L->h_original = (revision_t)P;
 
     return L;
 }
@@ -93,7 +93,7 @@
     if (P->h_original)
         L->h_original = P->h_original;
     else
-        L->h_original = P;
+        L->h_original = (revision_t)P;
 
     return L;
 }
@@ -114,28 +114,21 @@
     
     //p->h_original == NULL
     if (!(p->h_tid & GCFLAG_OLD)) {//(is_in_nursery(p)) {
-        // preallocate old "original" outside
+        struct tx_descriptor *d = thread_descriptor;
+        spinlock_acquire(d->public_descriptor->collection_lock, 'I');
+        // preallocate old "original" outside;
         // like stealing
-        gcptr O = stmgc_duplicate_old(L);
-        L->h_revision = (revision_t)O;
-        L->h_original = O;
-        L->h_tid |= GCFLAG_HAS_ID;
+        gcptr O = stmgc_duplicate_old(p);
+        p->h_revision = (revision_t)O;
+        p->h_original = (revision_t)O;
+        p->h_tid |= GCFLAG_HAS_ID;
         
-        
-        // could be stolen
-        if (p->h_tid & GCFLAG_NURSERY_MOVED) {
-            
-        }
-    }
-    else if (p->h_tid & GCFLAG_NURSERY_MOVED) {
-        if (p->h_tid & GCFLAG_PUBLIC) {
-            // moved by stealing
-            
-        }
-       
+        spinlock_release(d->public_descriptor->collection_lock);
+        return (revision_t)O;
     }
     else {
-        assert(0);
+        // p is the original itself
+        return (revision_t)p;
     }
 }
 
@@ -161,7 +154,7 @@
     return fresh_old_copy;
 }
 
-static inline void copy_to_old_id_copy(gcptr obj, gcptr id)
+inline void copy_to_old_id_copy(gcptr obj, gcptr id)
 {
     size_t size = stmcb_size(obj);
     memcpy(id, obj, size);
@@ -191,7 +184,8 @@
 
         if (obj->h_tid & GCFLAG_HAS_ID) {
             /* already has a place to go to */
-            fresh_old_copy = copy_to_old_id_copy(obj, obj->h_original);
+            copy_to_old_id_copy(obj, (gcptr)obj->h_original);
+            fresh_old_copy = (gcptr)obj->h_original;
         } 
         else {
             /* make a copy of it outside */
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -11,6 +11,8 @@
     struct stm_object_s stubs[STUB_NB_OBJS];
 };
 
+inline void copy_to_old_id_copy(gcptr obj, gcptr id);
+
 gcptr stm_stub_malloc(struct tx_public_descriptor *pd)
 {
     assert(pd->collection_lock != 0);
@@ -152,16 +154,16 @@
         if (!(L->h_tid & GCFLAG_OLD)) { 
             gcptr O;
             if (L->h_tid & GCFLAG_HAS_ID) {
+                O = (gcptr)L->h_original;
                 L->h_tid &= ~GCFLAG_HAS_ID;
-                L->h_revision = (revision_t)L->h_original;
-                copy_to_old_id_copy(L, L->h_original);
-                O = L->h_original;
+                L->h_revision = (revision_t)O;
+                copy_to_old_id_copy(L, (gcptr)L->h_original);
             } else {
                 /* Copy the object out of the other thread's nursery, 
                    if needed */
                 O = stmgc_duplicate_old(L);
                 L->h_revision = (revision_t)O;
-                L->h_original = O;
+                L->h_original = (revision_t)O;
             }
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
             /* subtle: we need to remove L from the fxcache of the target
diff --git a/c4/steal.h b/c4/steal.h
--- a/c4/steal.h
+++ b/c4/steal.h
@@ -2,7 +2,7 @@
 #define _SRCSTM_STEAL_H
 
 
-#define STUB_BLOCK_SIZE   (16 * WORD)    /* power of two */
+#define STUB_BLOCK_SIZE   (32 * WORD)    /* power of two */
 
 #define STUB_THREAD(h)    (*(struct tx_public_descriptor **)           \
                             (((revision_t)(h)) & ~(STUB_BLOCK_SIZE-1)))
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -10,7 +10,7 @@
 typedef struct stm_object_s {
     revision_t h_tid;
     revision_t h_revision;
-    gcptr h_original;
+    revision_t h_original;
 } *gcptr;
 
 

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:53 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:53 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: intermediate backup before rewrite
Message-ID: <20130621114653.A64D11C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r219:589d41f3af88
Date: 2013-06-21 11:50 +0200
http://bitbucket.org/pypy/stmgc/changeset/589d41f3af88/

Log:	intermediate backup before rewrite

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -27,6 +27,7 @@
 struct node {
     struct stm_object_s hdr;
     long value;
+    revision_t id;
     struct node *next;
 };
 typedef struct node * nodeptr;
@@ -82,6 +83,7 @@
     gcptr x = calloc(1, size);
     x->h_tid = PREBUILT_FLAGS | tid;
     x->h_revision = PREBUILT_REVISION;
+    x->h_original = 0;
     return x;
 }
 
@@ -260,7 +262,7 @@
     num = get_rand(SHARED_ROOTS);
     _sr = shared_roots[num];
 
-    k = get_rand(15);
+    k = get_rand(16);
 
     switch (k) {
     case 0: // remove a root
@@ -322,6 +324,17 @@
         pop_roots();
         p = NULL;
         break;
+    case 15:
+        w_r = (nodeptr)read_barrier(_r);
+        if (w_r->id) {
+            assert(w_r->id == stm_id((gcptr)w_r));
+            assert(w_r->id == stm_id((gcptr)_r));
+        } 
+        else {
+            w_r = (nodeptr)write_barrier(_r);
+            w_r->id = stm_id((gcptr)w_r);
+            assert(w_r->id == stm_id((gcptr)_r));
+        }
     }
     return p;
 }
@@ -348,8 +361,8 @@
 int interruptible_callback(gcptr arg1, int retry_counter)
 {
     td.num_roots = td.num_roots_outside_perform;
-    // done by the following pop_roots():
-    //copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
+    // done & overwritten by the following pop_roots():
+    // copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
 
     // refresh td.roots:
     arg1 = stm_pop_root();
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -446,7 +446,9 @@
 
   B = stmgc_duplicate_old(P);
   B->h_tid |= GCFLAG_BACKUP_COPY;
-
+  if (P->h_tid & GCFLAG_OLD)
+    B->h_original = P;
+  
   P->h_tid |= GCFLAG_PRIVATE_FROM_PROTECTED;
   P->h_revision = (revision_t)B;
 
@@ -473,6 +475,9 @@
   /* note that stmgc_duplicate() usually returns a young object, but may
      return an old one if the nursery is full at this moment. */
   gcptr L = stmgc_duplicate(R);
+  if (!(L->h_original))
+    L->h_original = (revision_t)R;
+
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(L->h_tid & GCFLAG_STUB));
   assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
@@ -1003,7 +1008,12 @@
       stub->h_tid = (L->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
                                                    | GCFLAG_STUB
                                                    | GCFLAG_OLD;
+      assert(!(L->h_tid & GCFLAG_HAS_ID));
       stub->h_revision = ((revision_t)L) | 2;
+      if (L->h_original)
+        stub->h_original = L->h_original;
+      else
+        L->h_original = (revision_t)stub;
       item->val = stub;
 
     } G2L_LOOP_END;
@@ -1069,6 +1079,8 @@
 
       if (B->h_tid & GCFLAG_PUBLIC)
         {
+          assert(!(P->h_tid & GCFLAG_HAS_ID));
+
           /* B was stolen */
           while (1)
             {
@@ -1080,6 +1092,13 @@
                 break;
             }
         }
+      else if (P->h_tid & GCFLAG_HAS_ID) {
+        /* The backup is the "id object".  */
+        B->h_tid &= ~GCFLAG_BACKUP_COPY;
+        B->h_tid |= GCFLAG_PUBLIC;
+        P->h_tid &= ~GCFLAG_HAS_ID;
+        B->h_revision = (revision_t)P;
+      }
       else
         {
           stmgcpage_free(B);
@@ -1111,6 +1130,7 @@
         {
           assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
           P->h_tid |= GCFLAG_PUBLIC;
+          P->h_tid &= ~GCFLAG_HAS_ID; // just in case
           if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
           /* P becomes a public outdated object.  It may create an
              exception documented in doc-objects.txt: a public but young
@@ -1119,10 +1139,20 @@
              stealing will follow its h_revision (to B).
           */
         }
+      else if (P->h_tid & GCFLAG_HAS_ID) {
+        /* The backup is the "id object".  P becomes outdated. */
+        P->h_tid |= GCFLAG_PUBLIC;
+        P->h_tid &= ~GCFLAG_HAS_ID;
+        B->h_tid |= GCFLAG_PUBLIC;
+        B->h_tid &= ~GCFLAG_BACKUP_COPY;
+        if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
+        fprintf(stderr, "%p made outdated, %p is current\n", P, B);
+      }
       else
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
+          assert(B->h_original == P);
           size_t size = stmcb_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -74,11 +74,6 @@
     L->h_tid &= ~GCFLAG_OLD;
     L->h_tid &= ~GCFLAG_HAS_ID;
 
-    if (P->h_original)
-        L->h_original = P->h_original;
-    else
-        L->h_original = (revision_t)P;
-
     return L;
 }
 
@@ -88,12 +83,7 @@
     gcptr L = (gcptr)stmgcpage_malloc(size);
     memcpy(L, P, size);
     L->h_tid |= GCFLAG_OLD;
-    L->h_tid &= ~GCFLAG_HAS_ID;
-
-    if (P->h_original)
-        L->h_original = P->h_original;
-    else
-        L->h_original = (revision_t)P;
+    assert(!(L->h_tid & GCFLAG_HAS_ID));
 
     return L;
 }
@@ -108,28 +98,92 @@
 
 revision_t stm_id(gcptr p)
 {
+    struct tx_descriptor *d = thread_descriptor;
+    revision_t result;
+
     if (p->h_original) {
+        fprintf(stderr, "stm_id(%p) has original: %p\n", p, (gcptr)p->h_original);
+        return p->h_original;
+    }
+
+    spinlock_acquire(d->public_descriptor->collection_lock, 'I');
+    if (p->h_tid & GCFLAG_OLD) {
+        /* it must be this exact object */
+        result = (revision_t)p;
+    }
+    else {
+        /* must create shadow original object */
+        gcptr O = stmgc_duplicate_old(p);
+        p->h_original = (revision_t)O;
+        p->h_tid |= GCFLAG_HAS_ID;
+        O->h_tid |= GCFLAG_PUBLIC;
+        
+        result = (revision_t)O;
+        
+        fprintf(stderr, "stm_id(%p): is young, preallocate old id-copy %p\n",
+                p, O);
+    }
+
+
+    if (p->h_original) {
+        // maybe in the meantime?
+        fprintf(stderr, "stm_id(%p) has original NOW: %p\n", p, (gcptr)p->h_original);
+        spinlock_release(d->public_descriptor->collection_lock);
         return p->h_original;
     }
     
     //p->h_original == NULL
-    if (!(p->h_tid & GCFLAG_OLD)) {//(is_in_nursery(p)) {
-        struct tx_descriptor *d = thread_descriptor;
-        spinlock_acquire(d->public_descriptor->collection_lock, 'I');
+
+    if (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+        gcptr B = (gcptr)p->h_revision;
+        if (p->h_tid & GCFLAG_OLD) {
+            /* may have become old after becoming priv_from_prot */
+            if (B->h_tid & GCFLAG_BACKUP_COPY) {
+                B->h_original = p;
+                result = (revision_t)p;
+                fprintf(stderr, "stm_id(%p): is priv_from_prot and old. make ");
+            }
+            else {
+                /* someone stole and assumes the backup as the ID */
+                p->h_original = B;
+                result = (revision_t)B;
+            }
+        }
+        else {
+            // use backup copy as ID object
+            p->h_tid |= GCFLAG_HAS_ID; // see AbortPrivateFromProtected
+            p->h_original = (revision_t)B;
+            // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
+            
+            result = (revision_t)B;
+            fprintf(stderr,
+                    "stm_id(%p): is private_from_protected, make the backup %p an id-copy\n",
+                    p, B);
+        }
+    } 
+    else if (!(p->h_tid & GCFLAG_OLD)) {//(is_in_nursery(p)) {
         // preallocate old "original" outside;
         // like stealing
         gcptr O = stmgc_duplicate_old(p);
-        p->h_revision = (revision_t)O;
+        //p->h_revision = (revision_t)O;
         p->h_original = (revision_t)O;
         p->h_tid |= GCFLAG_HAS_ID;
+        O->h_tid |= GCFLAG_PUBLIC;
         
-        spinlock_release(d->public_descriptor->collection_lock);
-        return (revision_t)O;
+        result = (revision_t)O;
+        
+        fprintf(stderr, "stm_id(%p): is young, preallocate old id-copy %p\n",
+                p, O);
     }
-    else {
-        // p is the original itself
-        return (revision_t)p;
+    else {// if (p->h_tid & GCFLAG_OLD) {
+        /* obj is old, possibly private or protected */
+        /* last, because there are old private(_from_public) objects */
+        result = (revision_t)p;
+        fprintf(stderr, "stm_id(%p): is itself\n", p);
     }
+
+    spinlock_release(d->public_descriptor->collection_lock);
+    return result;
 }
 
 revision_t stm_pointer_equal(gcptr p1, gcptr p2)
@@ -156,9 +210,14 @@
 
 inline void copy_to_old_id_copy(gcptr obj, gcptr id)
 {
+    assert(!is_in_nursery(thread_descriptor, id));
+    assert(id->h_tid & GCFLAG_OLD);
+
     size_t size = stmcb_size(obj);
     memcpy(id, obj, size);
     id->h_tid &= ~GCFLAG_HAS_ID;
+    id->h_tid |= GCFLAG_OLD;
+    fprintf(stderr, "copy_to_old_id_copy(%p -> %p)\n", obj, id);
 }
 
 static void visit_if_young(gcptr *root)
@@ -184,8 +243,16 @@
 
         if (obj->h_tid & GCFLAG_HAS_ID) {
             /* already has a place to go to */
-            copy_to_old_id_copy(obj, (gcptr)obj->h_original);
-            fresh_old_copy = (gcptr)obj->h_original;
+            gcptr id_obj = (gcptr)obj->h_original;
+
+            /* assert(!(id_obj->h_tid & GCFLAG_BACKUP_COPY));
+               well, if it is still a backup, then it wasn't
+               stolen. We can use it for our young obj.
+             */
+
+            copy_to_old_id_copy(obj, id_obj);
+            fresh_old_copy = id_obj;
+            obj->h_tid &= ~GCFLAG_HAS_ID;
         } 
         else {
             /* make a copy of it outside */
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -81,6 +81,10 @@
                                                    | GCFLAG_STUB
                                                    | GCFLAG_OLD;
     stub->h_revision = ((revision_t)obj) | 2;
+    if (obj->h_original)
+        stub->h_original = obj->h_original;
+    else
+        obj->h_original = (revision_t)stub;
     g2l_insert(&sd->all_stubs, obj, stub);
 
     if (!(obj->h_tid & GCFLAG_OLD))
@@ -108,6 +112,20 @@
     */
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
+        
+        if(L->h_tid & GCFLAG_HAS_ID) {
+            /* if L has ID, then the backup is the "original" */
+            assert(L->h_original == (revision_t)B);
+            L->h_tid &= ~GCFLAG_HAS_ID; // now it's simply stolen
+        } 
+        else if (L->h_tid & GCFLAG_OLD) {
+            /* became old after becoming priv_from_protected 
+             make L the original
+             */
+            B->h_original = (revision_t)L;
+        }
+        /* otherwise: L is the original */
+        assert (!(B->h_tid & GCFLAG_HAS_ID));
 
         /* B is now a backup copy, i.e. a protected object, and we own
            the foreign thread's collection_lock, so we can read/write the
@@ -154,6 +172,7 @@
         if (!(L->h_tid & GCFLAG_OLD)) { 
             gcptr O;
             if (L->h_tid & GCFLAG_HAS_ID) {
+                /* use id-copy for us */
                 O = (gcptr)L->h_original;
                 L->h_tid &= ~GCFLAG_HAS_ID;
                 L->h_revision = (revision_t)O;

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:56 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:56 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: amazingly seems to not crash..
Message-ID: <20130621114656.D0F801C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r220:c0ff042ed265
Date: 2013-06-21 13:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/c0ff042ed265/

Log:	amazingly seems to not crash..

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -446,8 +446,9 @@
 
   B = stmgc_duplicate_old(P);
   B->h_tid |= GCFLAG_BACKUP_COPY;
-  if (P->h_tid & GCFLAG_OLD)
-    B->h_original = P;
+  if (!(P->h_original) && (P->h_tid & GCFLAG_OLD)) {
+      B->h_original = (revision_t)P;
+  }
   
   P->h_tid |= GCFLAG_PRIVATE_FROM_PROTECTED;
   P->h_revision = (revision_t)B;
@@ -475,8 +476,10 @@
   /* note that stmgc_duplicate() usually returns a young object, but may
      return an old one if the nursery is full at this moment. */
   gcptr L = stmgc_duplicate(R);
-  if (!(L->h_original))
+  if (!(L->h_original)) {
+    assert(R->h_tid & GCFLAG_OLD); // if not, force stm_id??
     L->h_original = (revision_t)R;
+  }
 
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(L->h_tid & GCFLAG_STUB));
@@ -1010,10 +1013,16 @@
                                                    | GCFLAG_OLD;
       assert(!(L->h_tid & GCFLAG_HAS_ID));
       stub->h_revision = ((revision_t)L) | 2;
-      if (L->h_original)
+      if (L->h_original) {
         stub->h_original = L->h_original;
-      else
+      }
+      else if (L->h_tid & GCFLAG_OLD) {
+        stub->h_original = (revision_t)L;
+      }
+      else {
         L->h_original = (revision_t)stub;
+      }
+      
       item->val = stub;
 
     } G2L_LOOP_END;
@@ -1091,12 +1100,13 @@
               if (bool_cas(&B->h_revision, v, (revision_t)P))
                 break;
             }
-        }
-      else if (P->h_tid & GCFLAG_HAS_ID) {
-        /* The backup is the "id object".  */
+        }      
+      else if (P->h_original == (revision_t)B) {
+        /* The backup is the "id object" */
+        assert(!(P->h_tid & GCFLAG_HAS_ID));
+
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
         B->h_tid |= GCFLAG_PUBLIC;
-        P->h_tid &= ~GCFLAG_HAS_ID;
         B->h_revision = (revision_t)P;
       }
       else
@@ -1139,10 +1149,10 @@
              stealing will follow its h_revision (to B).
           */
         }
-      else if (P->h_tid & GCFLAG_HAS_ID) {
+      else if (P->h_original == (revision_t)B) {
         /* The backup is the "id object".  P becomes outdated. */
+        assert(!(P->h_tid & GCFLAG_HAS_ID));
         P->h_tid |= GCFLAG_PUBLIC;
-        P->h_tid &= ~GCFLAG_HAS_ID;
         B->h_tid |= GCFLAG_PUBLIC;
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
         if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
@@ -1152,7 +1162,8 @@
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
-          assert(B->h_original == P);
+          assert(!(P->h_original) 
+                 || (B->h_original == (revision_t)P->h_original));
           size_t size = stmcb_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -101,93 +101,51 @@
     struct tx_descriptor *d = thread_descriptor;
     revision_t result;
 
+    spinlock_acquire(d->public_descriptor->collection_lock, 'I');
     if (p->h_original) {
-        fprintf(stderr, "stm_id(%p) has original: %p\n", p, (gcptr)p->h_original);
+        spinlock_release(d->public_descriptor->collection_lock);
         return p->h_original;
     }
 
-    spinlock_acquire(d->public_descriptor->collection_lock, 'I');
+    /* old objects must have an h_original OR be
+       the original itself. 
+       if some thread stole p when it was still young,
+       it must have set h_original. stealing an old obj
+       makes the old obj "original".
+    */
     if (p->h_tid & GCFLAG_OLD) {
         /* it must be this exact object */
         result = (revision_t)p;
     }
     else {
-        /* must create shadow original object */
-        gcptr O = stmgc_duplicate_old(p);
-        p->h_original = (revision_t)O;
-        p->h_tid |= GCFLAG_HAS_ID;
-        O->h_tid |= GCFLAG_PUBLIC;
-        
-        result = (revision_t)O;
-        
-        fprintf(stderr, "stm_id(%p): is young, preallocate old id-copy %p\n",
-                p, O);
-    }
-
-
-    if (p->h_original) {
-        // maybe in the meantime?
-        fprintf(stderr, "stm_id(%p) has original NOW: %p\n", p, (gcptr)p->h_original);
-        spinlock_release(d->public_descriptor->collection_lock);
-        return p->h_original;
-    }
-    
-    //p->h_original == NULL
-
-    if (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
-        gcptr B = (gcptr)p->h_revision;
-        if (p->h_tid & GCFLAG_OLD) {
-            /* may have become old after becoming priv_from_prot */
-            if (B->h_tid & GCFLAG_BACKUP_COPY) {
-                B->h_original = p;
-                result = (revision_t)p;
-                fprintf(stderr, "stm_id(%p): is priv_from_prot and old. make ");
-            }
-            else {
-                /* someone stole and assumes the backup as the ID */
-                p->h_original = B;
-                result = (revision_t)B;
-            }
-        }
-        else {
-            // use backup copy as ID object
-            p->h_tid |= GCFLAG_HAS_ID; // see AbortPrivateFromProtected
+        /* must create shadow original object or use
+           backup, if exists */
+        if (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+            gcptr B = (gcptr)p->h_revision;
+            /* don't set, otherwise nursery will copy over backup */
+            //p->h_tid |= GCFLAG_HAS_ID; // see AbortPrivateFromProtected
             p->h_original = (revision_t)B;
             // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
             
             result = (revision_t)B;
-            fprintf(stderr,
-                    "stm_id(%p): is private_from_protected, make the backup %p an id-copy\n",
-                    p, B);
         }
-    } 
-    else if (!(p->h_tid & GCFLAG_OLD)) {//(is_in_nursery(p)) {
-        // preallocate old "original" outside;
-        // like stealing
-        gcptr O = stmgc_duplicate_old(p);
-        //p->h_revision = (revision_t)O;
-        p->h_original = (revision_t)O;
-        p->h_tid |= GCFLAG_HAS_ID;
-        O->h_tid |= GCFLAG_PUBLIC;
-        
-        result = (revision_t)O;
-        
-        fprintf(stderr, "stm_id(%p): is young, preallocate old id-copy %p\n",
-                p, O);
+        else {
+            gcptr O = stmgc_duplicate_old(p);
+            p->h_original = (revision_t)O;
+            p->h_tid |= GCFLAG_HAS_ID;
+            O->h_tid |= GCFLAG_PUBLIC;
+            
+            result = (revision_t)O;
+        }
     }
-    else {// if (p->h_tid & GCFLAG_OLD) {
-        /* obj is old, possibly private or protected */
-        /* last, because there are old private(_from_public) objects */
-        result = (revision_t)p;
-        fprintf(stderr, "stm_id(%p): is itself\n", p);
-    }
-
+    
     spinlock_release(d->public_descriptor->collection_lock);
     return result;
 }
 
 revision_t stm_pointer_equal(gcptr p1, gcptr p2)
 {
+    /* XXX: */
     return stm_id(p1) == stm_id(p2);
 }
 
@@ -245,11 +203,6 @@
             /* already has a place to go to */
             gcptr id_obj = (gcptr)obj->h_original;
 
-            /* assert(!(id_obj->h_tid & GCFLAG_BACKUP_COPY));
-               well, if it is still a backup, then it wasn't
-               stolen. We can use it for our young obj.
-             */
-
             copy_to_old_id_copy(obj, id_obj);
             fresh_old_copy = id_obj;
             obj->h_tid &= ~GCFLAG_HAS_ID;
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -81,10 +81,16 @@
                                                    | GCFLAG_STUB
                                                    | GCFLAG_OLD;
     stub->h_revision = ((revision_t)obj) | 2;
-    if (obj->h_original)
+    if (obj->h_original) {
         stub->h_original = obj->h_original;
-    else
+    }
+    else if (obj->h_tid & GCFLAG_OLD) {
+        stub->h_original = (revision_t)obj;
+    }
+    else {
         obj->h_original = (revision_t)stub;
+    }
+
     g2l_insert(&sd->all_stubs, obj, stub);
 
     if (!(obj->h_tid & GCFLAG_OLD))
@@ -113,19 +119,19 @@
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
         
-        if(L->h_tid & GCFLAG_HAS_ID) {
-            /* if L has ID, then the backup is the "original" */
-            assert(L->h_original == (revision_t)B);
-            L->h_tid &= ~GCFLAG_HAS_ID; // now it's simply stolen
-        } 
+        if (L->h_original) {
+            /* may have HAS_ID */
+            B->h_original = L->h_original;
+        }
         else if (L->h_tid & GCFLAG_OLD) {
-            /* became old after becoming priv_from_protected 
-             make L the original
-             */
+            assert(!(L->h_tid & GCFLAG_HAS_ID));
+            /* original must be L */
             B->h_original = (revision_t)L;
         }
-        /* otherwise: L is the original */
-        assert (!(B->h_tid & GCFLAG_HAS_ID));
+        else {
+            /* we can make the backup the "original" */
+            L->h_original = (revision_t)B;
+        }
 
         /* B is now a backup copy, i.e. a protected object, and we own
            the foreign thread's collection_lock, so we can read/write the
@@ -169,8 +175,10 @@
 
         fprintf(stderr, "stolen: %p -> %p\n", P, L);
 
+        
         if (!(L->h_tid & GCFLAG_OLD)) { 
             gcptr O;
+            
             if (L->h_tid & GCFLAG_HAS_ID) {
                 /* use id-copy for us */
                 O = (gcptr)L->h_original;
@@ -184,6 +192,7 @@
                 L->h_revision = (revision_t)O;
                 L->h_original = (revision_t)O;
             }
+
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
             /* subtle: we need to remove L from the fxcache of the target
                thread, otherwise its read barrier might not trigger on it.
@@ -195,6 +204,7 @@
             L = O;
             fprintf(stderr, "\t---> %p\n", L);
         }
+
         assert(L->h_tid & GCFLAG_OLD);
     }
 

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:57 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:57 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: minor changes
Message-ID: <20130621114657.E33931C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r221:16af90468dcd
Date: 2013-06-21 13:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/16af90468dcd/

Log:	minor changes

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -446,6 +446,7 @@
 
   B = stmgc_duplicate_old(P);
   B->h_tid |= GCFLAG_BACKUP_COPY;
+  B->h_tid &= ~GCFLAG_HAS_ID;
   if (!(P->h_original) && (P->h_tid & GCFLAG_OLD)) {
       B->h_original = (revision_t)P;
   }
@@ -1021,6 +1022,7 @@
       }
       else {
         L->h_original = (revision_t)stub;
+        assert(0);
       }
       
       item->val = stub;
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -83,7 +83,6 @@
     gcptr L = (gcptr)stmgcpage_malloc(size);
     memcpy(L, P, size);
     L->h_tid |= GCFLAG_OLD;
-    assert(!(L->h_tid & GCFLAG_HAS_ID));
 
     return L;
 }
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -89,6 +89,7 @@
     }
     else {
         obj->h_original = (revision_t)stub;
+        assert(0);
     }
 
     g2l_insert(&sd->all_stubs, obj, stub);

From noreply at buildbot.pypy.org  Fri Jun 21 13:46:58 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 13:46:58 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: merge default
Message-ID: <20130621114658.EB5571C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r222:2fe2cb9597de
Date: 2013-06-21 13:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/2fe2cb9597de/

Log:	merge default

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -199,6 +199,7 @@
     }
 
   fprintf(stderr, "readobj: %p\n", P);
+  assert(!(P->h_tid & GCFLAG_STUB));
   gcptrlist_insert(&d->list_of_read_objects, P);
 
  add_in_recent_reads_cache:
@@ -492,6 +493,8 @@
                 GCFLAG_WRITE_BARRIER     |
                 0);
   L->h_revision = stm_private_rev_num;
+  assert(stm_private_rev_num < 0);
+  assert(stm_private_rev_num & 1);
   g2l_insert(&d->public_to_private, R, L);
   fprintf(stderr, "write_barrier: adding %p -> %p to public_to_private\n",
           R, L);
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -195,8 +195,11 @@
         return;
 
  restart:
-    if (obj->h_tid & GCFLAG_VISITED)
+    if (obj->h_tid & GCFLAG_VISITED) {
+        fprintf(stderr, "[already visited: %p]\n", obj);
+        assert(obj == *pobj);
         return;    /* already seen */
+    }
 
     if (obj->h_revision & 1) {
         assert(!(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
@@ -223,7 +226,8 @@
                 assert(*pobj == prev_obj);
                 gcptr obj1 = obj;
                 visit(&obj1);       /* recursion, but should be only once */
-                prev_obj->h_revision = ((revision_t)obj1) + 2;
+                assert(prev_obj->h_tid & GCFLAG_STUB);
+                prev_obj->h_revision = ((revision_t)obj1) | 2;
                 return;
             }
         }
@@ -245,8 +249,11 @@
             B->h_tid |= GCFLAG_VISITED;
         }
         else {
+            /* a private_from_protected with a stolen backup copy B */
             assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
-            abort();  // XXX
+            gcptr obj1 = B;
+            visit(&obj1);     /* xxx recursion? */
+            obj->h_revision = (revision_t)obj1;
         }
     }
     obj->h_tid |= GCFLAG_VISITED;
@@ -286,8 +293,12 @@
 {
     //assert(*root == END_MARKER);
     //root++;
-    while (root != end)
-        visit(root++);
+    while (root != end) {
+        gcptr o = *root;
+        visit(root);
+        fprintf(stderr, "visit stack root: %p -> %p\n", o, *root);
+        root++;
+    }
 }
 
 static void mark_all_stack_roots(void)
@@ -346,14 +357,20 @@
 
     for (i = d->list_of_read_objects.size - 1; i >= 0; --i) {
         gcptr obj = items[i];
+        assert(!(obj->h_tid & GCFLAG_STUB));
 
         /* Warning: in case the object listed is outdated and has been
            replaced with a more recent revision, then it might be the
            case that obj->h_revision doesn't have GCFLAG_VISITED, but
            just removing it is very wrong --- we want 'd' to abort.
         */
+        if (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+            assert(IS_POINTER(obj->h_revision));
+            obj = (gcptr)obj->h_revision;
+        }
+
         revision_t v = obj->h_revision;
-        if (IS_POINTER(v) && !(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)) {
+        if (IS_POINTER(v)) {
             /* has a more recent revision.  Oups. */
             fprintf(stderr,
                     "ABRT_COLLECT_MAJOR: %p was read but modified already\n",
@@ -528,6 +545,7 @@
     struct tx_descriptor *d;
     struct tx_descriptor *saved = thread_descriptor;
     revision_t saved_private_rev = stm_private_rev_num;
+    char *read_barrier_cache = stm_read_barrier_cache;
     assert(saved_private_rev == *saved->private_revision_ref);
 
     for (d = stm_tx_head; d; d = d->tx_next) {
@@ -536,16 +554,19 @@
            collection was not preceeded by a minor collection if the
            thread is busy in a system call for example.
         */
-        if (stmgc_minor_collect_anything_to_do(d)) {
+        if (stmgc_minor_collect_anything_to_do(d) ||
+            (d->public_descriptor->stolen_objects.size != 0)) {
             /* Hack: temporarily pretend that we "are" the other thread...
              */
             thread_descriptor = d;
             stm_private_rev_num = *d->private_revision_ref;
+            fxcache_install(&d->recent_reads_cache);
             //assert(stmgc_nursery_hiding(d, 0));
             stmgc_minor_collect_no_abort();
             //assert(stmgc_nursery_hiding(d, 1));
             thread_descriptor = saved;
             stm_private_rev_num = saved_private_rev;
+            stm_read_barrier_cache = read_barrier_cache;
         }
     }
 }
@@ -582,7 +603,7 @@
     countdown_next_major_coll = next;
 }
 
-void stm_major_collect(void)
+static void major_collect(void)
 {
     stmgcpage_acquire_global_lock();
     fprintf(stderr, ",-----\n| running major collection...\n");
@@ -628,7 +649,7 @@
        threads will also acquire the RW lock in exclusive mode, but won't
        do anything. */
     if (countdown_next_major_coll == 0)
-        stm_major_collect();
+        major_collect();
 
     stm_stop_single_thread();
 
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -192,12 +192,17 @@
 
 void _fxcache_reset(struct FXCache *fxcache);
 
+static inline void fxcache_install(struct FXCache *fxcache)
+{
+    stm_read_barrier_cache = (char *)(fxcache->cache + fxcache->shift);
+}
+
 static inline void fxcache_clear(struct FXCache *fxcache)
 {
     fxcache->shift++;
     if (fxcache->shift > FX_TOTAL - FX_ENTRIES)
         _fxcache_reset(fxcache);
-    stm_read_barrier_cache = (char *)(fxcache->cache + fxcache->shift);
+    fxcache_install(fxcache);
 }
 
 #define FXCACHE_AT(obj)  \
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -278,6 +278,7 @@
     gcptr L = (gcptr)(w - 2);
     fprintf(stderr, "trace_stub: %p stub -> %p\n", S, L);
     visit_if_young(&L);
+    assert(S->h_tid & GCFLAG_STUB);
     S->h_revision = ((revision_t)L) | 2;
 }
 
@@ -385,6 +386,7 @@
         else if (obj->h_tid & GCFLAG_NURSERY_MOVED) {
             /* visited nursery objects are kept and updated */
             items[i] = (gcptr)obj->h_revision;
+            assert(!(items[i]->h_tid & GCFLAG_STUB));
             continue;
         }
         /* The listed object was not visited.  Unlist it. */
@@ -474,6 +476,14 @@
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
         assert(gcptrlist_size(&d->public_with_young_copy) == 0);
+        assert(gcptrlist_size(&d->list_of_read_objects) >=
+               d->num_read_objects_known_old);
+        assert(gcptrlist_size(&d->private_from_protected) >=
+               d->num_private_from_protected_known_old);
+        d->num_read_objects_known_old =
+            gcptrlist_size(&d->list_of_read_objects);
+        d->num_private_from_protected_known_old =
+            gcptrlist_size(&d->private_from_protected);
         return 0;
     }
     else {
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -72,6 +72,7 @@
 
     /* found already */
     stub = item->val;
+    assert(stub->h_tid & GCFLAG_STUB);
     assert(stub->h_revision == (((revision_t)obj) | 2));
     goto done;
 
@@ -276,12 +277,14 @@
         if (L == NULL)
             continue;
         assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+        assert(IS_POINTER(L->h_revision));
 
         g2l_insert(&d->public_to_private, B, L);
 
         /* this is definitely needed: all keys in public_to_private
            must appear in list_of_read_objects */
-        fprintf(stderr, "n.readobj: %p\n", B);
+        fprintf(stderr, "n.readobj: %p -> %p\n", B, L);
+        assert(!(B->h_tid & GCFLAG_STUB));
         gcptrlist_insert(&d->list_of_read_objects, B);
 
         /* must also list it here, in case the next minor collect moves it */
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -159,7 +159,7 @@
         else {
             /* atomic transaction: a common case is that callback() returned
                even though we are atomic because we need a major GC.  For
-               that case, release and require the rw lock here. */
+               that case, release and reaquire the rw lock here. */
             stm_possible_safe_point();
         }
 
@@ -225,10 +225,12 @@
     int err = pthread_rwlock_rdlock(&rwlock_shared);
     assert(err == 0);
     //assert(stmgc_nursery_hiding(thread_descriptor, 0));
+    fprintf(stderr, "stm_start_sharedlock\n");
 }
 
 void stm_stop_sharedlock(void)
 {
+    fprintf(stderr, "stm_stop_sharedlock\n");
     //assert(stmgc_nursery_hiding(thread_descriptor, 1));
     int err = pthread_rwlock_unlock(&rwlock_shared);
     assert(err == 0);
@@ -238,10 +240,12 @@
 {
     int err = pthread_rwlock_wrlock(&rwlock_shared);
     assert(err == 0);
+    fprintf(stderr, "start_exclusivelock\n");
 }
 
 static void stop_exclusivelock(void)
 {
+    fprintf(stderr, "stop_exclusivelock\n");
     int err = pthread_rwlock_unlock(&rwlock_shared);
     assert(err == 0);
 }
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -402,7 +402,9 @@
         # parallel_locks[0] is acquired, parallel_locks[1] is acquired
         print 'wait_while_in_parallel enter'
         self.parallel_locks[0].release()
+        lib.stm_stop_sharedlock()
         self.parallel_locks[1].acquire()
+        lib.stm_start_sharedlock()
         print 'wait_while_in_parallel leave'
         # parallel_locks[0] is acquired, parallel_locks[1] is acquired
         self.parallel_locks[1].release()
@@ -485,7 +487,9 @@
     return p.h_tid & GCFLAG_STUB
 
 def check_not_free(p):
+    print >> sys.stderr, "[checking %r..." % p,
     assert 42 < (p.h_tid & 0xFFFF) < 521
+    print >> sys.stderr, "ok]"
 
 def check_nursery_free(p):
     #assert p.h_tid == p.h_revision == 0
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -407,8 +407,10 @@
         assert classify(follow_revision(p)) == "stub"
         assert p1.h_revision & 1
         r.set(2)
+        r.wait(3)     # wait until the other thread really started
     def f2(r):
         r.wait(2)
+        r.set(3)
         p2 = lib.stm_read_barrier(p)    # steals
         assert classify(p2) == "public"
         q2 = lib.getptr(p2, 0)
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -313,3 +313,45 @@
     assert p1b == p1
     check_not_free(follow_revision(p1))
     assert follow_revision(p1).h_tid & GCFLAG_BACKUP_COPY
+
+def test_backup_stolen():
+    p = palloc(HDR)
+    def f1(r):
+        p1 = lib.stm_write_barrier(p)   # private copy
+        lib.stm_push_root(p1)
+        minor_collect()
+        p1 = lib.stm_pop_root()
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        assert classify(p) == "public"
+        assert classify(p1) == "protected"
+        assert classify(follow_revision(p)) == "stub"
+        assert p1.h_revision & 1
+        def cb(c):
+            assert c == 0
+            p1b = lib.stm_write_barrier(p1)
+            assert p1b == p1
+            assert not lib.in_nursery(p1)
+            assert classify(p1) == "private_from_protected"
+            assert classify(follow_revision(p1)) == "backup"
+            lib.stm_push_root(p1)
+            r.wait_while_in_parallel()
+            p1b = lib.stm_pop_root()
+            assert p1b == p1
+            check_not_free(p1)
+            assert classify(p1) == "private_from_protected"
+            assert classify(follow_revision(p1)) == "public"  # has been stolen
+        perform_transaction(cb)
+    def f2(r):
+        def cb(c):
+            assert c == 0
+            r.enter_in_parallel()
+            p2 = lib.stm_read_barrier(p)    # steals
+            assert classify(p2) == "public"
+            print p2
+            major_collect()
+            r.leave_in_parallel()
+            check_not_free(p2)
+            assert classify(p2) == "public"
+        perform_transaction(cb)
+    run_parallel(f1, f2)

From noreply at buildbot.pypy.org  Fri Jun 21 14:10:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Jun 2013 14:10:14 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Running major collections in
 test_random starts to work.
Message-ID: <20130621121014.C280C1C04A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r223:e5e6919b5a54
Date: 2013-06-21 14:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/e5e6919b5a54/

Log:	Running major collections in test_random starts to work.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1211,7 +1211,8 @@
   assert(newrev & 1);
   ACCESS_ONCE(stm_private_rev_num) = newrev;
   fprintf(stderr, "%p: stm_local_revision = %ld\n", d, (long)newrev);
-  assert(d->private_revision_ref = &stm_private_rev_num);
+  assert(d->private_revision_ref == &stm_private_rev_num);
+  assert(d->read_barrier_cache_ref == &stm_read_barrier_cache);
 
   UpdateChainHeads(d, cur_time, localrev);
 
@@ -1459,6 +1460,7 @@
       assert(d->my_lock >= LOCKED);
       stm_private_rev_num = -d->my_lock;
       d->private_revision_ref = &stm_private_rev_num;
+      d->read_barrier_cache_ref = &stm_read_barrier_cache;
       d->max_aborts = -1;
       d->tx_prev = NULL;
       d->tx_next = stm_tx_head;
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -153,6 +153,7 @@
   long long longest_abort_info_time;
   revision_t *private_revision_ref;
   struct FXCache recent_reads_cache;
+  char **read_barrier_cache_ref;
   struct tx_descriptor *tx_prev, *tx_next;
 };
 
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -389,7 +389,6 @@
     }
 
     d->num_read_objects_known_old = d->list_of_read_objects.size;
-    fxcache_clear(&d->recent_reads_cache);
 
     /* We are now after visiting all objects, and we know the
      * transaction isn't aborting because of this collection.  We have
@@ -545,8 +544,10 @@
     struct tx_descriptor *d;
     struct tx_descriptor *saved = thread_descriptor;
     revision_t saved_private_rev = stm_private_rev_num;
-    char *read_barrier_cache = stm_read_barrier_cache;
+    char *saved_read_barrier_cache = stm_read_barrier_cache;
+
     assert(saved_private_rev == *saved->private_revision_ref);
+    assert(saved_read_barrier_cache == *saved->read_barrier_cache_ref);
 
     for (d = stm_tx_head; d; d = d->tx_next) {
         /* Force a minor collection to run in the thread 'd'.
@@ -554,21 +555,24 @@
            collection was not preceeded by a minor collection if the
            thread is busy in a system call for example.
         */
-        if (stmgc_minor_collect_anything_to_do(d) ||
-            (d->public_descriptor->stolen_objects.size != 0)) {
+        if (d != saved) {
             /* Hack: temporarily pretend that we "are" the other thread...
              */
             thread_descriptor = d;
             stm_private_rev_num = *d->private_revision_ref;
-            fxcache_install(&d->recent_reads_cache);
-            //assert(stmgc_nursery_hiding(d, 0));
+            stm_read_barrier_cache = *d->read_barrier_cache_ref;
+
             stmgc_minor_collect_no_abort();
-            //assert(stmgc_nursery_hiding(d, 1));
+
+            assert(stm_private_rev_num == *d->private_revision_ref);
+            *d->read_barrier_cache_ref = stm_read_barrier_cache;
+
             thread_descriptor = saved;
             stm_private_rev_num = saved_private_rev;
-            stm_read_barrier_cache = read_barrier_cache;
+            stm_read_barrier_cache = saved_read_barrier_cache;
         }
     }
+    stmgc_minor_collect_no_abort();
 }
 
 
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -192,17 +192,12 @@
 
 void _fxcache_reset(struct FXCache *fxcache);
 
-static inline void fxcache_install(struct FXCache *fxcache)
-{
-    stm_read_barrier_cache = (char *)(fxcache->cache + fxcache->shift);
-}
-
 static inline void fxcache_clear(struct FXCache *fxcache)
 {
     fxcache->shift++;
     if (fxcache->shift > FX_TOTAL - FX_ENTRIES)
         _fxcache_reset(fxcache);
-    fxcache_install(fxcache);
+    stm_read_barrier_cache = (char *)(fxcache->cache + fxcache->shift);
 }
 
 #define FXCACHE_AT(obj)  \
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -5,10 +5,7 @@
 # a default seed that changes every day, but that can be easily recovered
 DEFAULT_SEED = int(time.strftime("%y%m%d", time.gmtime()))
 
-# XXXXXXXXXXXXXXXXXXXXXX
-# XXX reintroduce me XXX
-# XXXXXXXXXXXXXXXXXXXXXX
-DO_MAJOR_COLLECTS = False
+DO_MAJOR_COLLECTS = True
 
 
 def setup_function(_):

From noreply at buildbot.pypy.org  Fri Jun 21 14:18:30 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 14:18:30 +0200 (CEST)
Subject: [pypy-commit] stmgc default: close anonymous
Message-ID: <20130621121830.9B5141C04A4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r224:348eb80ca0c9
Date: 2013-06-21 13:37 +0200
http://bitbucket.org/pypy/stmgc/changeset/348eb80ca0c9/

Log:	close anonymous


From noreply at buildbot.pypy.org  Fri Jun 21 14:18:31 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 14:18:31 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: mark some paths with assert(0)
 because they don't seem to be reached in demo_random.c
Message-ID: <20130621121831.BB8CB1C0619@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r225:08871aa2ccfe
Date: 2013-06-21 14:11 +0200
http://bitbucket.org/pypy/stmgc/changeset/08871aa2ccfe/

Log:	mark some paths with assert(0) because they don't seem to be reached
	in demo_random.c

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -449,7 +449,11 @@
   B->h_tid |= GCFLAG_BACKUP_COPY;
   B->h_tid &= ~GCFLAG_HAS_ID;
   if (!(P->h_original) && (P->h_tid & GCFLAG_OLD)) {
-      B->h_original = (revision_t)P;
+    /* if P is old, it must be the original
+       if P is young, it will create a shadow original later
+       or it's getting decided when backup gets stolen.
+    */
+    B->h_original = (revision_t)P;
   }
   
   P->h_tid |= GCFLAG_PRIVATE_FROM_PROTECTED;
@@ -479,6 +483,8 @@
      return an old one if the nursery is full at this moment. */
   gcptr L = stmgc_duplicate(R);
   if (!(L->h_original)) {
+    /* if we don't have an original object yet,
+     it must be the old public R */
     assert(R->h_tid & GCFLAG_OLD); // if not, force stm_id??
     L->h_original = (revision_t)R;
   }
@@ -1161,7 +1167,6 @@
         B->h_tid |= GCFLAG_PUBLIC;
         B->h_tid &= ~GCFLAG_BACKUP_COPY;
         if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
-        fprintf(stderr, "%p made outdated, %p is current\n", P, B);
       }
       else
         {
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -100,12 +100,15 @@
     struct tx_descriptor *d = thread_descriptor;
     revision_t result;
 
+    if (p->h_original) { /* fast path */
+        return p->h_original;
+    }
+    
     spinlock_acquire(d->public_descriptor->collection_lock, 'I');
-    if (p->h_original) {
+    if (p->h_original) { /* maybe now? */
         spinlock_release(d->public_descriptor->collection_lock);
         return p->h_original;
     }
-
     /* old objects must have an h_original OR be
        the original itself. 
        if some thread stole p when it was still young,
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -84,9 +84,11 @@
     stub->h_revision = ((revision_t)obj) | 2;
     if (obj->h_original) {
         stub->h_original = obj->h_original;
+        assert(0);
     }
     else if (obj->h_tid & GCFLAG_OLD) {
         stub->h_original = (revision_t)obj;
+        assert(0);
     }
     else {
         obj->h_original = (revision_t)stub;
@@ -122,17 +124,21 @@
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
         
         if (L->h_original) {
-            /* may have HAS_ID */
+            /* L has an original, may be GCFLAG_HAS_ID */
             B->h_original = L->h_original;
         }
         else if (L->h_tid & GCFLAG_OLD) {
+            /* If old, it must be the original */
             assert(!(L->h_tid & GCFLAG_HAS_ID));
             /* original must be L */
             B->h_original = (revision_t)L;
+            assert(0);
         }
         else {
-            /* we can make the backup the "original" */
+            /* we can make the backup the "original"
+             since L hasn't decided yet */
             L->h_original = (revision_t)B;
+            assert(0);
         }
 
         /* B is now a backup copy, i.e. a protected object, and we own

From noreply at buildbot.pypy.org  Fri Jun 21 14:18:33 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 14:18:33 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: fix tests and remove some of the
 assert(0), as they at least seem to be hit in tests
Message-ID: <20130621121833.6D6901C04A4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r226:98c1eebad13c
Date: 2013-06-21 14:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/98c1eebad13c/

Log:	fix tests and remove some of the assert(0), as they at least seem to
	be hit in tests

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -84,15 +84,12 @@
     stub->h_revision = ((revision_t)obj) | 2;
     if (obj->h_original) {
         stub->h_original = obj->h_original;
-        assert(0);
     }
     else if (obj->h_tid & GCFLAG_OLD) {
         stub->h_original = (revision_t)obj;
-        assert(0);
     }
     else {
         obj->h_original = (revision_t)stub;
-        assert(0);
     }
 
     g2l_insert(&sd->all_stubs, obj, stub);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -32,6 +32,7 @@
     typedef struct stm_object_s {
         revision_t h_tid;
         revision_t h_revision;
+        revision_t h_original;
     } *gcptr;
 
     int gettid(gcptr);
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -12,7 +12,7 @@
 
 def test_HDR():
     import struct
-    assert HDR == struct.calcsize("PP")
+    assert HDR == struct.calcsize("PPP")
 
 def test_malloc_simple():
     assert count_pages() == 0

From noreply at buildbot.pypy.org  Fri Jun 21 15:11:34 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 15:11:34 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: fix issue when HAS_ID flag gets
	removed
Message-ID: <20130621131134.877681C0464@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r227:17d1d1ba2d0d
Date: 2013-06-21 15:11 +0200
http://bitbucket.org/pypy/stmgc/changeset/17d1d1ba2d0d/

Log:	fix issue when HAS_ID flag gets removed
	* updated demo_random

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -262,7 +262,7 @@
     num = get_rand(SHARED_ROOTS);
     _sr = shared_roots[num];
 
-    k = get_rand(16);
+    k = get_rand(17);
 
     switch (k) {
     case 0: // remove a root
@@ -324,7 +324,7 @@
         pop_roots();
         p = NULL;
         break;
-    case 15:
+    case 15: /* test stm_id on non-shared roots */
         w_r = (nodeptr)read_barrier(_r);
         if (w_r->id) {
             assert(w_r->id == stm_id((gcptr)w_r));
@@ -335,6 +335,17 @@
             w_r->id = stm_id((gcptr)w_r);
             assert(w_r->id == stm_id((gcptr)_r));
         }
+    case 16: /* test stm_id on shared roots */
+        w_sr = (nodeptr)read_barrier(_sr);
+        if (w_sr->id) {
+            assert(w_sr->id == stm_id((gcptr)w_sr));
+            assert(w_sr->id == stm_id((gcptr)_sr));
+        } 
+        else {
+            w_sr = (nodeptr)write_barrier(_sr);
+            w_sr->id = stm_id((gcptr)w_sr);
+            assert(w_sr->id == stm_id((gcptr)_sr));
+        }
     }
     return p;
 }
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -100,24 +100,34 @@
     struct tx_descriptor *d = thread_descriptor;
     revision_t result;
 
+    
     if (p->h_original) { /* fast path */
+        fprintf(stderr, "stm_id(%p) has orig fst: %p\n", p, p->h_original);
         return p->h_original;
+    } 
+    else if (!(p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+               && (p->h_tid & GCFLAG_OLD)) {
+        /* we can be sure that p->h_original doesn't
+           get set during the if and the else-if */
+        fprintf(stderr, "stm_id(%p) is old, orig=0 fst: %p\n", p, p);
+        return (revision_t)p;
     }
     
     spinlock_acquire(d->public_descriptor->collection_lock, 'I');
-    if (p->h_original) { /* maybe now? */
-        spinlock_release(d->public_descriptor->collection_lock);
-        return p->h_original;
-    }
-    /* old objects must have an h_original OR be
+    /* old objects must have an h_original xOR be
        the original itself. 
        if some thread stole p when it was still young,
        it must have set h_original. stealing an old obj
        makes the old obj "original".
     */
-    if (p->h_tid & GCFLAG_OLD) {
+    if (p->h_original) { /* maybe now? */
+        result = p->h_original;
+        fprintf(stderr, "stm_id(%p) has orig: %p\n", p, p->h_original);
+    }
+    else if (p->h_tid & GCFLAG_OLD) {
         /* it must be this exact object */
         result = (revision_t)p;
+        fprintf(stderr, "stm_id(%p) is old, orig=0: %p\n", p, p);
     }
     else {
         /* must create shadow original object or use
@@ -130,6 +140,8 @@
             // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
             
             result = (revision_t)B;
+            fprintf(stderr, "stm_id(%p) young, pfp, use backup %p\n", 
+                    p, p->h_original);
         }
         else {
             gcptr O = stmgc_duplicate_old(p);
@@ -138,6 +150,7 @@
             O->h_tid |= GCFLAG_PUBLIC;
             
             result = (revision_t)O;
+            fprintf(stderr, "stm_id(%p) young, make shadow %p\n", p, O); 
         }
     }
     
@@ -147,7 +160,9 @@
 
 revision_t stm_pointer_equal(gcptr p1, gcptr p2)
 {
-    /* XXX: */
+    /* types must be the same */
+    if ((p1->h_tid & STM_USER_TID_MASK) != (p2->h_tid & STM_USER_TID_MASK))
+        return 0;
     return stm_id(p1) == stm_id(p2);
 }
 
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -195,7 +195,11 @@
                    if needed */
                 O = stmgc_duplicate_old(L);
                 L->h_revision = (revision_t)O;
-                L->h_original = (revision_t)O;
+
+                /* young and without original?
+                   we may lose the HAS_ID flag like above */
+                if (!(L->h_original))
+                    L->h_original = (revision_t)O;
             }
 
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;

From noreply at buildbot.pypy.org  Fri Jun 21 16:47:11 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 21 Jun 2013 16:47:11 +0200 (CEST)
Subject: [pypy-commit] stmgc implement-id: add simple tests
Message-ID: <20130621144711.230521C02D6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: implement-id
Changeset: r228:d8b5144c17f4
Date: 2013-06-21 16:46 +0200
http://bitbucket.org/pypy/stmgc/changeset/d8b5144c17f4/

Log:	add simple tests

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -13,7 +13,7 @@
 
 
 #define NUMTHREADS 4
-#define STEPS 100000
+#define STEPS 1000000
 #define NUMROOTS 10 // per thread
 #define PREBUILT 3 // per thread
 #define MAXROOTS 1000
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -102,7 +102,8 @@
 
     
     if (p->h_original) { /* fast path */
-        fprintf(stderr, "stm_id(%p) has orig fst: %p\n", p, p->h_original);
+        fprintf(stderr, "stm_id(%p) has orig fst: %p\n", 
+                p, (gcptr)p->h_original);
         return p->h_original;
     } 
     else if (!(p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
@@ -122,7 +123,8 @@
     */
     if (p->h_original) { /* maybe now? */
         result = p->h_original;
-        fprintf(stderr, "stm_id(%p) has orig: %p\n", p, p->h_original);
+        fprintf(stderr, "stm_id(%p) has orig: %p\n", 
+                p, (gcptr)p->h_original);
     }
     else if (p->h_tid & GCFLAG_OLD) {
         /* it must be this exact object */
@@ -141,7 +143,7 @@
             
             result = (revision_t)B;
             fprintf(stderr, "stm_id(%p) young, pfp, use backup %p\n", 
-                    p, p->h_original);
+                    p, (gcptr)p->h_original);
         }
         else {
             gcptr O = stmgc_duplicate_old(p);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -42,6 +42,9 @@
     #define PREBUILT_REVISION      ...
 
     gcptr stm_allocate(size_t size, unsigned int tid);
+    revision_t stm_hash(gcptr);
+    revision_t stm_id(gcptr);
+    revision_t stm_pointer_equal(gcptr, gcptr);
     void stm_push_root(gcptr);
     gcptr stm_pop_root(void);
     void stm_set_max_aborts(int max_aborts);
@@ -108,6 +111,7 @@
     #define GCFLAG_NURSERY_MOVED     ...
     #define GCFLAG_STUB              ...
     #define GCFLAG_PRIVATE_FROM_PROTECTED  ...
+    #define GCFLAG_HAS_ID            ...
     #define ABRT_MANUAL              ...
     typedef struct { ...; } page_header_t;
 ''')
@@ -607,4 +611,10 @@
     assert (r % 4) == 0
     return ffi.cast("gcptr", r)
 
+def follow_original(p):
+    r = p.h_original
+    assert (r % 4) == 0
+    return ffi.cast("gcptr", r)
+
+    
 nrb_protected = ffi.cast("gcptr", -1)
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -198,6 +198,49 @@
     assert p4 == p2
     assert list_of_read_objects() == [p2]
 
+
+def test_id_young_to_old():
+    # move out of nursery with shadow original
+    p = nalloc(HDR)
+    assert p.h_original == 0
+    pid = lib.stm_id(p)
+    assert p.h_tid & GCFLAG_HAS_ID
+    porig = follow_original(p)
+    assert porig.h_tid & GCFLAG_OLD
+    lib.stm_push_root(p)
+    minor_collect()
+    p = lib.stm_pop_root()
+    assert not lib.in_nursery(p)
+    assert pid == lib.stm_id(p)
+
+def test_id_private_from_protected():
+    # read and write from protected
+    p = oalloc(HDR)
+    pid = lib.stm_id(p)
+    porig = follow_original(p)
+    # impl detail {
+    # old objects have id==itself, if not set differently
+    assert porig == ffi.NULL
+    assert ffi.cast("gcptr", pid) == p
+    # }
+
+    p1 = oalloc(HDR)
+    p1id = lib.stm_id(p1)
+    p1r = lib.stm_read_barrier(p1)
+    assert lib.stm_id(p1r) == p1id
+    p1w = lib.stm_write_barrier(p1)
+    assert lib.stm_id(p1w) == p1id
+
+    p2 = oalloc(HDR)
+    p2w = lib.stm_write_barrier(p2)
+    p2id = lib.stm_id(p2)
+    assert p2id == lib.stm_id(p2w)
+    # impl detail {
+    assert p2w.h_original == 0
+    assert follow_revision(p2w).h_original == lib.stm_id(p2w)
+    # }
+    
+
 def test_stealing_old():
     p = palloc(HDR + WORD)
     plist = [p]

From noreply at buildbot.pypy.org  Fri Jun 21 16:57:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Jun 2013 16:57:31 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Two tests and two fixes.
Message-ID: <20130621145731.7BB6E1C149B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r229:5d92b334f594
Date: 2013-06-21 16:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/5d92b334f594/

Log:	Two tests and two fixes.

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -243,6 +243,8 @@
     else {
         assert(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
         gcptr B = (gcptr)obj->h_revision;
+        gcptrlist_insert(&objects_to_trace, B);
+
         if (!(B->h_tid & GCFLAG_PUBLIC)) {
             /* a regular private_from_protected object with a backup copy B */
             assert(B->h_tid & GCFLAG_BACKUP_COPY);
@@ -409,6 +411,21 @@
         item->addr->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
 
     } G2L_LOOP_END;
+
+    /* It can occur that 'private_from_protected' contains an object that
+     * has not been visited at all (maybe only in inevitable
+     * transactions). 
+     */
+    items = d->private_from_protected.items;
+    for (i = d->private_from_protected.size - 1; i >= 0; i--) {
+        gcptr obj = items[i];
+        assert(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+
+        if (!(obj->h_tid & GCFLAG_VISITED)) {
+            /* forget 'obj' */
+            items[i] = items[--d->private_from_protected.size];
+        }
+    }
 }
 
 static void clean_up_lists_of_read_objects_and_fix_outdated_flags(void)
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -355,3 +355,26 @@
             assert classify(p2) == "public"
         perform_transaction(cb)
     run_parallel(f1, f2)
+
+def test_private_from_protected_inevitable():
+    p1 = nalloc(HDR)
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b == p1
+    major_collect()
+
+def test_private_from_protected_trace_backup():
+    p1 = nalloc_refs(1)
+    p2 = nalloc(HDR)
+    lib.setptr(p1, 0, p2)
+    def cb(c):
+        if c == 0:
+            lib.setptr(p1, 0, ffi.NULL)
+            major_collect()
+            abort_and_retry()
+    lib.stm_push_root(p1)
+    perform_transaction(cb)
+    p1 = lib.stm_pop_root()
+    check_not_free(p1)
+    check_not_free(lib.getptr(p1, 0))

From noreply at buildbot.pypy.org  Fri Jun 21 18:11:32 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Fri, 21 Jun 2013 18:11:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix issue with set
 optimization that made frozenset([1,
 2]) & set([2]) a set instead of an frozenset.
Message-ID: <20130621161132.0A15F1C1509@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: 
Changeset: r64950:a178d3a3cf1b
Date: 2013-06-21 18:09 +0200
http://bitbucket.org/pypy/pypy/changeset/a178d3a3cf1b/

Log:	Test and fix issue with set optimization that made frozenset([1, 2])
	& set([2]) a set instead of an frozenset.

diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1062,14 +1062,22 @@
 
     def _intersect_base(self, w_set, w_other):
         if self is w_other.strategy:
-            strategy = w_set.strategy
-            storage = strategy._intersect_unwrapped(w_set, w_other)
+            strategy = self
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = self._intersect_unwrapped(w_other, w_set)
+            else:
+                storage = self._intersect_unwrapped(w_set, w_other)
         elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
             strategy = self.space.fromcache(EmptySetStrategy)
             storage = strategy.get_empty_storage()
         else:
             strategy = self.space.fromcache(ObjectSetStrategy)
-            storage = self._intersect_wrapped(w_set, w_other)
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = w_other.strategy._intersect_wrapped(w_other, w_set)
+            else:
+                storage = self._intersect_wrapped(w_set, w_other)
         return storage, strategy
 
     def _intersect_wrapped(self, w_set, w_other):
@@ -1092,9 +1100,6 @@
         return self.erase(result)
 
     def intersect(self, w_set, w_other):
-        if w_set.length() > w_other.length():
-            return w_other.intersect(w_set)
-
         storage, strategy = self._intersect_base(w_set, w_other)
         return w_set.from_storage_and_strategy(storage, strategy)
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -953,3 +953,10 @@
         # getting a RuntimeError because iterating over the old storage
         # gives us 1, but 1 is not in the set any longer.
         raises(RuntimeError, list, it)
+
+    def test_intersect_frozenset_set(self):
+        # worked before
+        assert type(frozenset([2]) & set([1, 2])) is frozenset
+        # did not work before because of an optimization that swaps both
+        # operands when the first set is larger than the second
+        assert type(frozenset([1, 2]) & set([2])) is frozenset

From noreply at buildbot.pypy.org  Fri Jun 21 21:32:30 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 21 Jun 2013 21:32:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: kill remnants of smallintobject
Message-ID: <20130621193230.C9B6A1C154D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r64951:ed6f1ae8a8f8
Date: 2013-06-20 17:00 -0700
http://bitbucket.org/pypy/pypy/changeset/ed6f1ae8a8f8/

Log:	kill remnants of smallintobject

diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -50,10 +50,6 @@
 def delegate_Bool2IntObject(space, w_bool):
     return W_IntObject(int(w_bool.boolval))
 
-def delegate_Bool2SmallInt(space, w_bool):
-    from pypy.objspace.std.smallintobject import W_SmallIntObject
-    return W_SmallIntObject(int(w_bool.boolval))   # cannot overflow
-
 
 def nonzero__Bool(space, w_bool):
     return w_bool
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -59,11 +59,6 @@
 
 registerimplementation(W_IntObject)
 
-# NB: This code is shared by smallintobject.py, and thus no other Int
-# multimethods should be invoked from these implementations. Instead, add an
-# alias and then teach copy_multimethods in smallintobject.py to override
-# it. See int__Int for example.
-
 def repr__Int(space, w_int1):
     a = w_int1.intval
     res = str(a)

From noreply at buildbot.pypy.org  Fri Jun 21 21:32:32 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 21 Jun 2013 21:32:32 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.4: support --withoutmod-cpyext
Message-ID: <20130621193232.BA4841C154D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: stdlib-2.7.4
Changeset: r64952:489d7cdfdf2f
Date: 2013-06-20 17:00 -0700
http://bitbucket.org/pypy/pypy/changeset/489d7cdfdf2f/

Log:	support --withoutmod-cpyext

diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py
--- a/lib-python/2.7/test/test_support.py
+++ b/lib-python/2.7/test/test_support.py
@@ -19,9 +19,12 @@
 import re
 import time
 import struct
-import _testcapi
 import sysconfig
 try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
+try:
     import thread
 except ImportError:
     thread = None
@@ -952,7 +955,7 @@
     # add GC header size
     if ((type(o) == type) and (o.__flags__ & _TPFLAGS_HEAPTYPE) or\
         ((type(o) != type) and (type(o).__flags__ & _TPFLAGS_HAVE_GC))):
-        size += _testcapi.SIZEOF_PYGC_HEAD
+        size += 1 if _testcapi is None else _testcapi.SIZEOF_PYGC_HEAD
     msg = 'wrong size for %s: got %d, expected %d' \
             % (type(o), result, size)
     test.assertEqual(result, size, msg)

From noreply at buildbot.pypy.org  Fri Jun 21 21:32:34 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 21 Jun 2013 21:32:34 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.4: cpython issues #16029/16030: Fix
 pickling and repr of large xranges
Message-ID: <20130621193234.1A5F61C154D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: stdlib-2.7.4
Changeset: r64953:de804d0c19b6
Date: 2013-06-20 17:01 -0700
http://bitbucket.org/pypy/pypy/changeset/de804d0c19b6/

Log:	cpython issues #16029/16030: Fix pickling and repr of large xranges

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -2,6 +2,7 @@
 Interp-level definition of frequently used functionals.
 
 """
+import sys
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
@@ -367,13 +368,12 @@
         return space.wrap(obj)
 
     def descr_repr(self):
-        stop = self.start + self.len * self.step
         if self.start == 0 and self.step == 1:
-            s = "xrange(%d)" % (stop,)
+            s = "xrange(%d)" % (self._get_stop(),)
         elif self.step == 1:
-            s = "xrange(%d, %d)" % (self.start, stop)
+            s = "xrange(%d, %d)" % (self.start, self._get_stop())
         else:
-            s = "xrange(%d, %d, %d)" %(self.start, stop, self.step)
+            s = "xrange(%d, %d, %d)" %(self.start, self._get_stop(), self.step)
         return self.space.wrap(s)
 
     def descr_len(self):
@@ -402,19 +402,29 @@
                                                     self.len, self.step))
 
     def descr_reversed(self):
-        lastitem = self.start + (self.len-1) * self.step
-        return self.space.wrap(W_XRangeIterator(self.space, lastitem,
-                                                self.len, -self.step))
+        last = self.start + (self.len - 1) * self.step
+        return self.space.wrap(W_XRangeIterator(self.space, last, self.len,
+                                                -self.step))
 
     def descr_reduce(self):
         space = self.space
         return space.newtuple(
             [space.type(self),
              space.newtuple([space.wrap(self.start),
-                             space.wrap(self.start + self.len * self.step),
+                             space.wrap(self._get_stop()),
                              space.wrap(self.step)])
              ])
 
+    def _get_stop(self):
+        if not self.len:
+            return self.start
+        step = self.step
+        last = self.start + (self.len - 1) * step
+        if step > 0:
+            return sys.maxint if last > sys.maxint - step else last + step
+        minint = -sys.maxint - 1
+        return minint if last < minint - step else last + step
+
 def _toint(space, w_obj):
     # this also supports float arguments.  CPython still does, too.
     # needs a bit more thinking in general...
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -188,6 +188,22 @@
         x = xrange(1)
         assert type(reversed(x)) == type(iter(x))
 
+    def test_cpython_issue16029(self):
+        import sys
+        M = min(sys.maxint, sys.maxsize)
+        x = xrange(0, M, M - 1)
+        assert x.__reduce__() == (xrange, (0, M, M - 1))
+        x = xrange(0, -M, 1 - M)
+        assert x.__reduce__() == (xrange, (0, -M - 1, 1 - M))
+
+    def test_cpython_issue16030(self):
+        import sys
+        M = min(sys.maxint, sys.maxsize)
+        x = xrange(0, M, M - 1)
+        assert repr(x) == 'xrange(0, %s, %s)' % (M, M - 1)
+        x = xrange(0, -M, 1 - M)
+        assert repr(x) == 'xrange(0, %s, %s)' % (-M - 1, 1 - M)
+
 
 class AppTestReversed:
     def test_reversed(self):

From noreply at buildbot.pypy.org  Fri Jun 21 21:33:23 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 21 Jun 2013 21:33:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge upstream
Message-ID: <20130621193323.8761F1C154D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r64954:d0acdb6e6b8a
Date: 2013-06-21 12:31 -0700
http://bitbucket.org/pypy/pypy/changeset/d0acdb6e6b8a/

Log:	merge upstream

diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1062,14 +1062,22 @@
 
     def _intersect_base(self, w_set, w_other):
         if self is w_other.strategy:
-            strategy = w_set.strategy
-            storage = strategy._intersect_unwrapped(w_set, w_other)
+            strategy = self
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = self._intersect_unwrapped(w_other, w_set)
+            else:
+                storage = self._intersect_unwrapped(w_set, w_other)
         elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
             strategy = self.space.fromcache(EmptySetStrategy)
             storage = strategy.get_empty_storage()
         else:
             strategy = self.space.fromcache(ObjectSetStrategy)
-            storage = self._intersect_wrapped(w_set, w_other)
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = w_other.strategy._intersect_wrapped(w_other, w_set)
+            else:
+                storage = self._intersect_wrapped(w_set, w_other)
         return storage, strategy
 
     def _intersect_wrapped(self, w_set, w_other):
@@ -1092,9 +1100,6 @@
         return self.erase(result)
 
     def intersect(self, w_set, w_other):
-        if w_set.length() > w_other.length():
-            return w_other.intersect(w_set)
-
         storage, strategy = self._intersect_base(w_set, w_other)
         return w_set.from_storage_and_strategy(storage, strategy)
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -953,3 +953,10 @@
         # getting a RuntimeError because iterating over the old storage
         # gives us 1, but 1 is not in the set any longer.
         raises(RuntimeError, list, it)
+
+    def test_intersect_frozenset_set(self):
+        # worked before
+        assert type(frozenset([2]) & set([1, 2])) is frozenset
+        # did not work before because of an optimization that swaps both
+        # operands when the first set is larger than the second
+        assert type(frozenset([1, 2]) & set([2])) is frozenset

From noreply at buildbot.pypy.org  Sat Jun 22 14:42:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 14:42:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fixes probably (but the tests fail a
 bit unpredictably...)
Message-ID: <20130622124238.E3CA51C0212@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r230:3a3f0932121a
Date: 2013-06-21 17:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/3a3f0932121a/

Log:	Fixes probably (but the tests fail a bit unpredictably...)

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -661,11 +661,15 @@
                      commit, but if they both enter the SpinLoop()
                      above, then they will livelock.
 
-                     XXX This might lead both threads to cancel by
-                     reaching this point.  It might be possible to be
+                     But this might lead both threads to cancel by
+                     reaching this point.  For now we attempt to be
                      more clever and let one of the threads commit
-                     anyway.
+                     anyway (the choice of which one looks random).
                   */
+                  if (d->my_lock < v) {
+                      SpinLoop(SPLP_LOCKED_VALIDATE);
+                      goto retry;
+                  }
                   fprintf(stderr, "validation failed: "
                           "%p is locked by another thread\n", R);
                   return 0;
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -386,8 +386,15 @@
            inevitable, and since it started, it popped objects out of
            its shadow stack.  Some popped objects might become free even
            if they have been read from.  But for inevitable transactions,
-           we clear the 'list_of_read_objects' above anyway. */
-        assert(obj->h_tid & GCFLAG_VISITED);
+           we clear the 'list_of_read_objects' above anyway.
+           
+           However, some situations can occur (I believe) only in tests.
+           To be on the safe side, do the right thing and unlist the
+           non-visited object.
+           */
+        if (!(obj->h_tid & GCFLAG_VISITED)) {
+            items[i] = items[--d->list_of_read_objects.size];
+        }
     }
 
     d->num_read_objects_known_old = d->list_of_read_objects.size;

From noreply at buildbot.pypy.org  Sat Jun 22 14:42:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 14:42:40 +0200 (CEST)
Subject: [pypy-commit] stmgc default: More asserts. The next issue is that
 we need to reset GCFLAG_VISITED
Message-ID: <20130622124240.2AA7F1C02D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r231:0484208e300e
Date: 2013-06-21 17:32 +0200
http://bitbucket.org/pypy/stmgc/changeset/0484208e300e/

Log:	More asserts. The next issue is that we need to reset
	GCFLAG_VISITED on stubs (and free unused stubs, too).

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -198,6 +198,8 @@
     if (obj->h_tid & GCFLAG_VISITED) {
         fprintf(stderr, "[already visited: %p]\n", obj);
         assert(obj == *pobj);
+        assert((obj->h_revision & 3) ||   /* either odd, or stub */
+               (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
         return;    /* already seen */
     }
 
@@ -248,6 +250,7 @@
         if (!(B->h_tid & GCFLAG_PUBLIC)) {
             /* a regular private_from_protected object with a backup copy B */
             assert(B->h_tid & GCFLAG_BACKUP_COPY);
+            assert(B->h_revision & 1);
             B->h_tid |= GCFLAG_VISITED;
         }
         else {
@@ -285,8 +288,8 @@
     for (; pobj != pend; pobj++) {
         obj = *pobj;
         assert(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL);
-        assert(obj->h_tid & GCFLAG_VISITED);
-        assert((obj->h_revision & 1) == 0);   /* "is a pointer" */
+        obj->h_tid &= ~GCFLAG_VISITED;
+        assert(IS_POINTER(obj->h_revision));
         visit((gcptr *)&obj->h_revision);
     }
 }

From noreply at buildbot.pypy.org  Sat Jun 22 14:42:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 14:42:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: hg merge implement-id
Message-ID: <20130622124241.755181C0F10@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r232:db385d3d043d
Date: 2013-06-22 11:12 +0200
http://bitbucket.org/pypy/stmgc/changeset/db385d3d043d/

Log:	hg merge implement-id

	Right now test_multi_thread doesn't pass, but I'll investigate

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -13,7 +13,7 @@
 
 
 #define NUMTHREADS 4
-#define STEPS 100000
+#define STEPS 1000000
 #define NUMROOTS 10 // per thread
 #define PREBUILT 3 // per thread
 #define MAXROOTS 1000
@@ -27,6 +27,7 @@
 struct node {
     struct stm_object_s hdr;
     long value;
+    revision_t id;
     struct node *next;
 };
 typedef struct node * nodeptr;
@@ -82,6 +83,7 @@
     gcptr x = calloc(1, size);
     x->h_tid = PREBUILT_FLAGS | tid;
     x->h_revision = PREBUILT_REVISION;
+    x->h_original = 0;
     return x;
 }
 
@@ -183,12 +185,12 @@
     return result1;
 }
 
-static const revision_t C_PRIVATE_FROM_PROTECTED = 1;
-static const revision_t C_PRIVATE                = 2;
-static const revision_t C_STUB                   = 3;
-static const revision_t C_PUBLIC                 = 4;
-static const revision_t C_BACKUP                 = 5;
-static const revision_t C_PROTECTED              = 6;
+static const int C_PRIVATE_FROM_PROTECTED = 1;
+static const int C_PRIVATE                = 2;
+static const int C_STUB                   = 3;
+static const int C_PUBLIC                 = 4;
+static const int C_BACKUP                 = 5;
+static const int C_PROTECTED              = 6;
 int classify(gcptr p)
 {
     int priv_from_prot = (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) != 0;
@@ -260,7 +262,7 @@
     num = get_rand(SHARED_ROOTS);
     _sr = shared_roots[num];
 
-    k = get_rand(14);
+    k = get_rand(17);
 
     switch (k) {
     case 0: // remove a root
@@ -316,8 +318,34 @@
         w_sr = (nodeptr)write_barrier(_sr);
         w_sr->next = (nodeptr)shared_roots[get_rand(SHARED_ROOTS)];
         break;
-    default:
+    case 14:
+        push_roots();
+        stmgc_minor_collect();
+        pop_roots();
+        p = NULL;
         break;
+    case 15: /* test stm_id on non-shared roots */
+        w_r = (nodeptr)read_barrier(_r);
+        if (w_r->id) {
+            assert(w_r->id == stm_id((gcptr)w_r));
+            assert(w_r->id == stm_id((gcptr)_r));
+        } 
+        else {
+            w_r = (nodeptr)write_barrier(_r);
+            w_r->id = stm_id((gcptr)w_r);
+            assert(w_r->id == stm_id((gcptr)_r));
+        }
+    case 16: /* test stm_id on shared roots */
+        w_sr = (nodeptr)read_barrier(_sr);
+        if (w_sr->id) {
+            assert(w_sr->id == stm_id((gcptr)w_sr));
+            assert(w_sr->id == stm_id((gcptr)_sr));
+        } 
+        else {
+            w_sr = (nodeptr)write_barrier(_sr);
+            w_sr->id = stm_id((gcptr)w_sr);
+            assert(w_sr->id == stm_id((gcptr)_sr));
+        }
     }
     return p;
 }
@@ -344,17 +372,21 @@
 int interruptible_callback(gcptr arg1, int retry_counter)
 {
     td.num_roots = td.num_roots_outside_perform;
-    copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
+    // done & overwritten by the following pop_roots():
+    // copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
 
+    // refresh td.roots:
     arg1 = stm_pop_root();
+    assert(arg1 == NULL);
     pop_roots();
     push_roots();
     stm_push_root(arg1);
 
     int p = run_me();
-    int restart = p == -1 ? get_rand(3) != 1 : 0;
+    if (p == -1) // maybe restart transaction
+        return get_rand(3) != 1;
 
-    return restart;
+    return 0;
 }
 
 int run_me()
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -447,7 +447,15 @@
 
   B = stmgc_duplicate_old(P);
   B->h_tid |= GCFLAG_BACKUP_COPY;
-
+  B->h_tid &= ~GCFLAG_HAS_ID;
+  if (!(P->h_original) && (P->h_tid & GCFLAG_OLD)) {
+    /* if P is old, it must be the original
+       if P is young, it will create a shadow original later
+       or it's getting decided when backup gets stolen.
+    */
+    B->h_original = (revision_t)P;
+  }
+  
   P->h_tid |= GCFLAG_PRIVATE_FROM_PROTECTED;
   P->h_revision = (revision_t)B;
 
@@ -474,6 +482,13 @@
   /* note that stmgc_duplicate() usually returns a young object, but may
      return an old one if the nursery is full at this moment. */
   gcptr L = stmgc_duplicate(R);
+  if (!(L->h_original)) {
+    /* if we don't have an original object yet,
+     it must be the old public R */
+    assert(R->h_tid & GCFLAG_OLD); // if not, force stm_id??
+    L->h_original = (revision_t)R;
+  }
+
   assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
   assert(!(L->h_tid & GCFLAG_STUB));
   assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
@@ -1010,7 +1025,19 @@
       stub->h_tid = (L->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
                                                    | GCFLAG_STUB
                                                    | GCFLAG_OLD;
+      assert(!(L->h_tid & GCFLAG_HAS_ID));
       stub->h_revision = ((revision_t)L) | 2;
+      if (L->h_original) {
+        stub->h_original = L->h_original;
+      }
+      else if (L->h_tid & GCFLAG_OLD) {
+        stub->h_original = (revision_t)L;
+      }
+      else {
+        L->h_original = (revision_t)stub;
+        assert(0);
+      }
+      
       item->val = stub;
 
     } G2L_LOOP_END;
@@ -1076,6 +1103,8 @@
 
       if (B->h_tid & GCFLAG_PUBLIC)
         {
+          assert(!(P->h_tid & GCFLAG_HAS_ID));
+
           /* B was stolen */
           while (1)
             {
@@ -1086,7 +1115,15 @@
               if (bool_cas(&B->h_revision, v, (revision_t)P))
                 break;
             }
-        }
+        }      
+      else if (P->h_original == (revision_t)B) {
+        /* The backup is the "id object" */
+        assert(!(P->h_tid & GCFLAG_HAS_ID));
+
+        B->h_tid &= ~GCFLAG_BACKUP_COPY;
+        B->h_tid |= GCFLAG_PUBLIC;
+        B->h_revision = (revision_t)P;
+      }
       else
         {
           stmgcpage_free(B);
@@ -1118,6 +1155,7 @@
         {
           assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
           P->h_tid |= GCFLAG_PUBLIC;
+          P->h_tid &= ~GCFLAG_HAS_ID; // just in case
           if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
           /* P becomes a public outdated object.  It may create an
              exception documented in doc-objects.txt: a public but young
@@ -1126,10 +1164,20 @@
              stealing will follow its h_revision (to B).
           */
         }
+      else if (P->h_original == (revision_t)B) {
+        /* The backup is the "id object".  P becomes outdated. */
+        assert(!(P->h_tid & GCFLAG_HAS_ID));
+        P->h_tid |= GCFLAG_PUBLIC;
+        B->h_tid |= GCFLAG_PUBLIC;
+        B->h_tid &= ~GCFLAG_BACKUP_COPY;
+        if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
+      }
       else
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
+          assert(!(P->h_original) 
+                 || (B->h_original == (revision_t)P->h_original));
           size_t size = stmcb_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -57,6 +57,9 @@
  * but converted from a protected.  These are precisely the objects
  * that have a backup copy (in h_revision), which gives a copy of the
  * original protected object.
+ * 
+ * GCFLAG_HAS_ID is set on young objects that have an old reserved
+ * memory to be copied to in minor collections (obj->h_original)
  */
 static const revision_t GCFLAG_OLD                    = STM_FIRST_GCFLAG << 0;
 static const revision_t GCFLAG_VISITED                = STM_FIRST_GCFLAG << 1;
@@ -68,6 +71,7 @@
 static const revision_t GCFLAG_BACKUP_COPY  /*debug*/ = STM_FIRST_GCFLAG << 7;
 static const revision_t GCFLAG_STUB         /*debug*/ = STM_FIRST_GCFLAG << 8;
 static const revision_t GCFLAG_PRIVATE_FROM_PROTECTED = STM_FIRST_GCFLAG << 9;
+static const revision_t GCFLAG_HAS_ID                 = STM_FIRST_GCFLAG << 10;
 
 /* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
 #define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -59,6 +59,7 @@
     assert(tid == (tid & STM_USER_TID_MASK));
     P->h_tid = tid;
     P->h_revision = stm_private_rev_num;
+    P->h_original = 0;
     return P;
 }
 
@@ -71,6 +72,8 @@
 
     memcpy(L, P, size);
     L->h_tid &= ~GCFLAG_OLD;
+    L->h_tid &= ~GCFLAG_HAS_ID;
+
     return L;
 }
 
@@ -80,11 +83,93 @@
     gcptr L = (gcptr)stmgcpage_malloc(size);
     memcpy(L, P, size);
     L->h_tid |= GCFLAG_OLD;
+
     return L;
 }
 
 /************************************************************/
 
+
+revision_t stm_hash(gcptr p)
+{
+    return stm_id(p);
+}
+
+revision_t stm_id(gcptr p)
+{
+    struct tx_descriptor *d = thread_descriptor;
+    revision_t result;
+
+    
+    if (p->h_original) { /* fast path */
+        fprintf(stderr, "stm_id(%p) has orig fst: %p\n", 
+                p, (gcptr)p->h_original);
+        return p->h_original;
+    } 
+    else if (!(p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+               && (p->h_tid & GCFLAG_OLD)) {
+        /* we can be sure that p->h_original doesn't
+           get set during the if and the else-if */
+        fprintf(stderr, "stm_id(%p) is old, orig=0 fst: %p\n", p, p);
+        return (revision_t)p;
+    }
+    
+    spinlock_acquire(d->public_descriptor->collection_lock, 'I');
+    /* old objects must have an h_original xOR be
+       the original itself. 
+       if some thread stole p when it was still young,
+       it must have set h_original. stealing an old obj
+       makes the old obj "original".
+    */
+    if (p->h_original) { /* maybe now? */
+        result = p->h_original;
+        fprintf(stderr, "stm_id(%p) has orig: %p\n", 
+                p, (gcptr)p->h_original);
+    }
+    else if (p->h_tid & GCFLAG_OLD) {
+        /* it must be this exact object */
+        result = (revision_t)p;
+        fprintf(stderr, "stm_id(%p) is old, orig=0: %p\n", p, p);
+    }
+    else {
+        /* must create shadow original object or use
+           backup, if exists */
+        if (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+            gcptr B = (gcptr)p->h_revision;
+            /* don't set, otherwise nursery will copy over backup */
+            //p->h_tid |= GCFLAG_HAS_ID; // see AbortPrivateFromProtected
+            p->h_original = (revision_t)B;
+            // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
+            
+            result = (revision_t)B;
+            fprintf(stderr, "stm_id(%p) young, pfp, use backup %p\n", 
+                    p, (gcptr)p->h_original);
+        }
+        else {
+            gcptr O = stmgc_duplicate_old(p);
+            p->h_original = (revision_t)O;
+            p->h_tid |= GCFLAG_HAS_ID;
+            O->h_tid |= GCFLAG_PUBLIC;
+            
+            result = (revision_t)O;
+            fprintf(stderr, "stm_id(%p) young, make shadow %p\n", p, O); 
+        }
+    }
+    
+    spinlock_release(d->public_descriptor->collection_lock);
+    return result;
+}
+
+revision_t stm_pointer_equal(gcptr p1, gcptr p2)
+{
+    /* types must be the same */
+    if ((p1->h_tid & STM_USER_TID_MASK) != (p2->h_tid & STM_USER_TID_MASK))
+        return 0;
+    return stm_id(p1) == stm_id(p2);
+}
+
+/************************************************************/
+
 static inline gcptr create_old_object_copy(gcptr obj)
 {
     assert(!(obj->h_tid & GCFLAG_PUBLIC));
@@ -100,6 +185,18 @@
     return fresh_old_copy;
 }
 
+inline void copy_to_old_id_copy(gcptr obj, gcptr id)
+{
+    assert(!is_in_nursery(thread_descriptor, id));
+    assert(id->h_tid & GCFLAG_OLD);
+
+    size_t size = stmcb_size(obj);
+    memcpy(id, obj, size);
+    id->h_tid &= ~GCFLAG_HAS_ID;
+    id->h_tid |= GCFLAG_OLD;
+    fprintf(stderr, "copy_to_old_id_copy(%p -> %p)\n", obj, id);
+}
+
 static void visit_if_young(gcptr *root)
 {
     gcptr obj = *root;
@@ -111,17 +208,29 @@
     }
     else {
         /* it's a nursery object.  Was it already moved? */
-
         if (UNLIKELY(obj->h_tid & GCFLAG_NURSERY_MOVED)) {
             /* yes.  Such an object can be a public object in the nursery
                too (such objects are always NURSERY_MOVED).  For all cases,
-               we can just fix the ref. */
+               we can just fix the ref. 
+               Can be stolen objects or those we already moved.
+            */
             *root = (gcptr)obj->h_revision;
             return;
         }
 
-        /* make a copy of it outside */
-        fresh_old_copy = create_old_object_copy(obj);
+        if (obj->h_tid & GCFLAG_HAS_ID) {
+            /* already has a place to go to */
+            gcptr id_obj = (gcptr)obj->h_original;
+
+            copy_to_old_id_copy(obj, id_obj);
+            fresh_old_copy = id_obj;
+            obj->h_tid &= ~GCFLAG_HAS_ID;
+        } 
+        else {
+            /* make a copy of it outside */
+            fresh_old_copy = create_old_object_copy(obj);
+        }
+        
         obj->h_tid |= GCFLAG_NURSERY_MOVED;
         obj->h_revision = (revision_t)fresh_old_copy;
 
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -11,6 +11,8 @@
     struct stm_object_s stubs[STUB_NB_OBJS];
 };
 
+inline void copy_to_old_id_copy(gcptr obj, gcptr id);
+
 gcptr stm_stub_malloc(struct tx_public_descriptor *pd)
 {
     assert(pd->collection_lock != 0);
@@ -80,6 +82,16 @@
                                                    | GCFLAG_STUB
                                                    | GCFLAG_OLD;
     stub->h_revision = ((revision_t)obj) | 2;
+    if (obj->h_original) {
+        stub->h_original = obj->h_original;
+    }
+    else if (obj->h_tid & GCFLAG_OLD) {
+        stub->h_original = (revision_t)obj;
+    }
+    else {
+        obj->h_original = (revision_t)stub;
+    }
+
     g2l_insert(&sd->all_stubs, obj, stub);
 
     if (!(obj->h_tid & GCFLAG_OLD))
@@ -107,6 +119,24 @@
     */
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
+        
+        if (L->h_original) {
+            /* L has an original, may be GCFLAG_HAS_ID */
+            B->h_original = L->h_original;
+        }
+        else if (L->h_tid & GCFLAG_OLD) {
+            /* If old, it must be the original */
+            assert(!(L->h_tid & GCFLAG_HAS_ID));
+            /* original must be L */
+            B->h_original = (revision_t)L;
+            assert(0);
+        }
+        else {
+            /* we can make the backup the "original"
+             since L hasn't decided yet */
+            L->h_original = (revision_t)B;
+            assert(0);
+        }
 
         /* B is now a backup copy, i.e. a protected object, and we own
            the foreign thread's collection_lock, so we can read/write the
@@ -150,10 +180,28 @@
 
         fprintf(stderr, "stolen: %p -> %p\n", P, L);
 
-        /* Copy the object out of the other thread's nursery, if needed */
-        if (!(L->h_tid & GCFLAG_OLD)) {
-            gcptr O = stmgc_duplicate_old(L);
-            L->h_revision = (revision_t)O;
+        
+        if (!(L->h_tid & GCFLAG_OLD)) { 
+            gcptr O;
+            
+            if (L->h_tid & GCFLAG_HAS_ID) {
+                /* use id-copy for us */
+                O = (gcptr)L->h_original;
+                L->h_tid &= ~GCFLAG_HAS_ID;
+                L->h_revision = (revision_t)O;
+                copy_to_old_id_copy(L, (gcptr)L->h_original);
+            } else {
+                /* Copy the object out of the other thread's nursery, 
+                   if needed */
+                O = stmgc_duplicate_old(L);
+                L->h_revision = (revision_t)O;
+
+                /* young and without original?
+                   we may lose the HAS_ID flag like above */
+                if (!(L->h_original))
+                    L->h_original = (revision_t)O;
+            }
+
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
             /* subtle: we need to remove L from the fxcache of the target
                thread, otherwise its read barrier might not trigger on it.
@@ -165,6 +213,7 @@
             L = O;
             fprintf(stderr, "\t---> %p\n", L);
         }
+
         assert(L->h_tid & GCFLAG_OLD);
     }
 
diff --git a/c4/steal.h b/c4/steal.h
--- a/c4/steal.h
+++ b/c4/steal.h
@@ -2,7 +2,7 @@
 #define _SRCSTM_STEAL_H
 
 
-#define STUB_BLOCK_SIZE   (16 * WORD)    /* power of two */
+#define STUB_BLOCK_SIZE   (32 * WORD)    /* power of two */
 
 #define STUB_THREAD(h)    (*(struct tx_public_descriptor **)           \
                             (((revision_t)(h)) & ~(STUB_BLOCK_SIZE-1)))
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -10,6 +10,7 @@
 typedef struct stm_object_s {
     revision_t h_tid;
     revision_t h_revision;
+    revision_t h_original;
 } *gcptr;
 
 
@@ -28,6 +29,14 @@
 /* allocate an object out of the local nursery */
 gcptr stm_allocate(size_t size, unsigned long tid);
 
+/* returns a never changing hash for the object */
+revision_t stm_hash(gcptr);
+/* returns an for the object which is unique during its lifetime */
+revision_t stm_id(gcptr);
+/* returns nonzero if the two object-copy pointers belong to the
+same original object */
+revision_t stm_pointer_equal(gcptr, gcptr);
+
 /* to push/pop objects into the local shadowstack */
 /* (could be turned into macros or something later) */
 void stm_push_root(gcptr);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -32,6 +32,7 @@
     typedef struct stm_object_s {
         revision_t h_tid;
         revision_t h_revision;
+        revision_t h_original;
     } *gcptr;
 
     int gettid(gcptr);
@@ -41,6 +42,9 @@
     #define PREBUILT_REVISION      ...
 
     gcptr stm_allocate(size_t size, unsigned int tid);
+    revision_t stm_hash(gcptr);
+    revision_t stm_id(gcptr);
+    revision_t stm_pointer_equal(gcptr, gcptr);
     void stm_push_root(gcptr);
     gcptr stm_pop_root(void);
     void stm_set_max_aborts(int max_aborts);
@@ -107,6 +111,7 @@
     #define GCFLAG_NURSERY_MOVED     ...
     #define GCFLAG_STUB              ...
     #define GCFLAG_PRIVATE_FROM_PROTECTED  ...
+    #define GCFLAG_HAS_ID            ...
     #define ABRT_MANUAL              ...
     typedef struct { ...; } page_header_t;
 ''')
@@ -606,4 +611,10 @@
     assert (r % 4) == 0
     return ffi.cast("gcptr", r)
 
+def follow_original(p):
+    r = p.h_original
+    assert (r % 4) == 0
+    return ffi.cast("gcptr", r)
+
+    
 nrb_protected = ffi.cast("gcptr", -1)
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -198,6 +198,49 @@
     assert p4 == p2
     assert list_of_read_objects() == [p2]
 
+
+def test_id_young_to_old():
+    # move out of nursery with shadow original
+    p = nalloc(HDR)
+    assert p.h_original == 0
+    pid = lib.stm_id(p)
+    assert p.h_tid & GCFLAG_HAS_ID
+    porig = follow_original(p)
+    assert porig.h_tid & GCFLAG_OLD
+    lib.stm_push_root(p)
+    minor_collect()
+    p = lib.stm_pop_root()
+    assert not lib.in_nursery(p)
+    assert pid == lib.stm_id(p)
+
+def test_id_private_from_protected():
+    # read and write from protected
+    p = oalloc(HDR)
+    pid = lib.stm_id(p)
+    porig = follow_original(p)
+    # impl detail {
+    # old objects have id==itself, if not set differently
+    assert porig == ffi.NULL
+    assert ffi.cast("gcptr", pid) == p
+    # }
+
+    p1 = oalloc(HDR)
+    p1id = lib.stm_id(p1)
+    p1r = lib.stm_read_barrier(p1)
+    assert lib.stm_id(p1r) == p1id
+    p1w = lib.stm_write_barrier(p1)
+    assert lib.stm_id(p1w) == p1id
+
+    p2 = oalloc(HDR)
+    p2w = lib.stm_write_barrier(p2)
+    p2id = lib.stm_id(p2)
+    assert p2id == lib.stm_id(p2w)
+    # impl detail {
+    assert p2w.h_original == 0
+    assert follow_revision(p2w).h_original == lib.stm_id(p2w)
+    # }
+    
+
 def test_stealing_old():
     p = palloc(HDR + WORD)
     plist = [p]
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -12,7 +12,7 @@
 
 def test_HDR():
     import struct
-    assert HDR == struct.calcsize("PP")
+    assert HDR == struct.calcsize("PPP")
 
 def test_malloc_simple():
     assert count_pages() == 0

From noreply at buildbot.pypy.org  Sat Jun 22 14:42:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 14:42:42 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Reset this to False in the check-in
 version until it passes more reliably.
Message-ID: <20130622124242.809141C13F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r233:c7aa0eeec5b1
Date: 2013-06-22 14:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/c7aa0eeec5b1/

Log:	Reset this to False in the check-in version until it passes more
	reliably.

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -5,7 +5,7 @@
 # a default seed that changes every day, but that can be easily recovered
 DEFAULT_SEED = int(time.strftime("%y%m%d", time.gmtime()))
 
-DO_MAJOR_COLLECTS = True
+DO_MAJOR_COLLECTS = False
 
 
 def setup_function(_):

From noreply at buildbot.pypy.org  Sat Jun 22 14:56:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 14:56:20 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Simplify the allocation of stubs,
 at the cost of ~9/10th of a pointer :-)
Message-ID: <20130622125620.708CE1C02D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r234:87869a164209
Date: 2013-06-22 14:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/87869a164209/

Log:	Simplify the allocation of stubs, at the cost of ~9/10th of a
	pointer :-) For now it's good enough. We'll try later to compress
	one or two pointer fields off some objects.

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -1,50 +1,14 @@
 #include "stmimpl.h"
 
 
-#define STUB_PAGE         (4096 - 2*WORD)
-#define STUB_NB_OBJS      ((STUB_BLOCK_SIZE - 2*WORD) /    \
-                             sizeof(struct stm_object_s))
-
-struct stub_block_s {
-    struct tx_public_descriptor *thread;
-    struct stub_block_s *next;
-    struct stm_object_s stubs[STUB_NB_OBJS];
-};
-
 inline void copy_to_old_id_copy(gcptr obj, gcptr id);
 
 gcptr stm_stub_malloc(struct tx_public_descriptor *pd)
 {
     assert(pd->collection_lock != 0);
 
-    gcptr p = pd->stub_free_list;
-    if (p == NULL) {
-        assert(sizeof(struct stub_block_s) == STUB_BLOCK_SIZE);
-
-        /* XXX free! */
-        char *page = stm_malloc(STUB_PAGE);
-        char *page_end = page + STUB_PAGE;
-        page += (-(revision_t)page) & (STUB_BLOCK_SIZE-1);  /* round up */
-
-        struct stub_block_s *b = (struct stub_block_s *)page;
-        struct stub_block_s *nextb = NULL;
-        gcptr nextp = NULL;
-        int i;
-
-        while (((char *)(b + 1)) <= page_end) {
-            b->thread = pd;
-            b->next = nextb;
-            for (i = 0; i < STUB_NB_OBJS; i++) {
-                b->stubs[i].h_revision = (revision_t)nextp;
-                nextp = &b->stubs[i];
-            }
-            b++;
-        }
-        assert(nextp != NULL);
-        p = nextp;
-    }
-    pd->stub_free_list = (gcptr)p->h_revision;
-    assert(STUB_THREAD(p) == pd);
+    gcptr p = stmgcpage_malloc(sizeof(struct stm_stub_s));
+    STUB_THREAD(p) = pd;
     return p;
 }
 
diff --git a/c4/steal.h b/c4/steal.h
--- a/c4/steal.h
+++ b/c4/steal.h
@@ -2,10 +2,12 @@
 #define _SRCSTM_STEAL_H
 
 
-#define STUB_BLOCK_SIZE   (32 * WORD)    /* power of two */
+struct stm_stub_s {
+    struct stm_object_s s_header;
+    struct tx_public_descriptor *s_thread;
+};
 
-#define STUB_THREAD(h)    (*(struct tx_public_descriptor **)           \
-                            (((revision_t)(h)) & ~(STUB_BLOCK_SIZE-1)))
+#define STUB_THREAD(h)    (((struct stm_stub_s *)(h))->s_thread)
 
 gcptr stm_stub_malloc(struct tx_public_descriptor *);
 void stm_steal_stub(gcptr);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -100,7 +100,6 @@
     /* some constants normally private that are useful in the tests */
     #define WORD                     ...
     #define GC_PAGE_SIZE             ...
-    #define STUB_BLOCK_SIZE          ...
     #define GCFLAG_OLD               ...
     #define GCFLAG_VISITED           ...
     #define GCFLAG_PUBLIC            ...

From noreply at buildbot.pypy.org  Sat Jun 22 15:28:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 15:28:44 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix the test to avoid a
	thread-order-dependent failure.
Message-ID: <20130622132844.5D0A91C02D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r235:68143e7ceb28
Date: 2013-06-22 15:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/68143e7ceb28/

Log:	Fix the test to avoid a thread-order-dependent failure.

diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -58,6 +58,7 @@
     void stm_set_transaction_length(long length_max);
 
     /* extra non-public code */
+    void printfcolor(char *msg);
     void *stm_malloc(size_t size);
     gcptr stmgcpage_malloc(size_t size);
     void stmgcpage_free(gcptr obj);
@@ -122,6 +123,11 @@
     extern revision_t stm_global_cur_time(void);
     extern revision_t get_private_rev_num(void);
 
+    void printfcolor(char *msg)
+    {
+        fprintf(stderr, "%s\n", msg);
+    }
+
     int gettid(gcptr obj)
     {
         int result = stm_get_tid(obj);
@@ -329,6 +335,7 @@
             raise exc, val, tb
 
     def run(self, fn, lck, i):
+        printf('run: calling %r' % (fn,))
         try:
             try:
                 lib.stm_initialize_and_set_max_abort(self.max_aborts)
@@ -487,6 +494,8 @@
 def minor_collect():
     lib.stmgc_minor_collect()
 
+printf = lib.printfcolor
+
 def is_stub(p):
     return p.h_tid & GCFLAG_STUB
 
@@ -541,6 +550,8 @@
     def cb(_, retry_counter):
         del fine[:]
         try:
+            printf('perform_transaction: calling %r with retry_counter=%d' %
+                   (callback, retry_counter))
             loopback = callback(retry_counter)
         except Exception:
             if not sys.stdout.isatty():
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -341,6 +341,9 @@
             check_not_free(p1)
             assert classify(p1) == "private_from_protected"
             assert classify(follow_revision(p1)) == "public"  # has been stolen
+            # leave time for f2 to finish, before we commit changes to p1
+            # (which has got in its read set)
+            r.wait_while_in_parallel()
         perform_transaction(cb)
     def f2(r):
         def cb(c):
@@ -353,7 +356,9 @@
             r.leave_in_parallel()
             check_not_free(p2)
             assert classify(p2) == "public"
+            r.enter_in_parallel()
         perform_transaction(cb)
+        r.leave_in_parallel()
     run_parallel(f1, f2)
 
 def test_private_from_protected_inevitable():

From noreply at buildbot.pypy.org  Sat Jun 22 16:01:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 16:01:03 +0200 (CEST)
Subject: [pypy-commit] stmgc default: After a major GC,
 make sure the other threads abort if the major GC found
Message-ID: <20130622140103.5E6AF1C0212@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r236:531e5789c7dd
Date: 2013-06-22 15:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/531e5789c7dd/

Log:	After a major GC, make sure the other threads abort if the major GC
	found that they should.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -59,16 +59,23 @@
 
 static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
 
-static void inev_mutex_acquire(void)
+static void inev_mutex_release(void)
+{
+  pthread_mutex_unlock(&mutex_inevitable);
+}
+
+static void inev_mutex_acquire(struct tx_descriptor *d)
 {   /* must save roots around this call */
   stm_stop_sharedlock();
   pthread_mutex_lock(&mutex_inevitable);
   stm_start_sharedlock();
-}
 
-static void inev_mutex_release(void)
-{
-  pthread_mutex_unlock(&mutex_inevitable);
+  if (d->active < 0)
+    {
+      inev_mutex_release();
+      AbortNowIfDelayed();
+      abort();   /* unreachable */
+    }
 }
 
 /************************************************************/
@@ -882,6 +889,7 @@
 {
   assert(d->active == 0);
   stm_start_sharedlock();
+  assert(d->active == 0);
 
   if (clock_gettime(CLOCK_MONOTONIC, &d->start_real_time) < 0) {
     d->start_real_time.tv_nsec = -1;
@@ -1225,7 +1233,7 @@
             {                    // there is another inevitable transaction
               CancelLocks(d);
               spinlock_release(d->public_descriptor->collection_lock);
-              inev_mutex_acquire();   // wait until released
+              inev_mutex_acquire(d);   // wait until released
               inev_mutex_release();
               spinlock_acquire(d->public_descriptor->collection_lock, 'C');
               if (d->public_descriptor->stolen_objects.size != 0)
@@ -1284,11 +1292,12 @@
   update_reads_size_limit(d);
 }
 
-static revision_t acquire_inev_mutex_and_mark_global_cur_time(void)
+static revision_t acquire_inev_mutex_and_mark_global_cur_time(
+                      struct tx_descriptor *d)
 {   /* must save roots around this call */
   revision_t cur_time;
 
-  inev_mutex_acquire();
+  inev_mutex_acquire(d);
   while (1)
     {
       cur_time = ACCESS_ONCE(global_cur_time);
@@ -1312,7 +1321,7 @@
   fprintf(stderr, "[%lx] inevitable: %s\n",
           (long)d->public_descriptor_index, why);
 
-  cur_time = acquire_inev_mutex_and_mark_global_cur_time();
+  cur_time = acquire_inev_mutex_and_mark_global_cur_time(d);
   if (d->start_time != cur_time)
     {
       d->start_time = cur_time;
@@ -1332,7 +1341,7 @@
   revision_t cur_time;
 
   init_transaction(d);
-  cur_time = acquire_inev_mutex_and_mark_global_cur_time();
+  cur_time = acquire_inev_mutex_and_mark_global_cur_time(d);
   d->start_time = cur_time;
   make_inevitable(d);
 }
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -378,8 +378,8 @@
         if (IS_POINTER(v)) {
             /* has a more recent revision.  Oups. */
             fprintf(stderr,
-                    "ABRT_COLLECT_MAJOR: %p was read but modified already\n",
-                    obj);
+                    "ABRT_COLLECT_MAJOR %p: %p was read but modified already\n",
+                    d, obj);
             AbortTransactionAfterCollect(d, ABRT_COLLECT_MAJOR);
             return;
         }
@@ -410,6 +410,8 @@
      */
     wlog_t *item;
 
+    fprintf(stderr, "fix public_to_private on thread %p\n", d);
+
     G2L_LOOP_FORWARD(d->public_to_private, item) {
 
         assert(item->addr->h_tid & GCFLAG_PUBLIC);
@@ -419,6 +421,7 @@
                (item->val->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
 
         item->addr->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
+        fprintf(stderr, "\tpublic_to_private: %p\n", item->addr);
 
     } G2L_LOOP_END;
 
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -293,6 +293,8 @@
     /* another thread should be waiting in start_exclusivelock(),
        which takes priority here */
     stm_start_sharedlock();
+
+    AbortNowIfDelayed();   /* if another thread ran a major GC */
 }
 
 /************************************************************/
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -73,6 +73,7 @@
     void stm_start_sharedlock(void);
     void stm_stop_sharedlock(void);
     void AbortTransaction(int);
+    void AbortNowIfDelayed(void);
     gcptr stm_get_private_from_protected(long index);
     gcptr stm_get_read_obj(long index);
     void *STUB_THREAD(gcptr);
@@ -416,6 +417,7 @@
         lib.stm_stop_sharedlock()
         self.parallel_locks[1].acquire()
         lib.stm_start_sharedlock()
+        lib.AbortNowIfDelayed()
         print 'wait_while_in_parallel leave'
         # parallel_locks[0] is acquired, parallel_locks[1] is acquired
         self.parallel_locks[1].release()
@@ -500,9 +502,9 @@
     return p.h_tid & GCFLAG_STUB
 
 def check_not_free(p):
-    print >> sys.stderr, "[checking %r..." % p,
+    #print >> sys.stderr, "[checking %r..." % p,
     assert 42 < (p.h_tid & 0xFFFF) < 521
-    print >> sys.stderr, "ok]"
+    #print >> sys.stderr, "ok]"
 
 def check_nursery_free(p):
     #assert p.h_tid == p.h_revision == 0
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -5,7 +5,7 @@
 # a default seed that changes every day, but that can be easily recovered
 DEFAULT_SEED = int(time.strftime("%y%m%d", time.gmtime()))
 
-DO_MAJOR_COLLECTS = False
+DO_MAJOR_COLLECTS = True
 
 
 def setup_function(_):
@@ -452,6 +452,7 @@
         self.toggle = not threadid
         self.cond.notify()
         lib.stm_start_sharedlock()
+        lib.AbortNowIfDelayed()
 
     def wait1(self):
         self.wait(False)

From noreply at buildbot.pypy.org  Sat Jun 22 16:12:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 16:12:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix in the test: allow for other
 threads that want to abort us after
Message-ID: <20130622141252.6037C1C0212@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r237:511679409ba9
Date: 2013-06-22 16:12 +0200
http://bitbucket.org/pypy/stmgc/changeset/511679409ba9/

Log:	Fix in the test: allow for other threads that want to abort us after
	a major gc.

diff --git a/c4/test/model.py b/c4/test/model.py
--- a/c4/test/model.py
+++ b/c4/test/model.py
@@ -167,6 +167,12 @@
             assert self.previous is gs.most_recent_committed_revision
             gs.most_recent_committed_revision = self
 
+    def check_can_still_commit(self):
+        gs = self.globalstate
+        saved = self.previous
+        self._extend_timestamp(gs.most_recent_committed_revision)
+        self.previous = saved
+
 
 class GlobalState(object):
 
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -149,6 +149,20 @@
         #
         return x, y
 
+    def do_check_can_still_commit(self):
+        try:
+            self.current_rev.check_can_still_commit()
+        except (model.Deleted, model.Conflict), e:
+            # the model says that we might get an abort
+            self.dump('possible delayed abort!')
+            self.expected_abort()
+            lib.AbortNowIfDelayed()
+            # ok, it's fine if we don't actually get an abort
+            self.cancel_expected_abort()
+        else:
+            # the model says that we must not get an abort
+            lib.AbortNowIfDelayed()
+
     def get_ref(self, r, index):
         self.check(r)
         if r == emptypair:
@@ -356,6 +370,7 @@
             if do_wait:
                 self.push_roots(extra=p)
                 do_wait()
+                self.do_check_can_still_commit()
                 p = self.pop_roots(extra=p)
 
     def run_single_thread(self):
@@ -452,7 +467,6 @@
         self.toggle = not threadid
         self.cond.notify()
         lib.stm_start_sharedlock()
-        lib.AbortNowIfDelayed()
 
     def wait1(self):
         self.wait(False)
@@ -506,6 +520,6 @@
     test_multi_thread(1624)
 
 def test_more_multi_thread():
-    py.test.skip("more random tests")
+    #py.test.skip("more random tests")
     for i in range(200):
         yield test_multi_thread, 1624 + i

From noreply at buildbot.pypy.org  Sat Jun 22 17:55:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 17:55:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for issue1520.
Message-ID: <20130622155533.3FE851C13F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64955:30e24bcbd0e5
Date: 2013-06-22 17:54 +0200
http://bitbucket.org/pypy/pypy/changeset/30e24bcbd0e5/

Log:	Fix for issue1520.

diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -45,10 +45,7 @@
     raise NotImplementedError
 
 def resolvedirof(filename):
-    try:
-        filename = rpath.rabspath(filename)
-    except OSError:
-        pass
+    filename = rpath.rabspath(filename)
     dirname = rpath.rabspath(os.path.join(filename, '..'))
     if os.path.islink(filename):
         try:
diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py
--- a/rpython/rlib/rpath.py
+++ b/rpython/rlib/rpath.py
@@ -7,7 +7,12 @@
 
 if os.name == 'posix':
     # the posix version is already RPython, just use it
-    rabspath = os.path.abspath
+    # (but catch exceptions)
+    def rabspath(path):
+        try:
+            return os.path.abspath(path)
+        except OSError:
+            return path
 elif os.name == 'nt':
     def rabspath(path):
         if path == '':
diff --git a/rpython/rlib/test/test_rpath.py b/rpython/rlib/test/test_rpath.py
--- a/rpython/rlib/test/test_rpath.py
+++ b/rpython/rlib/test/test_rpath.py
@@ -12,6 +12,19 @@
 def test_rabspath_absolute_posix():
     assert rpath.rabspath('/foo') == '/foo'
 
+ at py.test.mark.skipif("IS_WINDOWS")
+def test_missing_current_dir(tmpdir):
+    tmpdir1 = str(tmpdir) + '/temporary_removed'
+    curdir1 = os.getcwd()
+    try:
+        os.mkdir(tmpdir1)
+        os.chdir(tmpdir1)
+        os.rmdir(tmpdir1)
+        result = rpath.rabspath('.')
+    finally:
+        os.chdir(curdir1)
+    assert result == '.'
+
 @py.test.mark.skipif("not IS_WINDOWS")
 def test_rabspath_absolute_nt():
     curdrive, _ = os.path.splitdrive(os.getcwd())

From noreply at buildbot.pypy.org  Sat Jun 22 18:24:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 18:24:27 +0200 (CEST)
Subject: [pypy-commit] stmgc default: typo
Message-ID: <20130622162427.5E3941C02D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r238:b7104acf55fd
Date: 2013-06-22 18:24 +0200
http://bitbucket.org/pypy/stmgc/changeset/b7104acf55fd/

Log:	typo

diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -31,7 +31,7 @@
 
 /* returns a never changing hash for the object */
 revision_t stm_hash(gcptr);
-/* returns an for the object which is unique during its lifetime */
+/* returns a number for the object which is unique during its lifetime */
 revision_t stm_id(gcptr);
 /* returns nonzero if the two object-copy pointers belong to the
 same original object */

From noreply at buildbot.pypy.org  Sat Jun 22 18:31:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 18:31:06 +0200 (CEST)
Subject: [pypy-commit] stmgc default: demo_random shows another livelock
 with this spinloop. Added it to the comment, revert again to always abort.
Message-ID: <20130622163106.93C8A1C02D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r239:fe90ba781e8a
Date: 2013-06-22 18:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/fe90ba781e8a/

Log:	demo_random shows another livelock with this spinloop. Added it to
	the comment, revert again to always abort.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -675,23 +675,23 @@
             {
               if (v != d->my_lock)         // not locked by me: conflict
                 {
-                  /* A case that can occur: two threads A and B are both
+                  /* It's delicate here to do a spinloop rather than
+                     just aborting.
+
+                     A case that can occur: two threads A and B are both
                      committing, thread A locked object a, thread B
                      locked object b, and then thread A tries to
                      validate the reads it did on object b and
                      vice-versa.  In this case both threads cannot
                      commit, but if they both enter the SpinLoop()
-                     above, then they will livelock.
+                     here, then they will livelock.
 
-                     But this might lead both threads to cancel by
-                     reaching this point.  For now we attempt to be
-                     more clever and let one of the threads commit
-                     anyway (the choice of which one looks random).
+                     Another case: thread A might be blocked in this
+                     spinloop, while thread B is blocked in the
+                     SpinLoop(SPLP_LOCKED_COMMIT) below.
+
+                     For now we always abort.
                   */
-                  if (d->my_lock < v) {
-                      SpinLoop(SPLP_LOCKED_VALIDATE);
-                      goto retry;
-                  }
                   fprintf(stderr, "validation failed: "
                           "%p is locked by another thread\n", R);
                   return 0;

From noreply at buildbot.pypy.org  Sat Jun 22 19:33:57 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 19:33:57 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next issue
Message-ID: <20130622173357.0B8B61C13F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r240:0a23ff34c542
Date: 2013-06-22 19:33 +0200
http://bitbucket.org/pypy/stmgc/changeset/0a23ff34c542/

Log:	Next issue

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -522,4 +522,4 @@
 def test_more_multi_thread():
     #py.test.skip("more random tests")
     for i in range(200):
-        yield test_multi_thread, 1624 + i
+        yield test_multi_thread, (3311//2) + i

From noreply at buildbot.pypy.org  Sat Jun 22 19:46:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 19:46:59 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130622174659.978CD1C14FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r241:ec718b479e29
Date: 2013-06-22 19:46 +0200
http://bitbucket.org/pypy/stmgc/changeset/ec718b479e29/

Log:	Fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1184,6 +1184,10 @@
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
+
+          if (P->h_original && !B->h_original) // might occur because of
+            B->h_original = P->h_original; //replace_ptr_to_protected_with_stub
+
           assert(!(P->h_original) 
                  || (B->h_original == (revision_t)P->h_original));
           size_t size = stmcb_size(B);

From noreply at buildbot.pypy.org  Sat Jun 22 19:53:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 19:53:06 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130622175306.C89201C14FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r242:f689f7d2a0b4
Date: 2013-06-22 19:52 +0200
http://bitbucket.org/pypy/stmgc/changeset/f689f7d2a0b4/

Log:	Fix

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1501,8 +1501,8 @@
           assert(descriptor_array_free_list >= 0);
           assert(pd->stolen_objects.size == 0);
           assert(pd->stolen_young_stubs.size == 0);
-          assert(pd->collection_lock == 0 || pd->collection_lock == -1);
-          pd->collection_lock = 0;
+          assert(pd->collection_lock == 0);
+          pd->shutdown = 0;
       }
       else {
           /* no item in the free list */
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -549,7 +549,7 @@
     revision_t index = -1;
 
     while ((gcp = stm_get_free_public_descriptor(&index)) != NULL) {
-        if (gcp->collection_lock == -1)
+        if (gcp->shutdown)
             continue;
 
         for (i = 1; i < GC_SMALL_REQUESTS; i++) {
@@ -557,7 +557,7 @@
             sweep_pages(gcp, i, gpage);
         }
         assert(gcp->collection_lock == 0);
-        gcp->collection_lock = -1;
+        gcp->shutdown = 1;
         /* XXX ...stub_blocks... */
         assert(gcp->stolen_objects.size == 0);
         assert(gcp->stolen_young_stubs.size == 0);
diff --git a/c4/gcpage.h b/c4/gcpage.h
--- a/c4/gcpage.h
+++ b/c4/gcpage.h
@@ -60,7 +60,12 @@
     gcptr free_loc_for_size[GC_SMALL_REQUESTS];                         \
                                                                         \
     /* For statistics */                                                \
-    size_t count_pages;
+    size_t count_pages;                                                 \
+                                                                        \
+    /* Set to 1 when a major collection shuts down a                    \
+       public_descriptor after the corresponding thread was             \
+       already finished */                                              \
+    int shutdown;
 
 
 #define LOCAL_GCPAGES()  (thread_descriptor->public_descriptor)

From noreply at buildbot.pypy.org  Sat Jun 22 19:54:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Jun 2013 19:54:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: This XXX is fine now,
 as stubs are no longer in specially allocated
Message-ID: <20130622175401.802851C14FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r243:f820ff645e44
Date: 2013-06-22 19:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/f820ff645e44/

Log:	This XXX is fine now, as stubs are no longer in specially allocated
	pieces of memory.

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -558,7 +558,6 @@
         }
         assert(gcp->collection_lock == 0);
         gcp->shutdown = 1;
-        /* XXX ...stub_blocks... */
         assert(gcp->stolen_objects.size == 0);
         assert(gcp->stolen_young_stubs.size == 0);
         gcptrlist_delete(&gcp->stolen_objects);

From noreply at buildbot.pypy.org  Sun Jun 23 11:31:02 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 23 Jun 2013 11:31:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: make shadowstack suggested and not
 required when translating for arm
Message-ID: <20130623093102.0C27C1C0619@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64956:5e10049c60ad
Date: 2013-06-22 14:52 -0500
http://bitbucket.org/pypy/pypy/changeset/5e10049c60ad/

Log:	make shadowstack suggested and not required when translating for arm

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -278,8 +278,8 @@
     ChoiceOption("platform",
                  "target platform", ['host'] + PLATFORMS, default='host',
                  cmdline='--platform',
-                 requires={"arm": [("translation.gcrootfinder", "shadowstack")]},
-                 suggests={"arm": [("translation.jit_backend", "arm")]}),
+                 suggests={"arm": [("translation.gcrootfinder", "shadowstack"),
+                                   ("translation.jit_backend", "arm")]}),
 
 ])
 

From noreply at buildbot.pypy.org  Sun Jun 23 11:31:04 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Sun, 23 Jun 2013 11:31:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: ctypes/_rawffi can generate unaligned
 access to floating point fields in
Message-ID: <20130623093104.D0F0E1C0619@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64957:162603593f00
Date: 2013-06-23 04:24 -0500
http://bitbucket.org/pypy/pypy/changeset/162603593f00/

Log:	ctypes/_rawffi can generate unaligned access to floating point
	fields in structures. Directly reading unaligned floats from memory
	is not supported on ARM. This change adds a wrapper around the reads
	and writes that checks the pointer alignment and uses memcpy in case
	of unaligned access on ARM.

diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -13,19 +13,9 @@
 from pypy.module._rawffi.interp_rawffi import TYPEMAP
 from pypy.module._rawffi.interp_rawffi import size_alignment
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
 
-def push_elem(ll_array, pos, value):
-    TP = lltype.typeOf(value)
-    ll_array = rffi.cast(rffi.CArrayPtr(TP), ll_array)
-    ll_array[pos] = value
-push_elem._annspecialcase_ = 'specialize:argtype(2)'
-
-def get_elem(ll_array, pos, ll_t):
-    ll_array = rffi.cast(rffi.CArrayPtr(ll_t), ll_array)
-    return ll_array[pos]
-get_elem._annspecialcase_ = 'specialize:arg(2)'
-
 
 class W_Array(W_DataShape):
     def __init__(self, basicffitype, size):
@@ -57,7 +47,7 @@
                                                 " array length"))
             for num in range(iterlength):
                 w_item = items_w[num]
-                unwrap_value(space, push_elem, result.ll_buffer, num,
+                unwrap_value(space, write_ptr, result.ll_buffer, num,
                              self.itemcode, w_item)
         return space.wrap(result)
 
@@ -118,7 +108,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, push_elem, self.ll_buffer, num,
+        unwrap_value(space, write_ptr, self.ll_buffer, num,
                      self.shape.itemcode, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
@@ -136,7 +126,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, get_elem, self.ll_buffer, num,
+        return wrap_value(space, read_ptr, self.ll_buffer, num,
                           self.shape.itemcode)
 
     def descr_getitem(self, space, w_index):
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._rawffi.array import push_elem
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from pypy.module._rawffi.structure import W_Structure
 from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp,
      unwrap_value, unpack_argshapes, got_libffi_error)
@@ -40,7 +40,7 @@
                 args_w[i] = space.wrap(rffi.cast(rffi.ULONG, ll_args[i]))
         w_res = space.call(w_callable, space.newtuple(args_w))
         if callback_ptr.result is not None: # don't return void
-            unwrap_value(space, push_elem, ll_res, 0,
+            unwrap_value(space, write_ptr, ll_res, 0,
                          callback_ptr.result, w_res)
     except OperationError, e:
         tbprint(space, space.wrap(e.get_traceback()),
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,7 @@
 
 from rpython.rlib.clibffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
 import rpython.rlib.rposix as rposix
 
@@ -43,6 +44,8 @@
 }
 TYPEMAP_PTR_LETTERS = "POszZ"
 TYPEMAP_NUMBER_LETTERS = "bBhHiIlLqQ?"
+TYPEMAP_FLOAT_LETTERS = "fd" # XXX long doubles are not propperly supported in
+                             # rpython, so we ignore then here
 
 if _MS_WINDOWS:
     TYPEMAP['X'] = ffi_type_pointer
@@ -239,6 +242,52 @@
 )
 
 unroll_letters_for_numbers = unrolling_iterable(TYPEMAP_NUMBER_LETTERS)
+unroll_letters_for_floats = unrolling_iterable(TYPEMAP_FLOAT_LETTERS)
+
+_ARM = rffi_platform.getdefined('__arm__', '')
+
+def read_ptr(ptr, ofs, TP):
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as t_array:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, t_array),
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.sizeof(TP))
+                ptr_val = t_array[0]
+                return ptr_val
+    else:
+        return rffi.cast(T, ptr)[ofs]
+read_ptr._annspecialcase_ = 'specialize:arg(2)'
+
+def write_ptr(ptr, ofs, value):
+    TP = lltype.typeOf(value)
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as s_array:
+                s_array[0] = value
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.cast(rffi.VOIDP, s_array),
+                    rffi.sizeof(TP))
+                return
+    else:
+        rffi.cast(T, ptr)[ofs] = value
+write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -12,8 +12,10 @@
 from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance
 from pypy.module._rawffi.interp_rawffi import wrap_value, unwrap_value
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
-from pypy.module._rawffi.interp_rawffi import size_alignment, LL_TYPEMAP
+from pypy.module._rawffi.interp_rawffi import LL_TYPEMAP
 from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers
+from pypy.module._rawffi.interp_rawffi import size_alignment
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib import clibffi
 from rpython.rlib.rarithmetic import intmask, signedtype, widen
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong
@@ -289,19 +291,18 @@
                 value = widen(value)
                 bitmask = BIT_MASK(numbits, TP)
                 #
-                current = widen(rffi.cast(T, ptr)[0])
+                current = widen(read_ptr(ptr, 0, TP))
                 current &= ~ (bitmask << lowbit)
                 current |= (value & bitmask) << lowbit
                 value = rffi.cast(TP, current)
             break
-
-    rffi.cast(T, ptr)[0] = value
+    write_ptr(ptr, 0, value)
 push_field._annspecialcase_ = 'specialize:argtype(2)'
 
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     TP = lltype.Ptr(rffi.CArray(ll_t))
-    value = rffi.cast(TP, pos)[0]
+    value = read_ptr(pos, 0, ll_t)
 
     # Handle bitfields
     for c in unroll_letters_for_numbers:
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -211,6 +211,7 @@
         cls.w_platform = space.wrap(platform.name)
         cls.w_sizes_and_alignments = space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
+        cls.w_typemap = space.wrap(TYPEMAP.keys())
 
     def test_libload(self):
         import _rawffi
@@ -751,6 +752,18 @@
             assert _rawffi.sizeof(k) == s
             assert _rawffi.alignment(k) == a
 
+    def test_unaligned(self):
+        import _rawffi
+        for k in self.typemap:
+            if k not in 'fdg':
+                continue
+            S = _rawffi.Structure([('pad', 'c'), ('value', k)], pack=1)
+            s = S()
+            s.value = 4
+            assert s.value == 4
+            s.free()
+
+
     def test_array_addressof(self):
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)

From noreply at buildbot.pypy.org  Sun Jun 23 11:53:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Jun 2013 11:53:23 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add http://bitbucket.org/arigo/duhton
Message-ID: <20130623095323.8B3E81C06BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r244:ee2e229895de
Date: 2013-06-23 10:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/ee2e229895de/

Log:	Add http://bitbucket.org/arigo/duhton

diff too long, truncating to 2000 out of 3401 lines

diff --git a/duhton/Makefile b/duhton/Makefile
new file mode 100644
--- /dev/null
+++ b/duhton/Makefile
@@ -0,0 +1,11 @@
+
+all: duhton_debug duhton
+
+duhton: *.c *.h stm/*.c stm/*.h
+	gcc -pthread -g -O2 -o duhton *.c stm/*.c -Wall
+
+duhton_debug: *.c *.h stm/*.c stm/*.h
+	gcc -pthread -g -DDu_DEBUG -o duhton_debug *.c stm/*.c -Wall
+
+clean:
+	rm -f duhton duhton_debug
diff --git a/duhton/PAPERS b/duhton/PAPERS
new file mode 100644
--- /dev/null
+++ b/duhton/PAPERS
@@ -0,0 +1,26 @@
+Papers
+======
+
+This is a list of papers I found that describe the general idea.
+
+* "Automatic Mutual Exclusion": the introduction paper at HotOS (may
+  2007) is interesting; the rest is far too much formalism with little
+  interest from my point of view.
+  http://research.microsoft.com/en-us/projects/ame/default.aspx
+
+* "Transactions Everywhere", 2002-2003: an early reference.
+
+* To read about the current status of STM research, which influences
+  the Hardware Transactional Memory on 2013 CPUs: "NOrec: Streamlining
+  STM by Abolishing Ownership Records".
+
+* "Grace": a system using Linux's paging (virtual memory) to emulate
+  an efficient STM.  Interesting idea but with probably a granularity
+  that is completely wrong for us --- one page, i.e. 4KB.  Two versions
+  of the same paper: http://plasma.cs.umass.edu/emery/grace  or
+  http://www.cs.umass.edu/~tingy/publications/grace.ps .
+
+* "OCM": http://ocm.dreamhosters.com/ .  A similar goal to the one here,
+  but modelling cooperative multithreading with an explicit "yield".
+  Also, presents two versions of Lua with different implementations
+  exposing the "yield" primitive.
diff --git a/duhton/README b/duhton/README
new file mode 100644
--- /dev/null
+++ b/duhton/README
@@ -0,0 +1,18 @@
+======
+Duhton
+======
+
+A minimalistic Python-like variant.  Written in the same style as
+CPython's C code (with "Du_" instead of "Py_" as a prefix to all names).
+
+Pronounciation: "duh"-ton.
+
+To compile, use "make".  Only tested with gcc on Linux, though it should
+work on other platforms too --- but so far, it requires the gcc-only
+extension "__thread".
+
+To run the tests, you need "py.test": http://pytest.org/
+
+There are demos: try "time duhton demo/many_square_roots.duh".
+
+For more general information see the PAPERS file.
diff --git a/duhton/ceval.c b/duhton/ceval.c
new file mode 100644
--- /dev/null
+++ b/duhton/ceval.c
@@ -0,0 +1,14 @@
+#include "duhton.h"
+
+
+DuObject *Du_Eval(DuObject *ob, DuObject *locals)
+{
+    eval_fn fn = ob->ob_type->dt_eval;
+    if (fn) {
+        return fn(ob, locals);
+    }
+    else {
+        Du_INCREF(ob);
+        return ob;
+    }
+}
diff --git a/duhton/compile.c b/duhton/compile.c
new file mode 100644
--- /dev/null
+++ b/duhton/compile.c
@@ -0,0 +1,110 @@
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include "duhton.h"
+
+
+DuObject *_Du_Parse(FILE *f, int level, int stop_after_newline)
+{
+    DuObject *cons, *list;
+    int c, i;
+
+    list = DuList_New();
+    if (level == 0) {
+        DuObject *item = DuSymbol_FromString("progn");
+        DuList_Append(list, item);
+        Du_DECREF(item);
+    }
+    c = fgetc(f);
+    while (1) {
+        DuObject *item;
+
+        switch (c) {
+
+        case EOF:
+            if (level > 0)
+                Du_FatalError("more '(' than ')'");
+            if (stop_after_newline) {
+                Du_DECREF(list);
+                return NULL;
+            }
+            goto done;
+
+        case '(':
+            item = _Du_Parse(f, level + 1, 0);
+            c = fgetc(f);
+            break;
+
+        case ')':
+            if (level == 0)
+                Du_FatalError("more ')' than '('");
+            goto done;
+
+        case '\n':
+            if (stop_after_newline)
+                goto done;
+            c = fgetc(f);
+            continue;
+
+        case ';':
+            while (c != '\n' && c != EOF)
+                c = fgetc(f);
+            continue;
+
+        default:
+            if (isspace(c)) {
+                c = fgetc(f);
+                continue;
+            }
+            else {
+                char token[201];
+                char *p = token;
+                char *end;
+                int number;
+                do {
+                    *p++ = c;
+                    c = fgetc(f);
+                } while (!(isspace(c) || c == '(' || c == ')' || c == EOF));
+                *p = '\0';
+                number = strtol(token, &end, 0);
+                if (*end == '\0') {
+                    item = DuInt_FromInt(number);
+                }
+                else {
+                    item = DuSymbol_FromString(token);
+                }
+                break;
+            }
+        }
+        DuList_Append(list, item);
+        Du_DECREF(item);
+    }
+
+ done:
+    Du_INCREF(Du_None);
+    cons = Du_None;
+    for (i = DuList_Size(list) - 1; i >= 0; i--) {
+        DuObject *item = DuList_GetItem(list, i);
+        DuObject *newcons = DuCons_New(item, cons);
+        Du_DECREF(cons);
+        Du_DECREF(item);
+        cons = newcons;
+    }
+    Du_DECREF(list);
+    return cons;
+}
+
+
+DuObject *Du_Compile(char *filename, int stop_after_newline)
+{
+    FILE *f;
+    if (strcmp(filename, "-") == 0)
+        f = stdin;
+    else
+        f = fopen(filename, "r");
+    if (!f) Du_FatalError("cannot open '%s'", filename);
+    DuObject *cons = _Du_Parse(f, 0, stop_after_newline);
+    if (f != stdin)
+        fclose(f);
+    return cons;
+}
diff --git a/duhton/consobject.c b/duhton/consobject.c
new file mode 100644
--- /dev/null
+++ b/duhton/consobject.c
@@ -0,0 +1,97 @@
+#include "duhton.h"
+
+typedef struct {
+    DuOBJECT_HEAD
+    DuObject *car, *cdr;
+} DuConsObject;
+
+void cons_free(DuConsObject *ob)
+{
+    Du_DECREF(ob->car);
+    Du_DECREF(ob->cdr);
+#ifdef Du_DEBUG
+    ob->car = ob->cdr = (DuObject *)0xDD;
+#endif
+    free(ob);
+}
+
+void cons_print(DuConsObject *ob)
+{
+    DuObject *p;
+    printf("( ");
+    while (1) {
+        Du_Print(ob->car, 0);
+        p = ob->cdr;
+        if (!DuCons_Check(p))
+            break;
+        ob = (DuConsObject *)p;
+        printf(" ");
+    }
+    if (p != Du_None) {
+        printf(" . ");
+        Du_Print(p, 0);
+    }
+    printf(" )");
+}
+
+
+
+DuObject *cons_eval(DuConsObject *ob, DuObject *locals)
+{
+    return _DuFrame_EvalCall(locals, ob->car, ob->cdr, 1);
+}
+
+DuTypeObject DuCons_Type = {
+    DuOBJECT_HEAD_INIT(&DuType_Type),
+    "cons",
+    sizeof(DuConsObject),
+    (destructor_fn)cons_free,
+    (print_fn)cons_print,
+    (eval_fn)cons_eval,
+};
+
+DuObject *DuCons_New(DuObject *car, DuObject *cdr)
+{
+    DuConsObject *ob = (DuConsObject *)DuObject_New(&DuCons_Type);
+    ob->car = car; Du_INCREF(car);
+    ob->cdr = cdr; Du_INCREF(cdr);
+    return (DuObject *)ob;
+}
+
+DuObject *DuCons_Car(DuObject *cons)
+{
+    DuCons_Ensure("DuCons_Car", cons);
+    DuObject *res = ((DuConsObject *)cons)->car;
+    Du_INCREF(res);
+    return res;
+}
+
+DuObject *DuCons_Cdr(DuObject *cons)
+{
+    DuCons_Ensure("DuCons_Cdr", cons);
+    DuObject *res = ((DuConsObject *)cons)->cdr;
+    Du_INCREF(res);
+    return res;
+}
+
+DuObject *_DuCons_CAR(DuObject *cons)
+{
+    assert(DuCons_Check(cons));
+    return ((DuConsObject *)cons)->car;
+}
+
+DuObject *_DuCons_NEXT(DuObject *cons)
+{
+    assert(DuCons_Check(cons));
+    DuObject *result = ((DuConsObject *)cons)->cdr;
+    if (result != Du_None && !DuCons_Check(cons))
+        Du_FatalError("_DuCons_NEXT: not a well-formed cons list");
+    return result;
+}
+
+void DuCons_Ensure(char *where, DuObject *ob)
+{
+    if (!DuCons_Check(ob))
+        Du_FatalError("%s: expected 'cons' argument, got '%s'",
+                      where, ob->ob_type->dt_name);
+}
diff --git a/duhton/containerobject.c b/duhton/containerobject.c
new file mode 100644
--- /dev/null
+++ b/duhton/containerobject.c
@@ -0,0 +1,69 @@
+#include "duhton.h"
+
+typedef struct {
+    DuOBJECT_HEAD
+    DuObject *ob_reference;
+} DuContainerObject;
+
+void container_free(DuContainerObject *ob)
+{
+    DuObject *x = ob->ob_reference;
+#ifdef Du_DEBUG
+    ob->ob_reference = (DuObject *)0xDD;
+#endif
+    free(ob);
+    Du_DECREF(x);
+}
+
+void container_print(DuContainerObject *ob)
+{
+    printf("<container ");
+    Du_Print(ob->ob_reference, 0);
+    printf(">");
+}
+
+DuObject *DuContainer_GetRef(DuObject *ob)
+{
+    DuObject *result;
+    DuContainer_Ensure("DuContainer_GetRef", ob);
+
+    Du_AME_READ(ob, (result = ((DuContainerObject *)ob)->ob_reference));
+
+    Du_INCREF(result);
+    return result;
+}
+
+void DuContainer_SetRef(DuObject *ob, DuObject *x)
+{
+    DuContainer_Ensure("DuContainer_SetRef", ob);
+
+    Du_AME_WRITE(ob);
+    DuObject *prev = ((DuContainerObject *)ob)->ob_reference;
+    Du_INCREF(x);
+    ((DuContainerObject *)ob)->ob_reference = x;
+    Du_DECREF(prev);
+}
+
+DuTypeObject DuContainer_Type = {
+    DuOBJECT_HEAD_INIT(&DuType_Type),
+    "container",
+    sizeof(DuContainerObject),
+    (destructor_fn)container_free,
+    (print_fn)container_print,
+};
+
+DuObject *DuContainer_New()
+{
+    DuContainerObject *ob =                                     \
+        (DuContainerObject *)DuObject_New(&DuContainer_Type);
+    Du_INCREF(Du_None);
+    ob->ob_reference = Du_None;
+    return (DuObject *)ob;
+}
+
+void DuContainer_Ensure(char *where, DuObject *ob)
+{
+    if (!DuContainer_Check(ob))
+        Du_FatalError("%s: expected 'container' argument, got '%s'",
+                      where, ob->ob_type->dt_name);
+}
diff --git a/duhton/demo/container_transaction.duh b/duhton/demo/container_transaction.duh
new file mode 100644
--- /dev/null
+++ b/duhton/demo/container_transaction.duh
@@ -0,0 +1,19 @@
+
+(setq c (container 0))
+
+(defun g (thread n)
+    (set c (+ (get c) 1))
+    (if (> (get c) 2000)
+        (print (quote overflow) (get c))
+      (if (< n 1000)
+          (transaction f thread (+ n 1))
+        (if (< (get c) 2000)
+            (print (quote not-enough))
+          (print (quote ok))))))
+
+(defun f (thread n)
+    (print (quote <) thread n (quote >))
+    (g thread n))
+
+(transaction f (quote t1) 1)
+(transaction f (quote t2) 1)
diff --git a/duhton/demo/list_transaction.duh b/duhton/demo/list_transaction.duh
new file mode 100644
--- /dev/null
+++ b/duhton/demo/list_transaction.duh
@@ -0,0 +1,17 @@
+
+(setq lst (list 0))
+(defun g (n)
+    (set lst 0 (+ (get lst 0) 1))
+    (if (< n 10)
+        (transaction f (+ n 1))
+      (sleepms 20)
+      (if (< (get lst 0) 20)
+          (print (quote not-enough))
+        (print (quote ok)))))
+(defun f (n)
+  (if (defined? marker)
+      (print (quote marker-already-defined)))
+  (setq marker 0)
+  (g n))
+(transaction f 1)
+(transaction f 1)
diff --git a/duhton/demo/many_square_roots.duh b/duhton/demo/many_square_roots.duh
new file mode 100644
--- /dev/null
+++ b/duhton/demo/many_square_roots.duh
@@ -0,0 +1,17 @@
+
+
+(defun square-root (n)
+  (setq i 0)
+  (while (< (* i i) n)
+    (setq i (+ i 1)))
+  i)
+
+(defun show-square-root (n)
+  (setq s (square-root n))
+  (print (quote square-root-of) n (quote is) s))
+
+
+(setq n 0)
+(while (< n 200)
+  (transaction show-square-root (+ 1000000000 (* n 100000)))
+  (setq n (+ n 1)))
diff --git a/duhton/demo/minimal.duh b/duhton/demo/minimal.duh
new file mode 100644
--- /dev/null
+++ b/duhton/demo/minimal.duh
@@ -0,0 +1,3 @@
+
+(print (+ 40 2))
+(print (- 45 3))
diff --git a/duhton/demo/simple_transaction.duh b/duhton/demo/simple_transaction.duh
new file mode 100644
--- /dev/null
+++ b/duhton/demo/simple_transaction.duh
@@ -0,0 +1,10 @@
+
+
+(defun do_stuff (i)
+  (if (> (* i i) 10)
+    (print i)))
+
+(setq i 0)
+(while (< i 10)
+  (transaction do_stuff i)
+  (setq i (+ i 1)))
diff --git a/duhton/demo/square_root.duh b/duhton/demo/square_root.duh
new file mode 100644
--- /dev/null
+++ b/duhton/demo/square_root.duh
@@ -0,0 +1,9 @@
+
+
+(defun square_root (n)
+  (setq i 0)
+  (while (< (* i i) n)
+    (setq i (+ i 1)))
+  i)
+
+(print (square_root 1000000000))
diff --git a/duhton/duhton.c b/duhton/duhton.c
new file mode 100644
--- /dev/null
+++ b/duhton/duhton.c
@@ -0,0 +1,44 @@
+#include "duhton.h"
+
+
+int main(int argc, char **argv)
+{
+    char *filename;
+    int interactive;
+    if (argc <= 1) {
+        filename = "-";   /* stdin */
+        interactive = 1;
+    }
+    else {
+        filename = argv[1];
+        interactive = 0;
+    }
+
+    Du_Initialize();
+
+    while (1) {
+        if (interactive) {
+            printf("))) ");
+            fflush(stdout);
+        }
+        DuObject *code = Du_Compile(filename, interactive);
+        if (code == NULL) {
+            printf("\n");
+            break;
+        }
+        /*Du_Print(code, 1);
+          printf("\n");*/
+        DuObject *res = Du_Eval(code, Du_Globals);
+        if (interactive) {
+            Du_Print(res, 1);
+        }
+        Du_DECREF(res);
+        Du_DECREF(code);
+        Du_TransactionRun();
+        if (!interactive)
+            break;
+    }
+
+    Du_Finalize();
+    return 0;
+}
diff --git a/duhton/duhton.h b/duhton/duhton.h
new file mode 100644
--- /dev/null
+++ b/duhton/duhton.h
@@ -0,0 +1,184 @@
+#ifndef _DUHTON_H_
+#define _DUHTON_H_
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+
+#define Du_AME     /* must always be on for now */
+
+#if defined(Du_DEBUG) || defined(Du_AME)
+#  define Du_TRACK_REFS
+#endif
+
+#ifdef Du_AME
+typedef long owner_version_t;
+#endif
+
+
+typedef struct _DuObject {
+    int ob_refcnt;
+    struct _DuTypeObject *ob_type;
+#ifdef Du_TRACK_REFS
+    struct _DuObject *ob_debug_prev, *ob_debug_next;
+#endif
+#ifdef Du_AME
+    owner_version_t ob_version;
+#endif
+} DuObject;
+
+#ifdef Du_TRACK_REFS
+#  define _DuObject_HEAD_EXTRA  NULL, NULL
+#else
+#  define _DuObject_HEAD_EXTRA  /* nothing */
+#endif
+
+#define DuOBJECT_HEAD   DuObject ob_base;
+
+#define DuOBJECT_HEAD_INIT(type)   { -1, type, _DuObject_HEAD_EXTRA }
+
+
+#ifdef __GNUC__
+#  define NORETURN  __attribute__((noreturn))
+#else
+#  define NORETURN  /* nothing */
+#endif
+
+
+typedef void(*destructor_fn)(DuObject *);
+typedef void(*print_fn)(DuObject *);
+typedef DuObject *(*eval_fn)(DuObject *, DuObject *);
+typedef int(*len_fn)(DuObject *);
+typedef void(*ame_copy_fn)(DuObject *);
+
+typedef struct _DuTypeObject {
+    DuOBJECT_HEAD
+    const char *dt_name;
+    int dt_size;
+    destructor_fn dt_destructor;
+    print_fn dt_print;
+    eval_fn dt_eval;
+    len_fn dt_is_true;
+    len_fn dt_length;
+    ame_copy_fn dt_ame_copy;
+} DuTypeObject;
+
+DuObject *DuObject_New(DuTypeObject *tp);
+void _Du_Dealloc(DuObject *ob);
+int DuObject_IsTrue(DuObject *ob);
+int DuObject_Length(DuObject *ob);
+
+#ifdef Du_TRACK_REFS
+void _Du_NewReference(DuObject *ob);
+void _Du_ForgetReference(DuObject *ob);
+#else
+#define _Du_NewReference(ob)       /* nothing */
+#define _Du_ForgetReference(ob)    /* nothing */
+#endif
+void _Du_BecomeImmortal(DuObject *ob);
+
+
+#define Du_AME_GLOBAL(ob)                       \
+    (((DuObject*)(ob))->ob_refcnt < 0)
+
+#define Du_INCREF(ob)                           \
+    do {                                        \
+        if (!Du_AME_GLOBAL(ob))                 \
+            ++((DuObject*)(ob))->ob_refcnt;     \
+    } while (0)
+
+#define Du_DECREF(ob)                                   \
+    do {                                                \
+        if (((DuObject*)(ob))->ob_refcnt > 1)           \
+            --((DuObject*)(ob))->ob_refcnt;             \
+        else if (!Du_AME_GLOBAL(ob))                    \
+            _Du_Dealloc((DuObject*)(ob));               \
+    } while (0)
+
+
+extern DuObject _Du_NoneStruct;
+#define Du_None (&_Du_NoneStruct)
+
+extern DuTypeObject DuType_Type;
+extern DuTypeObject DuInt_Type;
+extern DuTypeObject DuList_Type;
+extern DuTypeObject DuContainer_Type;
+extern DuTypeObject DuCons_Type;
+extern DuTypeObject DuSymbol_Type;
+extern DuTypeObject DuFrame_Type;
+
+#define DuType_Check(ob)      (((DuObject*)(ob))->ob_type == &DuType_Type)
+#define DuInt_Check(ob)       (((DuObject*)(ob))->ob_type == &DuInt_Type)
+#define DuList_Check(ob)      (((DuObject*)(ob))->ob_type == &DuList_Type)
+#define DuContainer_Check(ob) (((DuObject*)(ob))->ob_type == &DuContainer_Type)
+#define DuCons_Check(ob)      (((DuObject*)(ob))->ob_type == &DuCons_Type)
+#define DuSymbol_Check(ob)    (((DuObject*)(ob))->ob_type == &DuSymbol_Type)
+#define DuFrame_Check(ob)     (((DuObject*)(ob))->ob_type == &DuFrame_Type)
+
+void DuType_Ensure(char *where, DuObject *ob);
+void DuInt_Ensure(char *where, DuObject *ob);
+void DuList_Ensure(char *where, DuObject *ob);
+void DuContainer_Ensure(char *where, DuObject *ob);
+void DuCons_Ensure(char *where, DuObject *ob);
+void DuSymbol_Ensure(char *where, DuObject *ob);
+void DuFrame_Ensure(char *where, DuObject *ob);
+
+DuObject *DuInt_FromInt(int value);
+int DuInt_AsInt(DuObject *ob);
+
+DuObject *DuList_New(void);
+void DuList_Append(DuObject *list, DuObject *item);
+int DuList_Size(DuObject *list);
+DuObject *DuList_GetItem(DuObject *list, int index);
+void DuList_SetItem(DuObject *list, int index, DuObject *newobj);
+DuObject *DuList_Pop(DuObject *list, int index);
+
+DuObject *DuContainer_New(void);
+DuObject *DuContainer_GetRef(DuObject *container);
+void DuContainer_SetRef(DuObject *container, DuObject *newobj);
+
+DuObject *DuSymbol_FromString(char *name);
+char *DuSymbol_AsString(DuObject *ob);
+
+DuObject *DuCons_New(DuObject *car, DuObject *cdr);
+DuObject *DuCons_Car(DuObject *cons);
+DuObject *DuCons_Cdr(DuObject *cons);
+DuObject *_DuCons_CAR(DuObject *cons);
+DuObject *_DuCons_NEXT(DuObject *cons);
+
+void Du_FatalError(char *msg, ...) NORETURN;
+DuObject *Du_Compile(char *filename, int stop_after_newline);
+void Du_Print(DuObject *ob, int newline);
+
+DuObject *Du_Eval(DuObject *ob, DuObject *locals);
+DuObject *Du_Progn(DuObject *cons, DuObject *locals);
+
+DuObject *DuFrame_New();
+DuObject *DuFrame_Copy(DuObject *frame);
+DuObject *DuFrame_GetSymbol(DuObject *frame, DuObject *symbol);
+void DuFrame_SetSymbol(DuObject *frame, DuObject *symbol, DuObject *value);
+void DuFrame_SetSymbolStr(DuObject *frame, char *name, DuObject *value);
+void DuFrame_SetBuiltinMacro(DuObject *frame, char *name, eval_fn func);
+void DuFrame_SetUserFunction(DuObject *frame, DuObject *symbol,
+                             DuObject *arglist, DuObject *progn);
+DuObject *_DuFrame_EvalCall(DuObject *frame, DuObject *symbol,
+                            DuObject *rest, int execute_now);
+
+void Du_Initialize(void);
+void Du_Finalize(void);
+void _Du_InitializeObjects(void);
+void _Du_FinalizeObjects(void);
+#ifdef Du_TRACK_REFS
+void _Du_MakeImmortal(void);
+#endif
+extern DuObject *Du_Globals;
+
+void Du_TransactionAdd(DuObject *code, DuObject *frame);
+void Du_TransactionRun(void);
+
+
+#include "stm/ame.h"
+
+#endif  /* _DUHTON_H_ */
diff --git a/duhton/frame.c b/duhton/frame.c
new file mode 100644
--- /dev/null
+++ b/duhton/frame.c
@@ -0,0 +1,235 @@
+#include "duhton.h"
+#include <stdint.h>
+
+struct dictentry {
+    DuObject *symbol;
+    DuObject *value;
+    eval_fn builtin_macro;
+    DuObject *func_arglist;
+    DuObject *func_progn;
+};
+
+typedef struct {
+    DuOBJECT_HEAD
+    int entry_count;
+    struct dictentry *entries;
+} DuFrameObject;
+
+DuObject *DuFrame_New()
+{
+    DuFrameObject *ob = (DuFrameObject *)DuObject_New(&DuFrame_Type);
+    ob->entry_count = 0;
+    ob->entries = NULL;
+    return (DuObject *)ob;
+}
+
+DuObject *DuFrame_Copy(DuObject *frame)
+{
+    DuFrame_Ensure("DuFrame_Copy", frame);
+    int i;
+    DuFrameObject *src = (DuFrameObject *)frame;
+    DuFrameObject *dst = (DuFrameObject *)DuFrame_New();
+    dst->entry_count = src->entry_count;
+    dst->entries = malloc(sizeof(struct dictentry) * src->entry_count);
+    assert(dst->entries);
+    for (i=0; i<src->entry_count; i++) {
+        struct dictentry *e = &src->entries[i];
+        Du_INCREF(e->symbol);
+        if (e->value        != NULL) Du_INCREF(e->value       );
+        if (e->func_arglist != NULL) Du_INCREF(e->func_arglist);
+        if (e->func_progn   != NULL) Du_INCREF(e->func_progn  );
+        dst->entries[i] = *e;
+    }
+    return (DuObject *)dst;
+}
+
+void frame_free(DuFrameObject *ob)
+{
+    int i;
+    for (i=0; i<ob->entry_count; i++) {
+        struct dictentry *e = &ob->entries[i];
+        Du_DECREF(e->symbol);
+        if (e->value        != NULL) Du_DECREF(e->value       );
+        if (e->func_arglist != NULL) Du_DECREF(e->func_arglist);
+        if (e->func_progn   != NULL) Du_DECREF(e->func_progn  );
+    }
+    free(ob->entries);
+    free(ob);
+}
+
+void frame_print(DuFrameObject *ob)
+{
+    printf("<frame>");
+}
+
+static struct dictentry *
+find_entry(DuFrameObject *frame, DuObject *symbol, int add_if_missing)
+{
+    int left = 0;
+    int right = frame->entry_count;
+    struct dictentry *entries = frame->entries;
+    while (right > left) {
+        int middle = (left + right) / 2;
+        DuObject *found = entries[middle].symbol;
+        if ((intptr_t)found < (intptr_t)symbol)
+            right = middle;
+        else if (found == symbol)
+            return entries + middle;
+        else
+            left = middle + 1;
+    }
+    if (add_if_missing) {
+        int i;
+        int newcount = frame->entry_count + 1;
+        struct dictentry *newentries = malloc(sizeof(struct dictentry) *
+                                              newcount);
+        for (i=0; i<left; i++)
+            newentries[i] = entries[i];
+        DuSymbol_Ensure("find_entry", symbol);
+        newentries[left].symbol = symbol; Du_INCREF(symbol);
+        newentries[left].value = NULL;
+        newentries[left].builtin_macro = NULL;
+        newentries[left].func_arglist = NULL;
+        newentries[left].func_progn = NULL;
+        for (i=left+1; i<newcount; i++)
+            newentries[i] = entries[i-1];
+        frame->entries = newentries;
+        frame->entry_count = newcount;
+        free(entries);
+        return newentries + left;
+    }
+    else
+        return NULL;
+}
+
+void DuFrame_SetBuiltinMacro(DuObject *frame, char *name, eval_fn func)
+{
+    DuFrame_Ensure("DuFrame_SetBuiltinMacro", frame);
+    DuObject *sym = DuSymbol_FromString(name);
+    struct dictentry *e = find_entry((DuFrameObject *)frame, sym, 1);
+    e->builtin_macro = func;
+    Du_DECREF(sym);
+}
+
+static void
+_parse_arguments(DuObject *symbol, DuObject *arguments,
+                 DuObject *formallist, DuObject *caller, DuObject *callee)
+{
+    while (DuCons_Check(formallist)) {
+        if (!DuCons_Check(arguments))
+            Du_FatalError("call to '%s': not enough arguments",
+                          DuSymbol_AsString(symbol));
+        DuObject *sym = _DuCons_CAR(formallist);
+        DuObject *obj = Du_Eval(_DuCons_CAR(arguments), caller);
+        DuFrame_SetSymbol(callee, sym, obj);
+        Du_DECREF(obj);
+        formallist = _DuCons_NEXT(formallist);
+        arguments = _DuCons_NEXT(arguments);
+    }
+    if (arguments != Du_None)
+        Du_FatalError("call to '%s': too many arguments",
+                      DuSymbol_AsString(symbol));
+}
+
+DuObject *_DuFrame_EvalCall(DuObject *frame, DuObject *symbol,
+                            DuObject *rest, int execute_now)
+{
+    struct dictentry *e;
+    DuFrame_Ensure("_DuFrame_EvalCall", frame);
+
+    e = find_entry((DuFrameObject *)frame, symbol, 0);
+    if (!e) {
+        e = find_entry((DuFrameObject *)Du_Globals, symbol, 0);
+        if (!e) {
+            if (!DuSymbol_Check(symbol)) {
+                printf("_DuFrame_EvalCall: ");
+                Du_Print(symbol, 1);
+                Du_FatalError("expected a symbol to execute");
+            }
+            else
+                goto not_defined;
+        }
+    }
+    if (e->func_progn) {
+        DuObject *callee_frame = DuFrame_New();
+        DuObject *res;
+        _parse_arguments(symbol, rest, e->func_arglist, frame, callee_frame);
+        if (execute_now) {
+            res = Du_Progn(e->func_progn, callee_frame);
+        }
+        else {
+            Du_TransactionAdd(e->func_progn, callee_frame);
+            res = NULL;
+        }
+        Du_DECREF(callee_frame);
+        return res;
+    }
+    if (e->builtin_macro) {
+        if (!execute_now)
+            Du_FatalError("symbol refers to a macro: '%s'",
+                          DuSymbol_AsString(symbol));
+        return e->builtin_macro(rest, frame);
+    }
+ not_defined:
+    Du_FatalError("symbol not defined as a function: '%s'",
+                  DuSymbol_AsString(symbol));
+}
+
+DuObject *DuFrame_GetSymbol(DuObject *frame, DuObject *symbol)
+{
+    struct dictentry *e;
+    DuFrame_Ensure("DuFrame_GetSymbol", frame);
+
+    e = find_entry((DuFrameObject *)frame, symbol, 0);
+    if (e && e->value) {
+        Du_INCREF(e->value);
+        return e->value;
+    }
+    else
+        return NULL;
+}
+
+void DuFrame_SetSymbol(DuObject *frame, DuObject *symbol, DuObject *value)
+{
+    struct dictentry *e;
+    DuFrame_Ensure("DuFrame_SetSymbol", frame);
+
+    e = find_entry((DuFrameObject *)frame, symbol, 1);
+    if (e->value) Du_DECREF(e->value);
+    e->value = value; Du_INCREF(value);
+}
+
+void DuFrame_SetSymbolStr(DuObject *frame, char *name, DuObject *value)
+{
+    DuObject *sym = DuSymbol_FromString(name);
+    DuFrame_SetSymbol(frame, sym, value);
+    Du_DECREF(sym);
+}
+
+void DuFrame_SetUserFunction(DuObject *frame, DuObject *symbol,
+                             DuObject *arglist, DuObject *progn)
+{
+    struct dictentry *e;
+    DuFrame_Ensure("DuFrame_SetUserFunction", frame);
+
+    e = find_entry((DuFrameObject *)frame, symbol, 1);
+    if (e->func_arglist) Du_DECREF(e->func_arglist);
+    if (e->func_progn)   Du_DECREF(e->func_progn);
+    e->func_arglist = arglist; Du_INCREF(arglist);
+    e->func_progn   = progn;   Du_INCREF(progn);
+}
+
+void DuFrame_Ensure(char *where, DuObject *ob)
+{
+    if (!DuFrame_Check(ob))
+        Du_FatalError("%s: expected 'frame' argument, got '%s'",
+                      where, ob->ob_type->dt_name);
+}
+
+DuTypeObject DuFrame_Type = {
+    DuOBJECT_HEAD_INIT(&DuType_Type),
+    "frame",
+    sizeof(DuFrameObject),
+    (destructor_fn)frame_free,
+    (print_fn)frame_print,
+};
diff --git a/duhton/glob.c b/duhton/glob.c
new file mode 100644
--- /dev/null
+++ b/duhton/glob.c
@@ -0,0 +1,469 @@
+#include "duhton.h"
+#include <sys/select.h>
+
+
+DuObject *Du_Progn(DuObject *cons, DuObject *locals)
+{
+    DuObject *result = Du_None;
+    Du_INCREF(Du_None);
+    while (cons != Du_None) {
+        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        Du_DECREF(result);
+        result = obj;
+        cons = _DuCons_NEXT(cons);
+    }
+    return result;
+}
+
+DuObject *du_setq(DuObject *cons, DuObject *locals)
+{
+    DuObject *result = Du_None;
+    Du_INCREF(Du_None);
+    while (cons != Du_None) {
+        DuObject *symbol = _DuCons_CAR(cons);
+        cons = _DuCons_NEXT(cons);
+        if (cons == Du_None)
+            Du_FatalError("setq: number of arguments is odd");
+        DuObject *expr = _DuCons_CAR(cons);
+        DuObject *obj = Du_Eval(expr, locals);
+        DuFrame_SetSymbol(locals, symbol, obj);
+        Du_DECREF(result);
+        result = obj;
+        cons = _DuCons_NEXT(cons);
+    }
+    return result;
+}
+
+DuObject *du_print(DuObject *cons, DuObject *locals)
+{
+    DuObject *lst = DuList_New();
+
+    while (cons != Du_None) {
+        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        DuList_Append(lst, obj);
+        Du_DECREF(obj);
+        cons = _DuCons_NEXT(cons);
+    }
+
+    Du_AME_TryInevitable();
+
+    int i;
+    for (i=0; i<DuList_Size(lst); i++) {
+        if (i > 0) printf(" ");
+        DuObject *obj = DuList_GetItem(lst, i);
+        Du_Print(obj, 0);
+        Du_DECREF(obj);
+    }
+    Du_DECREF(lst);
+
+    printf("\n");
+    Du_INCREF(Du_None);
+    return Du_None;
+}
+
+DuObject *du_add(DuObject *cons, DuObject *locals)
+{
+    int result = 0;
+    while (cons != Du_None) {
+        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        result += DuInt_AsInt(obj);
+        Du_DECREF(obj);
+        cons = _DuCons_NEXT(cons);
+    }
+    return DuInt_FromInt(result);
+}
+
+DuObject *du_sub(DuObject *cons, DuObject *locals)
+{
+    int result = 0;
+    int sign = 1;
+    while (cons != Du_None) {
+        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        result += sign * DuInt_AsInt(obj);
+        sign = -1;
+        Du_DECREF(obj);
+        cons = _DuCons_NEXT(cons);
+    }
+    return DuInt_FromInt(result);
+}
+
+DuObject *du_mul(DuObject *cons, DuObject *locals)
+{
+    int result = 1;
+    while (cons != Du_None) {
+        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        result *= DuInt_AsInt(obj);
+        Du_DECREF(obj);
+        cons = _DuCons_NEXT(cons);
+    }
+    return DuInt_FromInt(result);
+}
+
+static DuObject *_du_intcmp(DuObject *cons, DuObject *locals, int mode)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None ||
+        _DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
+        Du_FatalError("get: expected two arguments");
+    DuObject *obj_a = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj_b = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    int a = DuInt_AsInt(obj_a);
+    int b = DuInt_AsInt(obj_b);
+    Du_DECREF(obj_a);
+    Du_DECREF(obj_b);
+    int r = 0;
+    switch (mode) {
+    case 0: r = a < b; break;
+    case 1: r = a <= b; break;
+    case 2: r = a == b; break;
+    case 3: r = a != b; break;
+    case 4: r = a > b; break;
+    case 5: r = a >= b; break;
+    }
+    return DuInt_FromInt(r);
+}
+
+DuObject *du_lt(DuObject *cons, DuObject *locals)
+{ return _du_intcmp(cons, locals, 0); }
+DuObject *du_le(DuObject *cons, DuObject *locals)
+{ return _du_intcmp(cons, locals, 1); }
+DuObject *du_eq(DuObject *cons, DuObject *locals)
+{ return _du_intcmp(cons, locals, 2); }
+DuObject *du_ne(DuObject *cons, DuObject *locals)
+{ return _du_intcmp(cons, locals, 3); }
+DuObject *du_gt(DuObject *cons, DuObject *locals)
+{ return _du_intcmp(cons, locals, 4); }
+DuObject *du_ge(DuObject *cons, DuObject *locals)
+{ return _du_intcmp(cons, locals, 5); }
+
+DuObject *du_type(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("type: expected one argument");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *res = (DuObject *)(obj->ob_type);
+    Du_INCREF(res);
+    Du_DECREF(obj);
+    return res;
+}
+
+DuObject *du_quote(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("quote: expected one argument");
+    DuObject *obj = _DuCons_CAR(cons);
+    Du_INCREF(obj);
+    return obj;
+}
+
+DuObject *du_list(DuObject *cons, DuObject *locals)
+{
+    DuObject *list = DuList_New();
+    while (cons != Du_None) {
+        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        DuList_Append(list, obj);
+        Du_DECREF(obj);
+        cons = _DuCons_NEXT(cons);
+    }
+    return list;
+}
+
+DuObject *du_container(DuObject *cons, DuObject *locals)
+{
+    if (cons != Du_None && _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("container: expected at most one argument");
+    DuObject *container = DuContainer_New();
+    if (cons != Du_None) {
+        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        DuContainer_SetRef(container, obj);
+        Du_DECREF(obj);
+    }
+    return container;
+}
+
+DuObject *du_get(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None)
+        Du_FatalError("get: expected at least one argument");
+    DuObject *res;
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+
+    if (DuList_Check(obj)) {
+        if (_DuCons_NEXT(cons) == Du_None ||
+            _DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
+            Du_FatalError("get with a list: expected two arguments");
+        DuObject *index = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+        res = DuList_GetItem(obj, DuInt_AsInt(index));
+        Du_DECREF(index);
+    }
+    else if (DuContainer_Check(obj)) {
+        if (_DuCons_NEXT(cons) != Du_None)
+            Du_FatalError("get with a container: expected one argument");
+        res = DuContainer_GetRef(obj);
+    }
+    else
+        Du_FatalError("get: bad argument type '%s'", obj->ob_type->dt_name);
+
+    Du_DECREF(obj);
+    return res;
+}
+
+DuObject *du_set(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
+        Du_FatalError("set: expected at least two arguments");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+
+    if (DuList_Check(obj)) {
+        if (_DuCons_NEXT(_DuCons_NEXT(cons)) == Du_None ||
+            _DuCons_NEXT(_DuCons_NEXT(_DuCons_NEXT(cons))) != Du_None)
+            Du_FatalError("set with a list: expected three arguments");
+        DuObject *index = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+        DuObject *newobj = Du_Eval(
+                    _DuCons_CAR(_DuCons_NEXT(_DuCons_NEXT(cons))), locals);
+        DuList_SetItem(obj, DuInt_AsInt(index), newobj);
+        Du_DECREF(index);
+        Du_DECREF(newobj);
+    }
+    else if (DuContainer_Check(obj)) {
+        if (_DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
+            Du_FatalError("set with a container: expected two arguments");
+        DuObject *newobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+        DuContainer_SetRef(obj, newobj);
+        Du_DECREF(newobj);
+    }
+    else
+        Du_FatalError("set: bad argument type '%s'", obj->ob_type->dt_name);
+
+    Du_DECREF(obj);
+    Du_INCREF(Du_None);
+    return Du_None;
+}
+
+DuObject *du_append(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None ||
+        _DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
+        Du_FatalError("append: expected two arguments");
+    DuObject *lst = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *newobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    DuList_Append(lst, newobj);
+    Du_DECREF(lst);
+    return newobj;
+}
+
+DuObject *du_pop(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None)
+        Du_FatalError("pop: expected at least one argument");
+    DuObject *lst = Du_Eval(_DuCons_CAR(cons), locals);
+    int index;
+    if (_DuCons_NEXT(cons) == Du_None) {
+        index = DuList_Size(lst) - 1;
+        if (index < 0)
+            Du_FatalError("pop: empty list");
+    }
+    else if (_DuCons_NEXT(_DuCons_NEXT(cons)) == Du_None) {
+        DuObject *indexobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+        index = DuInt_AsInt(indexobj);
+        Du_DECREF(indexobj);
+    }
+    else
+        Du_FatalError("pop: expected at most two arguments");
+
+    DuObject *res = DuList_Pop(lst, index);
+    Du_DECREF(lst);
+    return res;
+}
+
+DuObject *du_len(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("len: expected one argument");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    int length = DuObject_Length(obj);
+    Du_DECREF(obj);
+    return DuInt_FromInt(length);
+}
+
+DuObject *du_if(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
+        Du_FatalError("if: expected at least two arguments");
+    DuObject *cond = Du_Eval(_DuCons_CAR(cons), locals);
+    int cond_int = DuObject_IsTrue(cond);
+    Du_DECREF(cond);
+    if (cond_int != 0) {
+        /* true path */
+        return Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    }
+    else {
+        /* false path */
+        return Du_Progn(_DuCons_NEXT(_DuCons_NEXT(cons)), locals);
+    }
+}
+
+DuObject *du_while(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None)
+        Du_FatalError("while: expected at least one argument");
+    while (1) {
+        DuObject *cond = Du_Eval(_DuCons_CAR(cons), locals);
+        int cond_int = DuObject_IsTrue(cond);
+        Du_DECREF(cond);
+        if (cond_int == 0)
+            break;
+        DuObject *res = Du_Progn(_DuCons_NEXT(cons), locals);
+        Du_DECREF(res);
+    }
+    Du_INCREF(Du_None);
+    return Du_None;
+}
+
+DuObject *du_defun(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
+        Du_FatalError("defun: expected at least two arguments");
+    DuObject *name = _DuCons_CAR(cons);
+    DuObject *arglist = _DuCons_CAR(_DuCons_NEXT(cons));
+    DuObject *progn = _DuCons_NEXT(_DuCons_NEXT(cons));
+    DuFrame_SetUserFunction(locals, name, arglist, progn);
+    Du_INCREF(Du_None);
+    return Du_None;
+}
+
+DuObject *du_car(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("car: expected one argument");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *res = DuCons_Car(obj);
+    Du_DECREF(obj);
+    return res;
+}
+
+DuObject *du_cdr(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("car: expected one argument");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *res = DuCons_Cdr(obj);
+    Du_DECREF(obj);
+    return res;
+}
+
+DuObject *du_not(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("not: expected one argument");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    int res = !DuObject_IsTrue(obj);
+    Du_DECREF(obj);
+    return DuInt_FromInt(res);
+}
+
+DuObject *du_transaction(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None)
+        Du_FatalError("transaction: expected at least one argument");
+    DuObject *sym = _DuCons_CAR(cons);
+    DuObject *rest = _DuCons_NEXT(cons);
+    _DuFrame_EvalCall(locals, sym, rest, 0);
+    Du_INCREF(Du_None);
+    return Du_None;
+}
+
+DuObject *du_sleepms(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("sleepms: expected one argument");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    int ms = DuInt_AsInt(obj);
+    Du_DECREF(obj);
+
+    struct timeval t;
+    t.tv_sec = ms / 1000;
+    t.tv_usec = (ms % 1000) * 1000;
+    select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &t);
+
+    Du_INCREF(Du_None);
+    return Du_None;
+}
+
+DuObject *du_defined(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("defined?: expected one argument");
+    DuObject *ob = _DuCons_CAR(cons);
+    DuObject *res = DuFrame_GetSymbol(locals, ob);
+    if (res == NULL)
+        DuFrame_GetSymbol(Du_Globals, ob);
+    if (res != NULL)
+        Du_DECREF(res);
+    return DuInt_FromInt(res != NULL);
+}
+
+DuObject *du_assert(DuObject *cons, DuObject *locals)
+{
+    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
+        Du_FatalError("defined?: expected one argument");
+    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    if (!DuInt_AsInt(obj)) {
+        printf("assert failed: ");
+        Du_Print(_DuCons_CAR(cons), 1);
+        Du_FatalError("assert failed");
+    }
+    Du_DECREF(obj);
+    Du_INCREF(Du_None);
+    return Du_None;
+}
+
+DuObject *Du_Globals;
+
+void Du_Initialize(void)
+{
+    _Du_AME_InitThreadDescriptor();
+    _Du_InitializeObjects();
+
+    Du_Globals = DuFrame_New();
+    DuFrame_SetBuiltinMacro(Du_Globals, "progn", Du_Progn);
+    DuFrame_SetBuiltinMacro(Du_Globals, "setq", du_setq);
+    DuFrame_SetBuiltinMacro(Du_Globals, "print", du_print);
+    DuFrame_SetBuiltinMacro(Du_Globals, "+", du_add);
+    DuFrame_SetBuiltinMacro(Du_Globals, "-", du_sub);
+    DuFrame_SetBuiltinMacro(Du_Globals, "*", du_mul);
+    DuFrame_SetBuiltinMacro(Du_Globals, "<", du_lt);
+    DuFrame_SetBuiltinMacro(Du_Globals, "<=", du_le);
+    DuFrame_SetBuiltinMacro(Du_Globals, "==", du_eq);
+    DuFrame_SetBuiltinMacro(Du_Globals, "!=", du_ne);
+    DuFrame_SetBuiltinMacro(Du_Globals, ">", du_gt);
+    DuFrame_SetBuiltinMacro(Du_Globals, ">=", du_ge);
+    DuFrame_SetBuiltinMacro(Du_Globals, "type", du_type);
+    DuFrame_SetBuiltinMacro(Du_Globals, "quote", du_quote);
+    DuFrame_SetBuiltinMacro(Du_Globals, "list", du_list);
+    DuFrame_SetBuiltinMacro(Du_Globals, "container", du_container);
+    DuFrame_SetBuiltinMacro(Du_Globals, "get", du_get);
+    DuFrame_SetBuiltinMacro(Du_Globals, "set", du_set);
+    DuFrame_SetBuiltinMacro(Du_Globals, "append", du_append);
+    DuFrame_SetBuiltinMacro(Du_Globals, "pop", du_pop);
+    DuFrame_SetBuiltinMacro(Du_Globals, "len", du_len);
+    DuFrame_SetBuiltinMacro(Du_Globals, "if", du_if);
+    DuFrame_SetBuiltinMacro(Du_Globals, "while", du_while);
+    DuFrame_SetBuiltinMacro(Du_Globals, "defun", du_defun);
+    DuFrame_SetBuiltinMacro(Du_Globals, "car", du_car);
+    DuFrame_SetBuiltinMacro(Du_Globals, "cdr", du_cdr);
+    DuFrame_SetBuiltinMacro(Du_Globals, "not", du_not);
+    DuFrame_SetBuiltinMacro(Du_Globals, "transaction", du_transaction);
+    DuFrame_SetBuiltinMacro(Du_Globals, "sleepms", du_sleepms);
+    DuFrame_SetBuiltinMacro(Du_Globals, "defined?", du_defined);
+    DuFrame_SetBuiltinMacro(Du_Globals, "assert", du_assert);
+    DuFrame_SetSymbolStr(Du_Globals, "None", Du_None);
+}
+
+void Du_Finalize(void)
+{
+    Du_DECREF(Du_Globals);
+    Du_Globals = NULL;
+
+    _Du_FinalizeObjects();
+    _Du_AME_FiniThreadDescriptor();
+}
diff --git a/duhton/intobject.c b/duhton/intobject.c
new file mode 100644
--- /dev/null
+++ b/duhton/intobject.c
@@ -0,0 +1,47 @@
+#include "duhton.h"
+
+typedef struct {
+    DuOBJECT_HEAD
+    int ob_intval;
+} DuIntObject;
+
+void int_print(DuIntObject *ob)
+{
+    printf("%d", ob->ob_intval);
+}
+
+int int_is_true(DuIntObject *ob)
+{
+    return ob->ob_intval;
+}
+
+DuTypeObject DuInt_Type = {
+    DuOBJECT_HEAD_INIT(&DuType_Type),
+    "int",
+    sizeof(DuIntObject),
+    (destructor_fn)free,
+    (print_fn)int_print,
+    (eval_fn)NULL,
+    (len_fn)int_is_true,
+    (len_fn)NULL,
+};
+
+DuObject *DuInt_FromInt(int value)
+{
+    DuIntObject *ob = (DuIntObject *)DuObject_New(&DuInt_Type);
+    ob->ob_intval = value;
+    return (DuObject *)ob;
+}
+
+int DuInt_AsInt(DuObject *ob)
+{
+    DuInt_Ensure("DuInt_AsInt", ob);
+    return ((DuIntObject *)ob)->ob_intval;
+}
+
+void DuInt_Ensure(char *where, DuObject *ob)
+{
+    if (!DuInt_Check(ob))
+        Du_FatalError("%s: expected 'int' argument, got '%s'",
+                      where, ob->ob_type->dt_name);
+}
diff --git a/duhton/listobject.c b/duhton/listobject.c
new file mode 100644
--- /dev/null
+++ b/duhton/listobject.c
@@ -0,0 +1,165 @@
+#include <string.h>
+#include "duhton.h"
+
+typedef struct {
+    DuOBJECT_HEAD
+    int ob_count;
+    DuObject **ob_items;
+} DuListObject;
+
+void list_free(DuListObject *ob)
+{
+    int i;
+    for (i=0; i<ob->ob_count; i++) {
+        Du_DECREF(ob->ob_items[i]);
+#ifdef Du_DEBUG
+        ob->ob_items[i] = (DuObject *)0xDD;
+#endif
+    }
+    free(ob->ob_items);
+    free(ob);
+}
+
+void list_print(DuListObject *ob)
+{
+    int i;
+    Du_AME_INEVITABLE(ob);
+    if (ob->ob_count == 0) {
+        printf("[]");
+    }
+    else {
+        printf("[ ");
+        for (i=0; i<ob->ob_count; i++) {
+            Du_Print(ob->ob_items[i], 0);
+            printf(" ");
+        }
+        printf("]");
+    }
+}
+
+int list_length(DuListObject *ob)
+{
+    int length;
+    Du_AME_READ(ob, (length = ob->ob_count));
+    return length;
+}
+
+void list_ame_copy(DuListObject *ob)
+{
+    DuObject **globitems = ob->ob_items;
+    int count = ob->ob_count;
+    ob->ob_items = malloc(sizeof(DuObject*) * count);
+    assert(ob->ob_items);
+    memcpy((char*)ob->ob_items, globitems, sizeof(DuObject*) * count);
+    /* XXX either this ob_items or the original one is never freed */
+}
+
+void _list_append(DuListObject *ob, DuObject *x)
+{
+    Du_AME_WRITE(ob);
+    int i, newcount = ob->ob_count + 1;
+    DuObject **olditems = ob->ob_items;
+    DuObject **newitems = malloc(sizeof(DuObject*) * newcount);
+    assert(newitems);
+    for (i=0; i<newcount-1; i++)
+        newitems[i] = olditems[i];
+    Du_INCREF(x);
+    newitems[newcount-1] = x;
+    ob->ob_items = newitems;
+    ob->ob_count = newcount;
+    free(olditems);
+}
+
+void DuList_Append(DuObject *ob, DuObject *item)
+{
+    DuList_Ensure("DuList_Append", ob);
+    _list_append((DuListObject *)ob, item);
+}
+
+int DuList_Size(DuObject *ob)
+{
+    DuList_Ensure("DuList_Size", ob);
+    return list_length((DuListObject *)ob);
+}
+
+DuObject *_list_getitem(DuListObject *ob, int index)
+{
+    DuObject *result;
+    int length;
+    DuObject **items;
+    Du_AME_READ(ob, (length = ob->ob_count, items = ob->ob_items));
+    if (index < 0 || index >= length)
+        Du_FatalError("list_get: index out of range");
+    result = items[index];
+    Du_INCREF(result);
+    return result;
+}
+
+DuObject *DuList_GetItem(DuObject *ob, int index)
+{
+    DuList_Ensure("DuList_GetItem", ob);
+    return _list_getitem((DuListObject *)ob, index);
+}
+
+void _list_setitem(DuListObject *ob, int index, DuObject *newitem)
+{
+    Du_AME_WRITE(ob);
+    if (index < 0 || index >= ob->ob_count)
+        Du_FatalError("list_set: index out of range");
+    DuObject *prev = ob->ob_items[index];
+    Du_INCREF(newitem);
+    ob->ob_items[index] = newitem;
+    Du_DECREF(prev);
+}
+
+void DuList_SetItem(DuObject *list, int index, DuObject *newobj)
+{
+    DuList_Ensure("DuList_SetItem", list);
+    _list_setitem((DuListObject *)list, index, newobj);
+}
+
+DuObject *_list_pop(DuListObject *ob, int index)
+{
+    int i;
+    Du_AME_WRITE(ob);
+    if (index < 0 || index >= ob->ob_count)
+        Du_FatalError("list_pop: index out of range");
+    DuObject *result = ob->ob_items[index];
+    ob->ob_count--;
+    for (i=index; i<ob->ob_count; i++)
+        ob->ob_items[i] = ob->ob_items[i+1];
+    return result;
+}
+
+DuObject *DuList_Pop(DuObject *list, int index)
+{
+    DuList_Ensure("DuList_Pop", list);
+    return _list_pop((DuListObject *)list, index);
+}
+
+DuTypeObject DuList_Type = {
+    DuOBJECT_HEAD_INIT(&DuType_Type),
+    "list",
+    sizeof(DuListObject),
+    (destructor_fn)list_free,
+    (print_fn)list_print,
+    (eval_fn)NULL,
+    (len_fn)NULL,
+    (len_fn)list_length,
+    (ame_copy_fn)list_ame_copy,
+};
+
+DuObject *DuList_New()
+{
+    DuListObject *ob = (DuListObject *)DuObject_New(&DuList_Type);
+    ob->ob_count = 0;
+    ob->ob_items = NULL;
+    return (DuObject *)ob;
+}
+
+void DuList_Ensure(char *where, DuObject *ob)
+{
+    if (!DuList_Check(ob))
+        Du_FatalError("%s: expected 'list' argument, got '%s'",
+                      where, ob->ob_type->dt_name);
+}
diff --git a/duhton/object.c b/duhton/object.c
new file mode 100644
--- /dev/null
+++ b/duhton/object.c
@@ -0,0 +1,164 @@
+#include <stdarg.h>
+#include "duhton.h"
+
+
+static __thread DuObject chainedlist;
+
+DuObject *DuObject_New(DuTypeObject *tp)
+{
+    assert(tp->dt_size >= sizeof(DuObject));
+    DuObject *ob = malloc(tp->dt_size);
+    assert(ob);
+    ob->ob_refcnt = 1;
+    ob->ob_type = tp;
+    _Du_NewReference(ob);
+    return ob;
+}
+
+void _Du_Dealloc(DuObject *ob)
+{
+    assert(ob->ob_refcnt == 1);
+    destructor_fn destructor = ob->ob_type->dt_destructor;
+    assert(destructor != NULL);
+    _Du_ForgetReference(ob);
+    destructor(ob);
+}
+
+#ifdef Du_TRACK_REFS
+void _Du_NewReference(DuObject *ob)
+{
+    if (chainedlist.ob_debug_prev == NULL)
+        Du_FatalError("Du_Initialize() must be called first");
+    ob->ob_debug_prev = &chainedlist;
+    ob->ob_debug_next = chainedlist.ob_debug_next;
+#ifdef Du_AME
+    ob->ob_version = 0;
+#endif
+    chainedlist.ob_debug_next = ob;
+    ob->ob_debug_next->ob_debug_prev = ob;
+}
+
+void _Du_ForgetReference(DuObject *ob)
+{
+    ob->ob_debug_prev->ob_debug_next = ob->ob_debug_next;
+    ob->ob_debug_next->ob_debug_prev = ob->ob_debug_prev;
+    ob->ob_debug_prev = NULL;
+    ob->ob_debug_next = NULL;
+}
+#endif
+
+void _Du_BecomeImmortal(DuObject *ob)
+{
+    _Du_ForgetReference(ob);
+    ob->ob_refcnt = -1;
+}
+
+void type_print(DuTypeObject *ob)
+{
+    printf("<type '%s'>", ob->dt_name);
+}
+
+DuTypeObject DuType_Type = {
+    DuOBJECT_HEAD_INIT(&DuType_Type),
+    "type",
+    0,
+    NULL,
+    (print_fn)type_print,
+};
+
+void none_print(DuObject *ob)
+{
+    printf("None");
+}
+
+int none_is_true(DuObject *ob)
+{
+    return 0;
+}
+
+DuTypeObject DuNone_Type = {
+    DuOBJECT_HEAD_INIT(&DuType_Type),
+    "NoneType",
+    sizeof(DuObject),
+    NULL,
+    none_print,
+    (eval_fn)NULL,
+    none_is_true,
+};
+
+DuObject _Du_NoneStruct =
+    DuOBJECT_HEAD_INIT(&DuNone_Type);
+
+void Du_FatalError(char *msg, ...)
+{
+    va_list ap;
+    va_start(ap, msg);
+    fprintf(stderr, "Error: ");
+    vfprintf(stderr, msg, ap);
+    fprintf(stderr, "\n");
+    va_end(ap);
+    exit(1);
+}
+
+void Du_Print(DuObject *ob, int newline)
+{
+    ob->ob_type->dt_print(ob);
+    if (newline)
+        printf("\n");
+}
+
+int DuObject_IsTrue(DuObject *ob)
+{
+    len_fn fn = ob->ob_type->dt_is_true;
+    if (!fn) fn = ob->ob_type->dt_length;
+    if (!fn) return 1;
+    return fn(ob) != 0;
+}
+
+int DuObject_Length(DuObject *ob)
+{
+    if (!ob->ob_type->dt_length)
+        Du_FatalError("object of type '%s' has no length",
+                      ob->ob_type->dt_name);
+    return ob->ob_type->dt_length(ob);
+}
+
+void _Du_InitializeObjects(void)
+{
+#ifdef Du_TRACK_REFS
+    chainedlist.ob_debug_prev = &chainedlist;
+    chainedlist.ob_debug_next = &chainedlist;
+#endif
+}
+
+void _Du_FinalizeObjects(void)
+{
+#ifdef Du_DEBUG
+    DuObject *obj;
+    for (obj = chainedlist.ob_debug_next;
+         obj != &chainedlist;
+         obj = obj->ob_debug_next) {
+        printf("NOT FREED: ");
+        Du_Print(obj, 1);
+    }
+#endif
+#ifdef Du_TRACK_REFS
+    chainedlist.ob_debug_prev = NULL;
+    chainedlist.ob_debug_next = NULL;
+#endif
+}
+
+#ifdef Du_TRACK_REFS
+void _Du_MakeImmortal(void)
+{
+    DuObject *end_marker = &chainedlist;
+    DuObject *obj = end_marker;
+    do {
+        DuObject *next = obj->ob_debug_next;
+        obj->ob_refcnt = -1;
+        obj->ob_debug_prev = NULL;
+        obj->ob_debug_next = NULL;
+        obj = next;
+    } while (obj != end_marker);
+}
+#endif
diff --git a/duhton/stm/ame.c b/duhton/stm/ame.c
new file mode 100644
--- /dev/null
+++ b/duhton/stm/ame.c
@@ -0,0 +1,466 @@
+#include <string.h>
+#include <pthread.h>
+#include "ame.h"
+#include "lists.h"
+
+
+/* XXX assumes that time never wraps around (in a 'long'), which may be
+ * correct on 64-bit machines but not on 32-bit machines if the process
+ * runs for long enough.
+ */
+
+#define IS_LOCKED(num)  ((num) < 0)
+#define IS_LOCKED_OR_NEWER(num, max_age)                        \
+    (((unsigned long)(num)) > ((unsigned long)(max_age)))
+
+
+struct tx_descriptor {
+    jmp_buf *setjmp_buf;
+    int in_transaction;
+    owner_version_t start_time;
+    owner_version_t end_time;
+    owner_version_t my_lock_word;
+    struct OrecList reads;
+    struct RedoLog redolog;   /* last item, because it's the biggest one */
+};
+
+/* global_timestamp contains in its lowest bit a flag equal to 1
+   if there is an inevitable transaction running */
+static volatile owner_version_t global_timestamp = 2;
+static __thread struct tx_descriptor *thread_descriptor;
+
+
+static void common_cleanup(struct tx_descriptor *d)
+{
+    d->reads.size = 0;
+    redolog_clear(&d->redolog);
+    d->in_transaction = 0;
+}
+
+static void tx_spinloop(void)
+{
+    spinloop();
+}
+
+static void tx_abort(void);
+
+static int is_inevitable(struct tx_descriptor *d)
+{
+    assert(d->in_transaction);
+    return d->setjmp_buf == NULL;
+}
+
+
+typedef struct {
+    DuOBJECT_HEAD
+    DuObject *ob_reference;
+} DuContainerObject;
+
+/*** run the redo log to commit a transaction, and release the locks */
+static void tx_redo(struct tx_descriptor *d)
+{
+    owner_version_t newver = d->end_time;
+    wlog_t *item;
+    REDOLOG_LOOP_FORWARD(d->redolog, item)
+    {
+        DuObject *globalobj = item->addr;
+        DuObject *localobj = item->val;
+        long size = localobj->ob_type->dt_size;
+        assert(size >= sizeof(DuObject));
+        memcpy(((char *)globalobj) + sizeof(DuObject),
+               ((char *)localobj) + sizeof(DuObject),
+               size - sizeof(DuObject));
+        CFENCE;
+        globalobj->ob_version = newver;
+
+        //int num = DuInt_AsInt(((DuContainerObject*)localobj)->ob_reference);
+        //printf("COMMIT thread %lx: %p <- %p (%d), v. %ld\n",
+        //        (long)pthread_self(), globalobj, localobj, num, newver);
+
+    } REDOLOG_LOOP_END;
+}
+
+/*** on abort, release locks and restore the old version number. */
+static void releaseAndRevertLocks(struct tx_descriptor *d)
+{
+  wlog_t *item;
+  REDOLOG_LOOP_FORWARD(d->redolog, item)
+  {
+      if (item->p != -1) {
+          DuObject* o = item->addr;
+          o->ob_version = item->p;
+      }
+  } REDOLOG_LOOP_END;
+}
+
+/*** release locks and restore the old version number, ready to retry later */
+static void releaseLocksForRetry(struct tx_descriptor *d)
+{
+    wlog_t *item;
+    REDOLOG_LOOP_FORWARD(d->redolog, item)
+    {
+        if (item->p != -1) {
+            DuObject* o = item->addr;
+            o->ob_version = item->p;
+            item->p = -1;
+        }
+    } REDOLOG_LOOP_END;
+}
+
+/*** lock all locations */
+static void acquireLocks(struct tx_descriptor *d)
+{
+    wlog_t *item;
+    // try to lock every location in the write set
+    REDOLOG_LOOP_BACKWARD(d->redolog, item)
+    {
+        DuObject* o = item->addr;
+        owner_version_t ovt;
+
+    retry:
+        ovt = o->ob_version;
+
+        // if object not locked, lock it
+        //
+        // NB: if ovt > start time, we may introduce inconsistent reads.
+        // Since most writes are also reads, we'll just abort under this
+        // condition.  This can introduce false conflicts
+        if (!IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
+            if (!bool_cas(&o->ob_version, ovt, d->my_lock_word)) {
+                CFENCE;
+                goto retry;
+            }
+            // save old version to item->p.  Now we hold the lock.
+            item->p = ovt;
+        }
+        // else if the location is too recent...
+        else if (!IS_LOCKED(ovt))
+            tx_abort();
+        // else it is locked: check it's not by me (no duplicates in redolog)
+        else {
+            assert(ovt != d->my_lock_word);
+            // we can either abort or spinloop.  Because we are at the end of
+            // the transaction we might try to spinloop, even though after the
+            // lock is released the ovt will be very recent, possibly greater
+            // than d->start_time.  It is necessary to spinloop in case we are
+            // inevitable, so use that as a criteria.  Another solution to
+            // avoid deadlocks would be to sort the order in which we take the
+            // locks.
+            if (is_inevitable(d))
+                tx_spinloop();
+            else
+                tx_abort();
+            goto retry;
+        }
+    } REDOLOG_LOOP_END;
+}
+
+/**
+ * fast-path validation, assuming that I don't hold locks.
+ */
+static void validate_fast(struct tx_descriptor *d)
+{
+    int i;
+    owner_version_t ovt;
+    assert(!is_inevitable(d));
+    for (i=0; i<d->reads.size; i++) {
+     retry:

From noreply at buildbot.pypy.org  Sun Jun 23 11:53:24 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Jun 2013 11:53:24 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tweak the compilation.
Message-ID: <20130623095324.9712A1C06BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r245:ec040cc165db
Date: 2013-06-23 10:37 +0200
http://bitbucket.org/pypy/stmgc/changeset/ec040cc165db/

Log:	Tweak the compilation.

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,3 +1,5 @@
 syntax: glob
 *.pyc
 *~
+c4/build-demo*
+c4/debug-demo*
diff --git a/c4/Makefile b/c4/Makefile
--- a/c4/Makefile
+++ b/c4/Makefile
@@ -1,8 +1,9 @@
 #
-# Run with "make -jN" for maximum randomness.
+# Makefile for the demos.
 #
 
-debug: debug-demo1
+build: build-demo1 build-demo2 build-demo_random
+debug: debug-demo1 debug-demo2 debug-demo_random
 
 clean:
 	rm -f debug-demo1
@@ -18,8 +19,8 @@
 DEBUG = -g -DGC_NURSERY=0x10000  #-D_GC_DEBUG=1
 
 
-build-%: %.c ${H_FILES} ${C_FILES}
-	gcc -lrt -pthread -O2 -g $< -o $* -Wall ${C_FILES}
+build-%: %.c ${H_FILES} ${C_FILES} stmgc.c
+	gcc -lrt -pthread -O2 -g $< -o build-$* -Wall stmgc.c
 
 debug-%: %.c ${H_FILES} ${C_FILES}
 	gcc -lrt -pthread ${DEBUG} $< -o debug-$* -Wall ${C_FILES}
diff --git a/c4/stmgc.c b/c4/stmgc.c
new file mode 100644
--- /dev/null
+++ b/c4/stmgc.c
@@ -0,0 +1,13 @@
+/* This file is optional: it can be used to compile all .c files
+ * into one single big compilation unit.  You must compile either
+ * this, or individually the other .c files in this directory.
+ */
+
+#include "et.c"
+#include "lists.c"
+#include "steal.c"
+#include "nursery.c"
+#include "gcpage.c"
+#include "stmsync.c"
+#include "dbgmem.c"
+#include "fprintcolor.c"
diff --git a/duhton/Makefile b/duhton/Makefile
--- a/duhton/Makefile
+++ b/duhton/Makefile
@@ -1,11 +1,11 @@
 
 all: duhton_debug duhton
 
-duhton: *.c *.h stm/*.c stm/*.h
-	gcc -pthread -g -O2 -o duhton *.c stm/*.c -Wall
+duhton: *.c *.h ../c4/*.c ../c4/*.h
+	gcc -pthread -g -O2 -o duhton *.c ../c4/stmgc.c -Wall
 
-duhton_debug: *.c *.h stm/*.c stm/*.h
-	gcc -pthread -g -DDu_DEBUG -o duhton_debug *.c stm/*.c -Wall
+duhton_debug: *.c *.h ../c4/*.c ../c4/*.h
+	gcc -pthread -g -DDu_DEBUG -o duhton_debug *.c ../c4/stmgc.c -Wall
 
 clean:
 	rm -f duhton duhton_debug

From noreply at buildbot.pypy.org  Sun Jun 23 11:53:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Jun 2013 11:53:27 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130623095327.B32001C06BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r246:501a1ab04191
Date: 2013-06-23 11:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/501a1ab04191/

Log:	in-progress

diff too long, truncating to 2000 out of 2024 lines

diff --git a/duhton/ceval.c b/duhton/ceval.c
--- a/duhton/ceval.c
+++ b/duhton/ceval.c
@@ -3,12 +3,11 @@
 
 DuObject *Du_Eval(DuObject *ob, DuObject *locals)
 {
-    eval_fn fn = ob->ob_type->dt_eval;
+    eval_fn fn = Du_TYPE(ob)->dt_eval;
     if (fn) {
         return fn(ob, locals);
     }
     else {
-        Du_INCREF(ob);
         return ob;
     }
 }
diff --git a/duhton/compile.c b/duhton/compile.c
--- a/duhton/compile.c
+++ b/duhton/compile.c
@@ -11,9 +11,12 @@
 
     list = DuList_New();
     if (level == 0) {
+        _du_save1(list);
         DuObject *item = DuSymbol_FromString("progn");
+        _du_restore1(list);
+        _du_save1(list);
         DuList_Append(list, item);
-        Du_DECREF(item);
+        _du_restore1(list);
     }
     c = fgetc(f);
     while (1) {
@@ -25,7 +28,6 @@
             if (level > 0)
                 Du_FatalError("more '(' than ')'");
             if (stop_after_newline) {
-                Du_DECREF(list);
                 return NULL;
             }
             goto done;
@@ -76,21 +78,19 @@
                 break;
             }
         }
+        _du_save1(list);
         DuList_Append(list, item);
-        Du_DECREF(item);
+        _du_restore1(list);
     }
 
  done:
-    Du_INCREF(Du_None);
     cons = Du_None;
     for (i = DuList_Size(list) - 1; i >= 0; i--) {
         DuObject *item = DuList_GetItem(list, i);
-        DuObject *newcons = DuCons_New(item, cons);
-        Du_DECREF(cons);
-        Du_DECREF(item);
-        cons = newcons;
+        _du_save1(list);
+        cons = DuCons_New(item, cons);
+        _du_restore1(list);
     }
-    Du_DECREF(list);
     return cons;
 }
 
diff --git a/duhton/consobject.c b/duhton/consobject.c
--- a/duhton/consobject.c
+++ b/duhton/consobject.c
@@ -5,22 +5,15 @@
     DuObject *car, *cdr;
 } DuConsObject;
 
-void cons_free(DuConsObject *ob)
-{
-    Du_DECREF(ob->car);
-    Du_DECREF(ob->cdr);
-#ifdef Du_DEBUG
-    ob->car = ob->cdr = (DuObject *)0xDD;
-#endif
-    free(ob);
-}
-
 void cons_print(DuConsObject *ob)
 {
     DuObject *p;
     printf("( ");
     while (1) {
+        _du_read1(ob);
+        _du_save1(ob);
         Du_Print(ob->car, 0);
+        _du_restore1(ob);
         p = ob->cdr;
         if (!DuCons_Check(p))
             break;
@@ -38,51 +31,53 @@
 
 DuObject *cons_eval(DuConsObject *ob, DuObject *locals)
 {
+    _du_read1(ob);
     return _DuFrame_EvalCall(locals, ob->car, ob->cdr, 1);
 }
 
-DuTypeObject DuCons_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
+DuType DuCons_Type = {
     "cons",
+    DUTYPE_CONS,
     sizeof(DuConsObject),
-    (destructor_fn)cons_free,
     (print_fn)cons_print,
     (eval_fn)cons_eval,
 };
 
 DuObject *DuCons_New(DuObject *car, DuObject *cdr)
 {
+    _du_save2(car, cdr);
     DuConsObject *ob = (DuConsObject *)DuObject_New(&DuCons_Type);
-    ob->car = car; Du_INCREF(car);
-    ob->cdr = cdr; Du_INCREF(cdr);
+    _du_restore2(car, cdr);
+    ob->car = car;
+    ob->cdr = cdr;
     return (DuObject *)ob;
 }
 
 DuObject *DuCons_Car(DuObject *cons)
 {
     DuCons_Ensure("DuCons_Car", cons);
-    DuObject *res = ((DuConsObject *)cons)->car;
-    Du_INCREF(res);
-    return res;
+    _du_read1(cons);
+    return ((DuConsObject *)cons)->car;
 }
 
 DuObject *DuCons_Cdr(DuObject *cons)
 {
     DuCons_Ensure("DuCons_Cdr", cons);
-    DuObject *res = ((DuConsObject *)cons)->cdr;
-    Du_INCREF(res);
-    return res;
+    _du_read1(cons);
+    return ((DuConsObject *)cons)->cdr;
 }
 
 DuObject *_DuCons_CAR(DuObject *cons)
 {
     assert(DuCons_Check(cons));
+    _du_read1(cons);
     return ((DuConsObject *)cons)->car;
 }
 
 DuObject *_DuCons_NEXT(DuObject *cons)
 {
     assert(DuCons_Check(cons));
+    _du_read1(cons);
     DuObject *result = ((DuConsObject *)cons)->cdr;
     if (result != Du_None && !DuCons_Check(cons))
         Du_FatalError("_DuCons_NEXT: not a well-formed cons list");
@@ -93,5 +88,5 @@
 {
     if (!DuCons_Check(ob))
         Du_FatalError("%s: expected 'cons' argument, got '%s'",
-                      where, ob->ob_type->dt_name);
+                      where, Du_TYPE(ob)->dt_name);
 }
diff --git a/duhton/containerobject.c b/duhton/containerobject.c
--- a/duhton/containerobject.c
+++ b/duhton/containerobject.c
@@ -44,11 +44,10 @@
     Du_DECREF(prev);
 }
 
-DuTypeObject DuContainer_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
+DuType DuContainer_Type = {
     "container",
+    DUTYPE_CONTAINER,
     sizeof(DuContainerObject),
-    (destructor_fn)container_free,
     (print_fn)container_print,
 };
 
diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -1,43 +1,17 @@
 #ifndef _DUHTON_H_
 #define _DUHTON_H_
 
-
+#include "../c4/stmgc.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
 
 
-#define Du_AME     /* must always be on for now */
-
-#if defined(Du_DEBUG) || defined(Du_AME)
-#  define Du_TRACK_REFS
-#endif
-
-#ifdef Du_AME
-typedef long owner_version_t;
-#endif
-
-
-typedef struct _DuObject {
-    int ob_refcnt;
-    struct _DuTypeObject *ob_type;
-#ifdef Du_TRACK_REFS
-    struct _DuObject *ob_debug_prev, *ob_debug_next;
-#endif
-#ifdef Du_AME
-    owner_version_t ob_version;
-#endif
-} DuObject;
-
-#ifdef Du_TRACK_REFS
-#  define _DuObject_HEAD_EXTRA  NULL, NULL
-#else
-#  define _DuObject_HEAD_EXTRA  /* nothing */
-#endif
+typedef struct stm_object_s DuObject;
 
 #define DuOBJECT_HEAD   DuObject ob_base;
 
-#define DuOBJECT_HEAD_INIT(type)   { -1, type, _DuObject_HEAD_EXTRA }
+#define DuOBJECT_HEAD_INIT(type)   { type | PREBUILT_FLAGS, PREBUILT_REVISION }
 
 
 #ifdef __GNUC__
@@ -47,75 +21,59 @@
 #endif
 
 
-typedef void(*destructor_fn)(DuObject *);
 typedef void(*print_fn)(DuObject *);
 typedef DuObject *(*eval_fn)(DuObject *, DuObject *);
 typedef int(*len_fn)(DuObject *);
-typedef void(*ame_copy_fn)(DuObject *);
 
-typedef struct _DuTypeObject {
-    DuOBJECT_HEAD
+typedef struct {
     const char *dt_name;
+    int dt_typeindex;
     int dt_size;
-    destructor_fn dt_destructor;
     print_fn dt_print;
     eval_fn dt_eval;
     len_fn dt_is_true;
     len_fn dt_length;
-    ame_copy_fn dt_ame_copy;
-} DuTypeObject;
+} DuType;
 
-DuObject *DuObject_New(DuTypeObject *tp);
-void _Du_Dealloc(DuObject *ob);
+#define DUTYPE_INVALID       0
+#define DUTYPE_NONE          1
+#define DUTYPE_INT           2
+#define DUTYPE_SYMBOL        3
+#define DUTYPE_CONS          4
+#define DUTYPE_LIST          5
+#define DUTYPE_TUPLE         6
+#define DUTYPE_FRAME         7
+#define DUTYPE_CONTAINER     8
+#define _DUTYPE_TOTAL        9
+
+extern DuType DuNone_Type;
+extern DuType DuInt_Type;
+extern DuType DuSymbol_Type;
+extern DuType DuCons_Type;
+extern DuType DuList_Type;
+extern DuType DuTuple_Type;
+extern DuType DuFrame_Type;
+extern DuType DuContainer_Type;
+
+extern DuType *Du_Types[_DUTYPE_TOTAL];
+
+
+DuObject *DuObject_New(DuType *tp);
 int DuObject_IsTrue(DuObject *ob);
 int DuObject_Length(DuObject *ob);
 
-#ifdef Du_TRACK_REFS
-void _Du_NewReference(DuObject *ob);
-void _Du_ForgetReference(DuObject *ob);
-#else
-#define _Du_NewReference(ob)       /* nothing */
-#define _Du_ForgetReference(ob)    /* nothing */
-#endif
-void _Du_BecomeImmortal(DuObject *ob);
-
-
-#define Du_AME_GLOBAL(ob)                       \
-    (((DuObject*)(ob))->ob_refcnt < 0)
-
-#define Du_INCREF(ob)                           \
-    do {                                        \
-        if (!Du_AME_GLOBAL(ob))                 \
-            ++((DuObject*)(ob))->ob_refcnt;     \
-    } while (0)
-
-#define Du_DECREF(ob)                                   \
-    do {                                                \
-        if (((DuObject*)(ob))->ob_refcnt > 1)           \
-            --((DuObject*)(ob))->ob_refcnt;             \
-        else if (!Du_AME_GLOBAL(ob))                    \
-            _Du_Dealloc((DuObject*)(ob));               \
-    } while (0)
-
 
 extern DuObject _Du_NoneStruct;
 #define Du_None (&_Du_NoneStruct)
 
-extern DuTypeObject DuType_Type;
-extern DuTypeObject DuInt_Type;
-extern DuTypeObject DuList_Type;
-extern DuTypeObject DuContainer_Type;
-extern DuTypeObject DuCons_Type;
-extern DuTypeObject DuSymbol_Type;
-extern DuTypeObject DuFrame_Type;
-
-#define DuType_Check(ob)      (((DuObject*)(ob))->ob_type == &DuType_Type)
-#define DuInt_Check(ob)       (((DuObject*)(ob))->ob_type == &DuInt_Type)
-#define DuList_Check(ob)      (((DuObject*)(ob))->ob_type == &DuList_Type)
-#define DuContainer_Check(ob) (((DuObject*)(ob))->ob_type == &DuContainer_Type)
-#define DuCons_Check(ob)      (((DuObject*)(ob))->ob_type == &DuCons_Type)
-#define DuSymbol_Check(ob)    (((DuObject*)(ob))->ob_type == &DuSymbol_Type)
-#define DuFrame_Check(ob)     (((DuObject*)(ob))->ob_type == &DuFrame_Type)
+#define _DuObject_TypeNum(ob) stm_get_tid((DuObject*)(ob))
+#define Du_TYPE(ob)           (Du_Types[_DuObject_TypeNum(ob)])
+#define DuInt_Check(ob)       (_DuObject_TypeNum(ob) == DUTYPE_INT)
+#define DuSymbol_Check(ob)    (_DuObject_TypeNum(ob) == DUTYPE_SYMBOL)
+#define DuCons_Check(ob)      (_DuObject_TypeNum(ob) == DUTYPE_CONS)
+#define DuList_Check(ob)      (_DuObject_TypeNum(ob) == DUTYPE_LIST)
+#define DuFrame_Check(ob)     (_DuObject_TypeNum(ob) == DUTYPE_FRAME)
+#define DuContainer_Check(ob) (_DuObject_TypeNum(ob) == DUTYPE_CONTAINER)
 
 void DuType_Ensure(char *where, DuObject *ob);
 void DuInt_Ensure(char *where, DuObject *ob);
@@ -168,17 +126,28 @@
 
 void Du_Initialize(void);
 void Du_Finalize(void);
-void _Du_InitializeObjects(void);
-void _Du_FinalizeObjects(void);
-#ifdef Du_TRACK_REFS
-void _Du_MakeImmortal(void);
-#endif
 extern DuObject *Du_Globals;
 
 void Du_TransactionAdd(DuObject *code, DuObject *frame);
 void Du_TransactionRun(void);
 
 
-#include "stm/ame.h"
+#define _du_save1(p1)           (stm_push_root((DuObject *)(p1)))
+#define _du_save2(p1,p2)        (stm_push_root((DuObject *)(p1)),  \
+                                 stm_push_root((DuObject *)(p2)))
+#define _du_save3(p1,p2,p3)     (stm_push_root((DuObject *)(p1)),  \
+                                 stm_push_root((DuObject *)(p2)),  \
+                                 stm_push_root((DuObject *)(p3)))
+
+#define _du_restore1(p1)        (p1 = (typeof(p1))stm_pop_root())
+#define _du_restore2(p1,p2)     (p2 = (typeof(p2))stm_pop_root(),  \
+                                 p1 = (typeof(p1))stm_pop_root())
+#define _du_restore3(p1,p2,p3)  (p3 = (typeof(p3))stm_pop_root(),  \
+                                 p2 = (typeof(p2))stm_pop_root(),  \
+                                 p1 = (typeof(p1))stm_pop_root())
+
+#define _du_read1(p1)    (p1 = (typeof(p1))stm_read_barrier((DuObject *)(p1)))
+#define _du_write1(p1)   (p1 = (typeof(p1))stm_write_barrier((DuObject *)(p1)))
+
 
 #endif  /* _DUHTON_H_ */
diff --git a/duhton/frame.c b/duhton/frame.c
--- a/duhton/frame.c
+++ b/duhton/frame.c
@@ -226,10 +226,9 @@
                       where, ob->ob_type->dt_name);
 }
 
-DuTypeObject DuFrame_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
+DuType DuFrame_Type = {
     "frame",
+    DUTYPE_FRAME,
     sizeof(DuFrameObject),
-    (destructor_fn)frame_free,
     (print_fn)frame_print,
 };
diff --git a/duhton/intobject.c b/duhton/intobject.c
--- a/duhton/intobject.c
+++ b/duhton/intobject.c
@@ -7,19 +7,20 @@
 
 void int_print(DuIntObject *ob)
 {
+    _du_read1(ob);
     printf("%d", ob->ob_intval);
 }
 
 int int_is_true(DuIntObject *ob)
 {
+    _du_read1(ob);
     return ob->ob_intval;
 }
 
-DuTypeObject DuInt_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
+DuType DuInt_Type = {
     "int",
+    DUTYPE_INT,
     sizeof(DuIntObject),
-    (destructor_fn)free,
     (print_fn)int_print,
     (eval_fn)NULL,
     (len_fn)int_is_true,
@@ -36,6 +37,7 @@
 int DuInt_AsInt(DuObject *ob)
 {
     DuInt_Ensure("DuInt_AsInt", ob);
+    _du_read1(ob);
     return ((DuIntObject *)ob)->ob_intval;
 }
 
@@ -43,5 +45,5 @@
 {
     if (!DuInt_Check(ob))
         Du_FatalError("%s: expected 'int' argument, got '%s'",
-                      where, ob->ob_type->dt_name);
+                      where, Du_TYPE(ob)->dt_name);
 }
diff --git a/duhton/listobject.c b/duhton/listobject.c
--- a/duhton/listobject.c
+++ b/duhton/listobject.c
@@ -1,36 +1,35 @@
 #include <string.h>
 #include "duhton.h"
 
+
+/* 'tuple' objects are only used internally as the current items
+   of 'list' objects
+*/
 typedef struct {
     DuOBJECT_HEAD
     int ob_count;
-    DuObject **ob_items;
+    DuObject *ob_items[1];
+} DuTupleObject;
+
+typedef struct {
+    DuOBJECT_HEAD
+    DuTupleObject *ob_tuple;
 } DuListObject;
 
-void list_free(DuListObject *ob)
-{
-    int i;
-    for (i=0; i<ob->ob_count; i++) {
-        Du_DECREF(ob->ob_items[i]);
-#ifdef Du_DEBUG
-        ob->ob_items[i] = (DuObject *)0xDD;
-#endif
-    }
-    free(ob->ob_items);
-    free(ob);
-}
 
 void list_print(DuListObject *ob)
 {
     int i;
-    Du_AME_INEVITABLE(ob);
+    _du_read1(ob);
     if (ob->ob_count == 0) {
         printf("[]");
     }
     else {
+        DuTupleObject *p = ob->ob_tuple;
+        _du_read1(p);
         printf("[ ");
-        for (i=0; i<ob->ob_count; i++) {
-            Du_Print(ob->ob_items[i], 0);
+        for (i=0; i<p->ob_count; i++) {
+            Du_Print(p->ob_items[i], 0);
             printf(" ");
         }
         printf("]");
@@ -39,35 +38,35 @@
 
 int list_length(DuListObject *ob)
 {
-    int length;
-    Du_AME_READ(ob, (length = ob->ob_count));
-    return length;
+    _du_read1(ob);
+    DuTupleObject *p = ob->ob_tuple;
+    _du_read1(p);
+    return p->ob_count;
 }
 
-void list_ame_copy(DuListObject *ob)
+DuTupleObject *DuTuple_New(int length)
 {
-    DuObject **globitems = ob->ob_items;
-    int count = ob->ob_count;
-    ob->ob_items = malloc(sizeof(DuObject*) * count);
-    assert(ob->ob_items);
-    memcpy((char*)ob->ob_items, globitems, sizeof(DuObject*) * count);
-    /* XXX either this ob_items or the original one is never freed */
+    DuTupleObject *ob;
+    size_t size = sizeof(DuTupleObject) + (length-1)*sizeof(DuObject *);
+    ob = (DuTupleObject *)stm_allocate(size, DUTYPE_TUPLE);
+    ob->ob_count = length;
+    return ob;
 }
 
 void _list_append(DuListObject *ob, DuObject *x)
 {
-    Du_AME_WRITE(ob);
-    int i, newcount = ob->ob_count + 1;
-    DuObject **olditems = ob->ob_items;
-    DuObject **newitems = malloc(sizeof(DuObject*) * newcount);
-    assert(newitems);
+    _du_write1(ob);
+    DuTupleObject *olditems = ob->ob_tuple;
+
+    _du_read1(olditems);
+    int i, newcount = olditems->ob_count + 1;
+    DuTupleObject *newitems = DuTuple_New(newcount);
+
     for (i=0; i<newcount-1; i++)
         newitems[i] = olditems[i];
-    Du_INCREF(x);
     newitems[newcount-1] = x;
-    ob->ob_items = newitems;
-    ob->ob_count = newcount;
-    free(olditems);
+
+    ob->ob_tuple = newitems;
 }
 
 void DuList_Append(DuObject *ob, DuObject *item)
@@ -84,15 +83,13 @@
 
 DuObject *_list_getitem(DuListObject *ob, int index)
 {
-    DuObject *result;
-    int length;
-    DuObject **items;
-    Du_AME_READ(ob, (length = ob->ob_count, items = ob->ob_items));
-    if (index < 0 || index >= length)
+    _du_read1(ob);
+    DuTupleObject *p = ob->ob_tuple;
+
+    _du_read1(p);
+    if (index < 0 || index >= p->ob_count)
         Du_FatalError("list_get: index out of range");
-    result = items[index];
-    Du_INCREF(result);
-    return result;
+    return p->ob_items[index];
 }
 
 DuObject *DuList_GetItem(DuObject *ob, int index)
@@ -103,13 +100,13 @@
 
 void _list_setitem(DuListObject *ob, int index, DuObject *newitem)
 {
-    Du_AME_WRITE(ob);
-    if (index < 0 || index >= ob->ob_count)
+    _du_read1(ob);
+    DuTupleObject *p = ob->ob_tuple;
+
+    _du_write1(p);
+    if (index < 0 || index >= p->ob_count)
         Du_FatalError("list_set: index out of range");
-    DuObject *prev = ob->ob_items[index];
-    Du_INCREF(newitem);
-    ob->ob_items[index] = newitem;
-    Du_DECREF(prev);
+    p->ob_items[index] = newitem;
 }
 
 void DuList_SetItem(DuObject *list, int index, DuObject *newobj)
@@ -120,14 +117,16 @@
 
 DuObject *_list_pop(DuListObject *ob, int index)
 {
-    int i;
-    Du_AME_WRITE(ob);
-    if (index < 0 || index >= ob->ob_count)
+    _du_read1(ob);
+    DuTupleObject *p = ob->ob_tuple;
+
+    _du_write1(p);
+    if (index < 0 || index >= p->ob_count)
         Du_FatalError("list_pop: index out of range");
-    DuObject *result = ob->ob_items[index];
-    ob->ob_count--;
-    for (i=index; i<ob->ob_count; i++)
-        ob->ob_items[i] = ob->ob_items[i+1];
+    DuObject *result = p->ob_items[index];
+    p->ob_count--;
+    for (i=index; i<p->ob_count; i++)
+        p->ob_items[i] = p->ob_items[i+1];
     return result;
 }
 
@@ -137,23 +136,25 @@
     return _list_pop((DuListObject *)list, index);
 }
 
-DuTypeObject DuList_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
+DuType DuList_Type = {
     "list",
+    DUTYPE_LIST,
     sizeof(DuListObject),
-    (destructor_fn)list_free,
     (print_fn)list_print,
     (eval_fn)NULL,
     (len_fn)NULL,
     (len_fn)list_length,
-    (ame_copy_fn)list_ame_copy,
 };
 
+static DuTupleObject du_empty_tuple = {
+    DuOBJECT_HEAD_INIT(DUTYPE_TUPLE),
+    0,
+}
+
 DuObject *DuList_New()
 {
     DuListObject *ob = (DuListObject *)DuObject_New(&DuList_Type);
-    ob->ob_count = 0;
-    ob->ob_items = NULL;
+    ob->ob_tuple = &du_empty_tuple;
     return (DuObject *)ob;
 }
 
@@ -161,5 +162,5 @@
 {
     if (!DuList_Check(ob))
         Du_FatalError("%s: expected 'list' argument, got '%s'",
-                      where, ob->ob_type->dt_name);
+                      where, Du_TYPE(ob)->dt_name);
 }
diff --git a/duhton/object.c b/duhton/object.c
--- a/duhton/object.c
+++ b/duhton/object.c
@@ -2,70 +2,26 @@
 #include "duhton.h"
 
 
-static __thread DuObject chainedlist;
+DuType *Du_Types[_DUTYPE_TOTAL] = {
+    NULL,
+    &DuNone_Type,
+    &DuInt_Type,
+    &DuSymbol_Type,
+    &DuCons_Type,
+    &DuList_Type,
+    &DuFrame_Type,
+    &DuContainer_Type,
+};
 
-DuObject *DuObject_New(DuTypeObject *tp)
+
+DuObject *DuObject_New(DuType *tp)
 {
     assert(tp->dt_size >= sizeof(DuObject));
-    DuObject *ob = malloc(tp->dt_size);
+    DuObject *ob = stm_allocate(tp->dt_size, tp->dt_typeindex);
     assert(ob);
-    ob->ob_refcnt = 1;
-    ob->ob_type = tp;
-    _Du_NewReference(ob);
     return ob;
 }
 
-void _Du_Dealloc(DuObject *ob)
-{
-    assert(ob->ob_refcnt == 1);
-    destructor_fn destructor = ob->ob_type->dt_destructor;
-    assert(destructor != NULL);
-    _Du_ForgetReference(ob);
-    destructor(ob);
-}
-
-#ifdef Du_TRACK_REFS
-void _Du_NewReference(DuObject *ob)
-{
-    if (chainedlist.ob_debug_prev == NULL)
-        Du_FatalError("Du_Initialize() must be called first");
-    ob->ob_debug_prev = &chainedlist;
-    ob->ob_debug_next = chainedlist.ob_debug_next;
-#ifdef Du_AME
-    ob->ob_version = 0;
-#endif
-    chainedlist.ob_debug_next = ob;
-    ob->ob_debug_next->ob_debug_prev = ob;
-}
-
-void _Du_ForgetReference(DuObject *ob)
-{
-    ob->ob_debug_prev->ob_debug_next = ob->ob_debug_next;
-    ob->ob_debug_next->ob_debug_prev = ob->ob_debug_prev;
-    ob->ob_debug_prev = NULL;
-    ob->ob_debug_next = NULL;
-}
-#endif
-
-void _Du_BecomeImmortal(DuObject *ob)
-{
-    _Du_ForgetReference(ob);
-    ob->ob_refcnt = -1;
-}
-
-void type_print(DuTypeObject *ob)
-{
-    printf("<type '%s'>", ob->dt_name);
-}
-
-DuTypeObject DuType_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
-    "type",
-    0,
-    NULL,
-    (print_fn)type_print,
-};
-
 void none_print(DuObject *ob)
 {
     printf("None");
@@ -76,18 +32,17 @@
     return 0;
 }
 
-DuTypeObject DuNone_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
+DuType DuNone_Type = {
     "NoneType",
+    DUTYPE_NONE,
     sizeof(DuObject),
-    NULL,
     none_print,
     (eval_fn)NULL,
     none_is_true,
 };
 
 DuObject _Du_NoneStruct =
-    DuOBJECT_HEAD_INIT(&DuNone_Type);
+    DuOBJECT_HEAD_INIT(DUTYPE_NONE);
 
 void Du_FatalError(char *msg, ...)
 {
@@ -102,63 +57,23 @@
 
 void Du_Print(DuObject *ob, int newline)
 {
-    ob->ob_type->dt_print(ob);
+    Du_TYPE(ob)->dt_print(ob);
     if (newline)
         printf("\n");
 }
 
 int DuObject_IsTrue(DuObject *ob)
 {
-    len_fn fn = ob->ob_type->dt_is_true;
-    if (!fn) fn = ob->ob_type->dt_length;
+    len_fn fn = Du_TYPE(ob)->dt_is_true;
+    if (!fn) fn = Du_TYPE(ob)->dt_length;
     if (!fn) return 1;
     return fn(ob) != 0;
 }
 
 int DuObject_Length(DuObject *ob)
 {
-    if (!ob->ob_type->dt_length)
+    if (!Du_TYPE(ob)->dt_length)
         Du_FatalError("object of type '%s' has no length",
-                      ob->ob_type->dt_name);
-    return ob->ob_type->dt_length(ob);
+                      Du_TYPE(ob)->dt_name);
+    return Du_TYPE(ob)->dt_length(ob);
 }
-
-void _Du_InitializeObjects(void)
-{
-#ifdef Du_TRACK_REFS
-    chainedlist.ob_debug_prev = &chainedlist;
-    chainedlist.ob_debug_next = &chainedlist;
-#endif
-}
-
-void _Du_FinalizeObjects(void)
-{
-#ifdef Du_DEBUG
-    DuObject *obj;
-    for (obj = chainedlist.ob_debug_next;
-         obj != &chainedlist;
-         obj = obj->ob_debug_next) {
-        printf("NOT FREED: ");
-        Du_Print(obj, 1);
-    }
-#endif
-#ifdef Du_TRACK_REFS
-    chainedlist.ob_debug_prev = NULL;
-    chainedlist.ob_debug_next = NULL;
-#endif
-}
-
-#ifdef Du_TRACK_REFS
-void _Du_MakeImmortal(void)
-{
-    DuObject *end_marker = &chainedlist;
-    DuObject *obj = end_marker;
-    do {
-        DuObject *next = obj->ob_debug_next;
-        obj->ob_refcnt = -1;
-        obj->ob_debug_prev = NULL;
-        obj->ob_debug_next = NULL;
-        obj = next;
-    } while (obj != end_marker);
-}
-#endif
diff --git a/duhton/stm/ame.c b/duhton/stm/ame.c
deleted file mode 100644
--- a/duhton/stm/ame.c
+++ /dev/null
@@ -1,466 +0,0 @@
-#include <string.h>
-#include <pthread.h>
-#include "ame.h"
-#include "lists.h"
-
-
-/* XXX assumes that time never wraps around (in a 'long'), which may be
- * correct on 64-bit machines but not on 32-bit machines if the process
- * runs for long enough.
- */
-
-#define IS_LOCKED(num)  ((num) < 0)
-#define IS_LOCKED_OR_NEWER(num, max_age)                        \
-    (((unsigned long)(num)) > ((unsigned long)(max_age)))
-
-
-struct tx_descriptor {
-    jmp_buf *setjmp_buf;
-    int in_transaction;
-    owner_version_t start_time;
-    owner_version_t end_time;
-    owner_version_t my_lock_word;
-    struct OrecList reads;
-    struct RedoLog redolog;   /* last item, because it's the biggest one */
-};
-
-/* global_timestamp contains in its lowest bit a flag equal to 1
-   if there is an inevitable transaction running */
-static volatile owner_version_t global_timestamp = 2;
-static __thread struct tx_descriptor *thread_descriptor;
-
-
-static void common_cleanup(struct tx_descriptor *d)
-{
-    d->reads.size = 0;
-    redolog_clear(&d->redolog);
-    d->in_transaction = 0;
-}
-
-static void tx_spinloop(void)
-{
-    spinloop();
-}
-
-static void tx_abort(void);
-
-static int is_inevitable(struct tx_descriptor *d)
-{
-    assert(d->in_transaction);
-    return d->setjmp_buf == NULL;
-}
-
-
-typedef struct {
-    DuOBJECT_HEAD
-    DuObject *ob_reference;
-} DuContainerObject;
-
-/*** run the redo log to commit a transaction, and release the locks */
-static void tx_redo(struct tx_descriptor *d)
-{
-    owner_version_t newver = d->end_time;
-    wlog_t *item;
-    REDOLOG_LOOP_FORWARD(d->redolog, item)
-    {
-        DuObject *globalobj = item->addr;
-        DuObject *localobj = item->val;
-        long size = localobj->ob_type->dt_size;
-        assert(size >= sizeof(DuObject));
-        memcpy(((char *)globalobj) + sizeof(DuObject),
-               ((char *)localobj) + sizeof(DuObject),
-               size - sizeof(DuObject));
-        CFENCE;
-        globalobj->ob_version = newver;
-
-        //int num = DuInt_AsInt(((DuContainerObject*)localobj)->ob_reference);
-        //printf("COMMIT thread %lx: %p <- %p (%d), v. %ld\n",
-        //        (long)pthread_self(), globalobj, localobj, num, newver);
-
-    } REDOLOG_LOOP_END;
-}
-
-/*** on abort, release locks and restore the old version number. */
-static void releaseAndRevertLocks(struct tx_descriptor *d)
-{
-  wlog_t *item;
-  REDOLOG_LOOP_FORWARD(d->redolog, item)
-  {
-      if (item->p != -1) {
-          DuObject* o = item->addr;
-          o->ob_version = item->p;
-      }
-  } REDOLOG_LOOP_END;
-}
-
-/*** release locks and restore the old version number, ready to retry later */
-static void releaseLocksForRetry(struct tx_descriptor *d)
-{
-    wlog_t *item;
-    REDOLOG_LOOP_FORWARD(d->redolog, item)
-    {
-        if (item->p != -1) {
-            DuObject* o = item->addr;
-            o->ob_version = item->p;
-            item->p = -1;
-        }
-    } REDOLOG_LOOP_END;
-}
-
-/*** lock all locations */
-static void acquireLocks(struct tx_descriptor *d)
-{
-    wlog_t *item;
-    // try to lock every location in the write set
-    REDOLOG_LOOP_BACKWARD(d->redolog, item)
-    {
-        DuObject* o = item->addr;
-        owner_version_t ovt;
-
-    retry:
-        ovt = o->ob_version;
-
-        // if object not locked, lock it
-        //
-        // NB: if ovt > start time, we may introduce inconsistent reads.
-        // Since most writes are also reads, we'll just abort under this
-        // condition.  This can introduce false conflicts
-        if (!IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
-            if (!bool_cas(&o->ob_version, ovt, d->my_lock_word)) {
-                CFENCE;
-                goto retry;
-            }
-            // save old version to item->p.  Now we hold the lock.
-            item->p = ovt;
-        }
-        // else if the location is too recent...
-        else if (!IS_LOCKED(ovt))
-            tx_abort();
-        // else it is locked: check it's not by me (no duplicates in redolog)
-        else {
-            assert(ovt != d->my_lock_word);
-            // we can either abort or spinloop.  Because we are at the end of
-            // the transaction we might try to spinloop, even though after the
-            // lock is released the ovt will be very recent, possibly greater
-            // than d->start_time.  It is necessary to spinloop in case we are
-            // inevitable, so use that as a criteria.  Another solution to
-            // avoid deadlocks would be to sort the order in which we take the
-            // locks.
-            if (is_inevitable(d))
-                tx_spinloop();
-            else
-                tx_abort();
-            goto retry;
-        }
-    } REDOLOG_LOOP_END;
-}
-
-/**
- * fast-path validation, assuming that I don't hold locks.
- */
-static void validate_fast(struct tx_descriptor *d)
-{
-    int i;
-    owner_version_t ovt;
-    assert(!is_inevitable(d));
-    for (i=0; i<d->reads.size; i++) {
-     retry:
-        ovt = d->reads.items[i]->ob_version;
-        if (IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
-            // If locked, we wait until it becomes unlocked.  The chances are
-            // that it will then have a very recent start_time, likely greater
-            // than d->start_time, but it might still be better than always
-            // aborting
-            if (IS_LOCKED(ovt)) {
-                tx_spinloop();
-                goto retry;
-            }
-            else
-                // abort if the timestamp is newer than my start time.  
-                tx_abort();
-        }
-    }
-}
-
-/**
- * validate the read set by making sure that all orecs that we've read have
- * timestamps at least as old as our start time, unless we locked those orecs.
- */
-static void validate(struct tx_descriptor *d)
-{
-    int i;
-    owner_version_t ovt;
-    assert(!is_inevitable(d));
-    for (i=0; i<d->reads.size; i++) {
-        ovt = d->reads.items[i]->ob_version;
-        if (IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
-            if (!IS_LOCKED(ovt)) {
-                // if unlocked and newer than start time, abort
-                tx_abort();
-            }
-            else {
-                // if locked and not by me, abort
-                if (ovt != d->my_lock_word)
-                    tx_abort();
-            }
-        }
-    }
-}
-
-static void wait_end_inevitability(struct tx_descriptor *d)
-{
-    owner_version_t curts;
-    releaseLocksForRetry(d);
-
-    // We are going to wait until the other inevitable transaction
-    // finishes.  XXX We could do better here: we could check if
-    // committing 'd' would create a conflict for the other inevitable
-    // thread 'd_inev' or not.  It requires peeking in 'd_inev' from this
-    // thread (which we never do so far) in order to do something like
-    // 'validate_fast(d_inev); d_inev->start_time = updated;'
-
-    while ((curts = global_timestamp) & 1) {
-        // while we're about to wait anyway, we can do a validate_fast
-        if (d->start_time < curts - 1) {
-            validate_fast(d);
-            d->start_time = curts - 1;
-        }
-        tx_spinloop();
-    }
-    acquireLocks(d);
-}
-
-static void commitInevitableTransaction(struct tx_descriptor *d)
-{
-    owner_version_t ts;
-
-    // no-one else can modify global_timestamp if I'm inevitable
-    ts = global_timestamp;
-    assert(ts & 1);
-    global_timestamp = ts + 1;
-    d->end_time = ts + 1;
-    assert(d->end_time == (d->start_time + 2));
-
-    // run the redo log, and release the locks
-    tx_redo(d);
-}
-
-static void tx_abort(void)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    assert(!is_inevitable(d));
-    // release the locks and restore version numbers
-    releaseAndRevertLocks(d);
-    // reset all lists
-    common_cleanup(d);
-
-    tx_spinloop();
-    longjmp(*d->setjmp_buf, 1);
-}
-
-
-static inline owner_version_t prepare_read(struct tx_descriptor *d,
-                                           DuObject *glob)
-{
-    owner_version_t ovt;
- retry:
-    ovt = glob->ob_version;
-    if (IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
-        if (IS_LOCKED(ovt)) {
-            tx_spinloop();
-            goto retry;
-        }
-        /* else this location is too new, scale forward */
-        owner_version_t newts = global_timestamp & ~1;
-        validate_fast(d);
-        d->start_time = newts;
-    }
-    return ovt;
-}
-
-DuObject *_Du_AME_read_from_global(DuObject *glob, owner_version_t *vers_out)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    wlog_t *found;
-    REDOLOG_FIND(d->redolog, glob, found, goto not_found);
-    DuObject *localobj = found->val;
-    assert(!Du_AME_GLOBAL(localobj));
-    return localobj;
-
- not_found:
-    *vers_out = prepare_read(d, glob);
-    return glob;
-}
-
-void _Du_AME_oreclist_insert(DuObject *glob)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    oreclist_insert(&d->reads, glob);
-}
-
-DuObject *_Du_AME_writebarrier(DuObject *glob)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    if (!d->in_transaction)
-        return glob;
-
-    DuObject *localobj;
-    wlog_t* found;
-    REDOLOG_FIND(d->redolog, glob, found, goto not_found);
-    localobj = found->val;
-    assert(!Du_AME_GLOBAL(localobj));
-    return localobj;
-
- not_found:;
-    /* We need to really make a local copy */
-    owner_version_t version;
-    int size = glob->ob_type->dt_size;
-    localobj = malloc(size);
-    assert(localobj != NULL);
-
-    do {
-        version = prepare_read(d, glob);
-        CFENCE;
-        /* Initialize the copy by doing an stm raw copy of the bytes */
-        memcpy((char *)localobj, (char *)glob, size);
-        CFENCE;
-        /* Check the copy for validity */
-    } while (glob->ob_version != version);
-
-    /* Initialize the copy's refcount to be a valid local object */
-    localobj->ob_refcnt = 42;   /* XXX */
-    /* Set the ob_debug_prev field to NULL and the ob_debug_next field to
-       point to the global object */
-    localobj->ob_debug_prev = NULL;
-    localobj->ob_debug_next = glob;
-    /* Ask for additional type-specific copy */
-    if (localobj->ob_type->dt_ame_copy)
-        localobj->ob_type->dt_ame_copy(localobj);
-    /* Register the object as a valid copy */
-    redolog_insert(&d->redolog, glob, localobj);
-    return localobj;
-}
-
-DuObject *_Du_AME_getlocal(DuObject *glob)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    DuObject *localobj;
-    wlog_t* found;
-    REDOLOG_FIND(d->redolog, glob, found, return glob);
-    localobj = found->val;
-    assert(!Du_AME_GLOBAL(localobj));
-    return localobj;
-}
-
-void _Du_AME_InitThreadDescriptor(void)
-{
-    assert(thread_descriptor == NULL);
-    struct tx_descriptor *d = malloc(sizeof(struct tx_descriptor));
-    memset(d, 0, sizeof(struct tx_descriptor));
-
-    /* initialize 'my_lock_word' to be a unique negative number */
-    d->my_lock_word = (owner_version_t)d;
-    if (!IS_LOCKED(d->my_lock_word))
-        d->my_lock_word = ~d->my_lock_word;
-    assert(IS_LOCKED(d->my_lock_word));
-
-    common_cleanup(d);
-    thread_descriptor = d;
-}
-
-void _Du_AME_FiniThreadDescriptor(void)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    assert(d);
-    thread_descriptor = NULL;
-    free(d);
-}
-
-void _Du_AME_StartTransaction(jmp_buf *setjmp_buf)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    assert(!d->in_transaction);
-    d->setjmp_buf = setjmp_buf;
-    d->in_transaction = 1;
-    d->start_time = global_timestamp & ~1;
-}
-
-void _Du_AME_CommitTransaction(void)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    assert(d->in_transaction);
-
-    // if I don't have writes, I'm committed
-    if (!redolog_any_entry(&d->redolog)) {
-        if (is_inevitable(d)) {
-            owner_version_t ts = global_timestamp;
-            assert(ts & 1);
-            global_timestamp = ts - 1;
-        }
-        common_cleanup(d);
-        return;
-    }
-
-    //printf("COMMIT thread %lx: START\n",
-    //        (long)pthread_self());
-
-    // acquire locks
-    acquireLocks(d);
-
-    if (is_inevitable(d)) {
-        commitInevitableTransaction(d);
-    }
-    else {
-        while (1) {
-            owner_version_t expected = global_timestamp;
-            if (expected & 1) {
-                // wait until it is done.  hopefully we can then proceed
-                // without conflicts.
-                wait_end_inevitability(d);
-                continue;
-            }
-            if (bool_cas(&global_timestamp, expected, expected + 2)) {
-                d->end_time = expected + 2;
-                break;
-            }
-        }
-
-        // validate (but skip validation if nobody else committed)
-        if (d->end_time != (d->start_time + 2))
-            validate(d);
-
-        // run the redo log, and release the locks
-        tx_redo(d);
-    }
-
-    //printf("COMMIT thread %lx: DONE\n",
-    //        (long)pthread_self());
-
-    common_cleanup(d);
-}
-
-void Du_AME_TryInevitable(void)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    if (!d->in_transaction || is_inevitable(d))
-        return;
-
-    while (1) {
-        owner_version_t curtime = global_timestamp;
-        if (d->start_time != (curtime & ~1)) {
-            /* scale forward */
-            validate_fast(d);
-            d->start_time = curtime & ~1;
-        }
-        if (curtime & 1) { /* there is, or was, already an inevitable thread */
-            /* should we spinloop here, or abort (and likely come back
-               in try_inevitable() very soon)?  unclear.  For now
-               let's try to spinloop. */
-            tx_spinloop();
-            continue;
-        }
-        if (bool_cas(&global_timestamp, curtime, curtime + 1))
-            break;
-    }
-    d->setjmp_buf = NULL;   /* inevitable from now on */
-}
diff --git a/duhton/stm/ame.h b/duhton/stm/ame.h
deleted file mode 100644
--- a/duhton/stm/ame.h
+++ /dev/null
@@ -1,62 +0,0 @@
-#ifndef _DUHTON_AME_H_
-#define _DUHTON_AME_H_
-
-#include <setjmp.h>
-#include "atomic_ops.h"
-#include "../duhton.h"
-
-
-DuObject *_Du_AME_read_from_global(DuObject *glob, owner_version_t *vers_out);
-void _Du_AME_oreclist_insert(DuObject *glob);
-DuObject *_Du_AME_writebarrier(DuObject *glob);
-DuObject *_Du_AME_getlocal(DuObject *glob);
-
-void _Du_AME_InitThreadDescriptor(void);
-void _Du_AME_FiniThreadDescriptor(void);
-void _Du_AME_StartTransaction(jmp_buf *);
-void _Du_AME_CommitTransaction(void);
-
-void Du_AME_TryInevitable(void);
-
-
-#define Du_AME_READ(ob, READ_OPERATIONS)                                \
-    while (1) {                                                         \
-        DuObject *__du_ame_ob = (DuObject *)(ob);                       \
-        if (Du_AME_GLOBAL(__du_ame_ob)) {                               \
-            owner_version_t __du_ame_version;                           \
-            __du_ame_ob = _Du_AME_read_from_global(__du_ame_ob,         \
-                                                   &__du_ame_version);  \
-            (ob) = (typeof(ob))__du_ame_ob;                             \
-            if (Du_AME_GLOBAL(__du_ame_ob)) {                           \
-                CFENCE;                                                 \
-                READ_OPERATIONS;                                        \
-                CFENCE;                                                 \
-                __du_ame_ob = (DuObject *)(ob);                         \
-                if (__du_ame_ob->ob_version != __du_ame_version)        \
-                    continue;                                           \
-                _Du_AME_oreclist_insert(__du_ame_ob);                   \
-                break;                                                  \
-            }                                                           \
-        }                                                               \
-        READ_OPERATIONS;                                                \
-        break;                                                          \
-    }
-
-
-#define Du_AME_WRITE(ob)                                                \
-    do {                                                                \
-        if (Du_AME_GLOBAL(ob))                                          \
-            (ob) = (typeof(ob))_Du_AME_writebarrier((DuObject *)(ob));  \
-    } while (0)
-
-
-#define Du_AME_INEVITABLE(ob)                                           \
-    do {                                                                \
-        Du_AME_TryInevitable();                                         \
-        if (Du_AME_GLOBAL(ob))                                          \
-            (ob) = (typeof(ob))_Du_AME_getlocal((DuObject *)(ob));      \
-    } while (0)
-/* XXX must also wait for ob to be unlocked, if necessary */
-
-
-#endif  /* _DUHTON_AME_H_ */
diff --git a/duhton/stm/atomic_ops.h b/duhton/stm/atomic_ops.h
deleted file mode 100644
--- a/duhton/stm/atomic_ops.h
+++ /dev/null
@@ -1,52 +0,0 @@
-#ifndef _SRCSTM_ATOMIC_OPS_
-#define _SRCSTM_ATOMIC_OPS_
-
-
-/* "compiler fence" for preventing reordering of loads/stores to
-   non-volatiles */
-#define CFENCE          asm volatile ("":::"memory")
-
-
-#ifdef __llvm__
-#  define HAS_SYNC_BOOL_COMPARE_AND_SWAP
-#endif
-
-#ifdef __GNUC__
-#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
-#    define HAS_SYNC_BOOL_COMPARE_AND_SWAP
-#  endif
-#endif
-
-
-#ifdef HAS_SYNC_BOOL_COMPARE_AND_SWAP
-#  define bool_cas __sync_bool_compare_and_swap
-#else
-/* x86 (32 bits and 64 bits) */
-static inline _Bool
-bool_cas(volatile long* ptr, long old, long _new)
-{
-    long prev;
-    asm volatile("lock;"
-#if defined(__amd64__)
-                 "cmpxchgq %1, %2;"
-#else
-                 "cmpxchgl %1, %2;"
-#endif
-                 : "=a"(prev)
-                 : "q"(_new), "m"(*ptr), "a"(old)
-                 : "memory");
-    return prev == old;
-}
-/* end */
-#endif
-
-
-static inline void spinloop(void)
-{
-  /* use "memory" here to make sure that gcc will reload the
-     relevant data from memory after the spinloop */
-  asm volatile ("pause":::"memory");
-}
-
-
-#endif  /* _SRCSTM_ATOMIC_OPS_ */
diff --git a/duhton/stm/fifo.c b/duhton/stm/fifo.c
deleted file mode 100644
--- a/duhton/stm/fifo.c
+++ /dev/null
@@ -1,51 +0,0 @@
-#include "fifo.h"
-
-
-void fifo_init(fifo_t *self)
-{
-    self->ff_first = NULL;
-    self->ff_last = NULL;
-}
-
-void fifo_append(fifo_t *self, fifonode_t *newnode)
-{
-    newnode->fn_next = NULL;
-    if (self->ff_last == NULL)
-        self->ff_first = newnode;
-    else
-        self->ff_last->fn_next = newnode;
-    self->ff_last = newnode;
-}
-
-int fifo_is_empty(fifo_t *self)
-{
-    assert((self->ff_first == NULL) == (self->ff_last == NULL));
-    return (self->ff_first == NULL);
-}
-
-int fifo_is_of_length_1(fifo_t *self)
-{
-    return (self->ff_first != NULL && self->ff_first == self->ff_last);
-}
-
-fifonode_t *fifo_pop_left(fifo_t *self)
-{
-    fifonode_t *item = self->ff_first;
-    self->ff_first = item->fn_next;
-    if (self->ff_first == NULL)
-        self->ff_last = NULL;
-    return item;
-}
-
-void fifo_steal(fifo_t *self, fifo_t *other)
-{
-    if (other->ff_last != NULL) {
-        if (self->ff_last == NULL)
-            self->ff_first = other->ff_first;
-        else
-            self->ff_last->fn_next = other->ff_first;
-        self->ff_last = other->ff_last;
-        other->ff_first = NULL;
-        other->ff_last = NULL;
-    }
-}
diff --git a/duhton/stm/fifo.h b/duhton/stm/fifo.h
deleted file mode 100644
--- a/duhton/stm/fifo.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#include "../duhton.h"
-
-
-typedef struct {
-    struct fifonode_s *ff_first;
-    struct fifonode_s *ff_last;
-} fifo_t;
-
-typedef struct fifonode_s {
-    struct fifonode_s *fn_next;
-    DuObject *fn_frame;
-    DuObject *fn_code;
-} fifonode_t;
-
-
-#define FIFO_INITIALIZER  { NULL, NULL }
-
-void fifo_init(fifo_t *self);
-void fifo_append(fifo_t *self, fifonode_t *newnode);
-int fifo_is_empty(fifo_t *self);
-int fifo_is_of_length_1(fifo_t *self);
-fifonode_t *fifo_pop_left(fifo_t *self);
-void fifo_steal(fifo_t *self, fifo_t *other);
diff --git a/duhton/stm/lists.h b/duhton/stm/lists.h
deleted file mode 100644
--- a/duhton/stm/lists.h
+++ /dev/null
@@ -1,240 +0,0 @@
-/* -*- c-basic-offset: 2 -*- */
-
-#include <limits.h>
-
-/************************************************************/
-
-/* The redolog_xx functions are implemented as a tree, supporting
-   very high performance in REDOLOG_FIND in the common case where
-   there are no or few elements in the tree, but scaling correctly
-   if the number of items becomes large. */
-
-#define TREE_BITS   4
-#define TREE_ARITY  (1 << TREE_BITS)
-
-#define TREE_DEPTH_MAX   ((sizeof(DuObject*)*8 - 2 + TREE_BITS-1) / TREE_BITS)
-/* sizeof(void*) = total number of bits
-   2 = bits that we ignore anyway (2 or 3, conservatively 2)
-   (x + TREE_BITS-1) / TREE_BITS = divide by TREE_BITS, rounding up
-*/
-
-#define TREE_MASK   ((TREE_ARITY - 1) * sizeof(DuObject*))
-
-typedef struct {
-  DuObject *addr;
-  DuObject *val;
-  owner_version_t p;   // the previous version number (if locked)
-} wlog_t;
-
-typedef struct {
-  char *items[TREE_ARITY];
-} wlog_node_t;
-
-struct RedoLog {
-  char *raw_start, *raw_current, *raw_end;
-  wlog_node_t toplevel;
-};
-
-static void _redolog_clear_node(wlog_node_t *node)
-{
-  memset(node, 0, sizeof(wlog_node_t));
-}
-
-static void redolog_clear(struct RedoLog *redolog)
-{
-  if (redolog->raw_current != redolog->raw_start)
-    {
-      _redolog_clear_node(&redolog->toplevel);
-      redolog->raw_current = redolog->raw_start;
-    }
-}
-
-static int redolog_any_entry(struct RedoLog *redolog)
-{
-  return redolog->raw_current != redolog->raw_start;
-}
-
-#define _REDOLOG_LOOP(redolog, item, INITIAL, _PLUS_)                   \
-{                                                                       \
-  struct { char **next; char **end; } _stack[TREE_DEPTH_MAX], *_stackp; \
-  char **_next, **_end, *_entry;                                        \
-  /* initialization */                                                  \
-  _stackp = _stack;      /* empty stack */                              \
-  _next = (redolog).toplevel.items + INITIAL;                           \
-  _end = _next _PLUS_ TREE_ARITY;                                       \
-  /* loop */                                                            \
-  while (1)                                                             \
-    {                                                                   \
-      if (_next == _end)                                                \
-        {                                                               \
-          if (_stackp == _stack)                                        \
-            break;   /* done */                                         \
-          /* finished with this level, go to the next one */            \
-          _stackp--;                                                    \
-          _next = _stackp->next;                                        \
-          _end = _stackp->end;                                          \
-          continue;                                                     \
-        }                                                               \
-      _entry = *_next;                                                  \
-      _next = _next _PLUS_ 1;                                           \
-      if (_entry == NULL)   /* empty entry */                           \
-        continue;                                                       \
-      if (((long)_entry) & 1)                                           \
-        {  /* points to a further level: enter it */                    \
-          _stackp->next = _next;                                        \
-          _stackp->end = _end;                                          \
-          _stackp++;                                                    \
-          _next = ((wlog_node_t *)(_entry - 1))->items + INITIAL;       \
-          _end = _next _PLUS_ TREE_ARITY;                               \
-          continue;                                                     \
-        }                                                               \
-      /* points to a wlog_t item */                                     \
-      item = (wlog_t *)_entry;
-
-#define REDOLOG_LOOP_FORWARD(redolog, item)                             \
-                       _REDOLOG_LOOP(redolog, item, 0, +)
-#define REDOLOG_LOOP_BACKWARD(redolog, item)                            \
-                       _REDOLOG_LOOP(redolog, item, (TREE_ARITY-1), -)
-#define REDOLOG_LOOP_END     } }
-
-#define REDOLOG_FIND(redolog, addr1, result, goto_not_found)    \
-{                                                               \
-  unsigned long _key = (unsigned long)(addr1);                  \
-  char *_p = (char *)((redolog).toplevel.items);                \
-  char *_entry = *(char **)(_p + (_key & TREE_MASK));           \
-  if (__builtin_expect(_entry == NULL, 1))                      \
-    goto_not_found;    /* common case, hopefully */             \
-  result = _redolog_find(_entry, addr1);                        \
-  if (result == NULL || result->addr != (addr1))                \
-    goto_not_found;                                             \
-}
-
-static wlog_t *_redolog_find(char *entry, DuObject *addr)
-{
-  unsigned long key = (unsigned long)addr;
-  while (((long)entry) & 1)
-    {   /* points to a further level */
-      key >>= TREE_BITS;
-      entry = *(char **)((entry - 1) + (key & TREE_MASK));
-    }
-  return (wlog_t *)entry;   /* may be NULL */
-}
-
-static void redolog_insert(struct RedoLog *redolog,
-                           DuObject *addr, DuObject *val);
-
-static void _redolog_grow(struct RedoLog *redolog, long extra)
-{
-  struct RedoLog newredolog;
-  wlog_t *item;
-  long alloc = redolog->raw_end - redolog->raw_start;
-  long newalloc = (alloc + extra + (alloc >> 2) + 31) & ~15;
-  //fprintf(stderr, "growth: %ld\n", newalloc);
-  char *newitems = malloc(newalloc);
-  newredolog.raw_start = newitems;
-  newredolog.raw_current = newitems;
-  newredolog.raw_end = newitems + newalloc;
-  _redolog_clear_node(&newredolog.toplevel);
-  REDOLOG_LOOP_FORWARD(*redolog, item)
-    {
-      assert(item->p == -1);
-      redolog_insert(&newredolog, item->addr, item->val);
-    } REDOLOG_LOOP_END;
-  free(redolog->raw_start);
-  *redolog = newredolog;
-}
-
-static char *_redolog_grab(struct RedoLog *redolog, long size)
-{
-  char *result;
-  result = redolog->raw_current;
-  redolog->raw_current += size;
-  if (redolog->raw_current > redolog->raw_end)
-    {
-      _redolog_grow(redolog, size);
-      return NULL;
-    }
-  return result;
-}
-
-static void redolog_insert(struct RedoLog *redolog,
-                           DuObject *addr, DuObject *val)
-{
- retry:;
-  wlog_t *wlog;
-  unsigned long key = (unsigned long)addr;
-  int shift = 0;
-  char *p = (char *)(redolog->toplevel.items);
-  char *entry;
-  assert((key & (sizeof(DuObject*)-1)) == 0);   /* only for aligned keys */
-  while (1)
-    {
-      p += (key >> shift) & TREE_MASK;
-      shift += TREE_BITS;
-      entry = *(char **)p;
-      if (entry == NULL)
-        break;
-      else if (((long)entry) & 1)
-        {   /* points to a further level */
-          p = entry - 1;
-        }
-      else
-        {
-          wlog_t *wlog1 = (wlog_t *)entry;
-          /* the key must not already be present */
-          assert(wlog1->addr != addr);
-          /* collision: there is already a different wlog here */
-          wlog_node_t *node = (wlog_node_t *)
-                _redolog_grab(redolog, sizeof(wlog_node_t));
-          if (node == NULL) goto retry;
-          _redolog_clear_node(node);
-          unsigned long key1 = (unsigned long)(wlog1->addr);
-          char *p1 = (char *)(node->items);
-          *(wlog_t **)(p1 + ((key1 >> shift) & TREE_MASK)) = wlog1;
-          *(char **)p = ((char *)node) + 1;
-          p = p1;
-        }
-    }
-  wlog = (wlog_t *)_redolog_grab(redolog, sizeof(wlog_t));
-  if (wlog == NULL) goto retry;
-  wlog->addr = addr;
-  wlog->val = val;
-  wlog->p = -1;
-  *(char **)p = (char *)wlog;
-}
-
-/************************************************************/
-
-/* The oreclist_xx functions are implemented as an array that grows
-   as needed. */
-
-struct OrecList {
-  long size, alloc;
-  long locked;
-  DuObject **items;
-};
-
-static void _oreclist_grow(struct OrecList *oreclist)
-{
-  long newalloc = oreclist->alloc + (oreclist->alloc >> 1) + 16;
-  DuObject **newitems = malloc(newalloc * sizeof(DuObject *));
-  long i;
-  for (i=0; i<oreclist->size; i++)
-    newitems[i] = oreclist->items[i];
-  while (!bool_cas(&oreclist->locked, 0, 1))
-    /* rare case */ ;
-  free(oreclist->items);
-  oreclist->items = newitems;
-  oreclist->alloc = newalloc;
-  CFENCE;
-  oreclist->locked = 0;
-}
-
-static void oreclist_insert(struct OrecList *oreclist, DuObject *newitem)
-{
-  if (oreclist->size == oreclist->alloc)
-    _oreclist_grow(oreclist);
-  oreclist->items[oreclist->size++] = newitem;
-}
-
-/************************************************************/
diff --git a/duhton/stm/thread.c b/duhton/stm/thread.c
deleted file mode 100644
--- a/duhton/stm/thread.c
+++ /dev/null
@@ -1,173 +0,0 @@
-#include <pthread.h>
-#include <assert.h>
-#include "../duhton.h"
-#include "fifo.h"
-
-
-#define NUMTHREADS    2
-
-
-static int num_waiting_threads;
-static int finished;
-static pthread_mutex_t ll_state = PTHREAD_MUTEX_INITIALIZER;
-static pthread_mutex_t ll_no_tasks_pending = PTHREAD_MUTEX_INITIALIZER;
-static fifo_t global_list_pending = FIFO_INITIALIZER;
-static __thread fifo_t *local_list_pending = &global_list_pending;
-
-static void lock(pthread_mutex_t *mutex)
-{
-    if (pthread_mutex_lock(mutex) != 0)
-        Du_FatalError("lock() failed??");
-}
-
-static void unlock(pthread_mutex_t *mutex)
-{
-    if (pthread_mutex_unlock(mutex) != 0)
-        Du_FatalError("unlock() failed: the lock is not acquired?");
-}
-
-static int is_locked(pthread_mutex_t *mutex)
-{
-    /* Test a lock for debugging. */
-    if (pthread_mutex_trylock(mutex) != 0)
-        return 1;
-    unlock(mutex);
-    return 0;
-}
-
-void Du_TransactionAdd(DuObject *code, DuObject *frame)
-{
-    fifonode_t *fnode = malloc(sizeof(fifonode_t));
-    fnode->fn_code = code;   Du_INCREF(code);
-    fnode->fn_frame = frame; Du_INCREF(frame);
-    fifo_append(local_list_pending, fnode);
-}
-
-
-//static pthread_mutex_t tmp_GIL = PTHREAD_MUTEX_INITIALIZER;
-
-static void execute_fifo_node(fifonode_t *fnode)
-{
-    jmp_buf env;
-    assert(fifo_is_empty(local_list_pending));
-    //lock(&tmp_GIL);
-    if (setjmp(env) == 0) {
-        _Du_AME_StartTransaction(&env);
-        _Du_InitializeObjects();
-        DuObject *framecopy = DuFrame_Copy(fnode->fn_frame);
-        DuObject *res = Du_Progn(fnode->fn_code, framecopy);
-        Du_DECREF(res);
-        Du_DECREF(framecopy);
-        Du_DECREF(fnode->fn_frame);
-        Du_DECREF(fnode->fn_code);
-        _Du_MakeImmortal();
-        _Du_AME_CommitTransaction();
-        free(fnode);
-    }
-    else {    /* transaction aborted, re-schedule fnode for later */
-        fifo_t *local_list = local_list_pending;
-        fifo_init(local_list);
-        fifo_append(local_list, fnode);
-    }
-    //unlock(&tmp_GIL);
-}
-
-static void _add_list(fifo_t *new_pending_list)
-{
-    if (fifo_is_empty(new_pending_list))
-        return;
-    int was_empty = fifo_is_empty(&global_list_pending);
-    fifo_steal(&global_list_pending, new_pending_list);
-    if (was_empty)
-        unlock(&ll_no_tasks_pending);
-}
-
-static void *_run_thread(void *arg)
-{
-    lock(&ll_state);
-    _Du_AME_InitThreadDescriptor();
-    fifo_t my_transactions_pending = FIFO_INITIALIZER;
-    local_list_pending = &my_transactions_pending;
-
-    while (1) {
-        if (fifo_is_empty(&global_list_pending)) {
-            assert(is_locked(&ll_no_tasks_pending));
-            num_waiting_threads++;
-            if (num_waiting_threads == NUMTHREADS) {
-                finished = 1;
-                unlock(&ll_no_tasks_pending);
-            }
-            unlock(&ll_state);
-
-            lock(&ll_no_tasks_pending);
-            unlock(&ll_no_tasks_pending);
-
-            lock(&ll_state);
-            num_waiting_threads--;
-            if (finished)
-                break;
-        }
-        else {
-            fifonode_t *pending = fifo_pop_left(&global_list_pending);
-            if (fifo_is_empty(&global_list_pending))
-                lock(&ll_no_tasks_pending);
-            unlock(&ll_state);
-
-            while (1) {
-                execute_fifo_node(pending);
-                /* for now, always break out of this loop, unless
-                   'my_transactions_pending' contains precisely one item */
-                if (!fifo_is_of_length_1(&my_transactions_pending))
-                    break;
-                pending = fifo_pop_left(&my_transactions_pending);
-            }
-
-            lock(&ll_state);
-            _add_list(&my_transactions_pending);
-        }
-    }
-
-    local_list_pending = NULL;
-    _Du_AME_FiniThreadDescriptor();
-    unlock(&ll_state);
-    return NULL;
-}
-
-static void _run(void)
-{
-    pthread_t th[NUMTHREADS];
-    int i;
-    for (i=0; i<NUMTHREADS; i++) {
-        int status = pthread_create(&th[i], NULL, _run_thread, NULL);
-        assert(status == 0);
-    }
-    for (i=0; i<NUMTHREADS; i++) {
-        void *result;
-        int status = pthread_join(th[i], &result);
-        assert(status == 0);
-        assert(result == NULL);
-    }
-}
-
-void Du_TransactionRun(void)
-{
-    assert(!is_locked(&ll_state));
-    assert(!is_locked(&ll_no_tasks_pending));
-    if (fifo_is_empty(&global_list_pending))
-        return;
-
-    _Du_MakeImmortal();
-
-    num_waiting_threads = 0;
-    finished = 0;
-
-    _run();
-
-    assert(finished);
-    assert(num_waiting_threads == 0);
-    assert(fifo_is_empty(&global_list_pending));
-    assert(!is_locked(&ll_state));
-    assert(!is_locked(&ll_no_tasks_pending));
-
-    _Du_InitializeObjects();
-}
diff --git a/duhton/symbol.c b/duhton/symbol.c
--- a/duhton/symbol.c
+++ b/duhton/symbol.c
@@ -7,18 +7,27 @@
     struct _Du_Symbol *next;
 } DuSymbolObject;
 
-DuSymbolObject *_Du_AllSymbols = NULL;
+static DuSymbolObject _Du_AllSymbols = {
+    DuOBJECT_HEAD_INIT(DUTYPE_SYMBOL),
+    "",
+    NULL};
+
 
 void symbol_print(DuSymbolObject *ob)
 {
+    _du_read1(ob);
     printf("'%s'", ob->name);
 }
 
 DuObject *symbol_eval(DuObject *ob, DuObject *locals)
 {
+    _du_save1(ob);
     DuObject *res = DuFrame_GetSymbol(locals, ob);
+    _du_restore1(ob);
     if (res == NULL) {
+        _du_save1(ob);
         res = DuFrame_GetSymbol(Du_Globals, ob);
+        _du_restore1(ob);
         if (res == NULL)
             Du_FatalError("symbol not defined as a variable: '%s'",
                           DuSymbol_AsString(ob));
@@ -26,36 +35,37 @@
     return res;
 }
 
-DuTypeObject DuSymbol_Type = {
-    DuOBJECT_HEAD_INIT(&DuType_Type),
+DuType DuSymbol_Type = {
     "symbol",
+    DUTYPE_SYMBOL,
     sizeof(DuSymbolObject),
-    (destructor_fn)NULL,
     (print_fn)symbol_print,
     (eval_fn)symbol_eval,
 };
 
 DuObject *DuSymbol_FromString(char *name)
 {
-    DuSymbolObject *p;
-    for (p=_Du_AllSymbols; p != NULL; p=p->next) {
+    DuSymbolObject *p, *head = &_Du_AllSymbols;
+    for (p=head; p != NULL; p=p->next) {
+        _du_read1(p);
         if (strcmp(name, p->name) == 0) {
-            Du_INCREF(p);
             return (DuObject *)p;
         }
     }
     p = (DuSymbolObject *)DuObject_New(&DuSymbol_Type);
-    _Du_BecomeImmortal((DuObject *)p);    /* make the symbol object immortal */
     p->name = strdup(name);
-    p->next = _Du_AllSymbols;

From noreply at buildbot.pypy.org  Sun Jun 23 19:38:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Jun 2013 19:38:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Silence the strangest RPython warning
 so far: "prebuilt instance
Message-ID: <20130623173808.AB4DE1C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64958:a2aada708ad4
Date: 2013-06-23 19:37 +0200
http://bitbucket.org/pypy/pypy/changeset/a2aada708ad4/

Log:	Silence the strangest RPython warning so far: "prebuilt instance
	Ellipsis has no attribute 'errno'"

diff --git a/rpython/rtyper/lltypesystem/rclass.py b/rpython/rtyper/lltypesystem/rclass.py
--- a/rpython/rtyper/lltypesystem/rclass.py
+++ b/rpython/rtyper/lltypesystem/rclass.py
@@ -442,8 +442,10 @@
                     except AttributeError:
                         attrvalue = self.classdef.classdesc.read_attribute(name, None)
                         if attrvalue is None:
-                            warning("prebuilt instance %r has no attribute %r" % (
-                                    value, name))
+                            # Ellipsis from get_reusable_prebuilt_instance()
+                            if value is not Ellipsis:
+                                warning("prebuilt instance %r has no "
+                                        "attribute %r" % (value, name))
                             llattrvalue = r.lowleveltype._defl()
                         else:
                             llattrvalue = r.convert_desc_or_const(attrvalue)

From noreply at buildbot.pypy.org  Sun Jun 23 21:05:34 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Jun 2013 21:05:34 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130623190534.121231C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r247:ad70dac44a95
Date: 2013-06-23 21:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/ad70dac44a95/

Log:	in-progress

diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -56,10 +56,12 @@
    this call. */
 void stm_perform_transaction(gcptr arg, int (*callback)(gcptr, int));
 
-/* finish the current transaction, start a new one.  Must save roots
-   around calls to these two functions. */
+/* finish the current transaction, start a new one, or turn the current
+   transaction inevitable.  Must save roots around calls to these three
+   functions. */
 void stm_commit_transaction(void);
 void stm_begin_inevitable_transaction(void);
+void stm_become_inevitable(const char *reason);
 
 /* debugging: check if we're currently running a transaction or not. */
 int stm_in_transaction(void);
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -203,6 +203,11 @@
         BeginInevitableTransaction();
 }
 
+void stm_become_inevitable(const char *reason)
+{
+    BecomeInevitable(reason);
+}
+
 int stm_in_transaction(void)
 {
     struct tx_descriptor *d = thread_descriptor;
diff --git a/duhton/consobject.c b/duhton/consobject.c
--- a/duhton/consobject.c
+++ b/duhton/consobject.c
@@ -70,14 +70,12 @@
 DuObject *_DuCons_CAR(DuObject *cons)
 {
     assert(DuCons_Check(cons));
-    _du_read1(cons);
     return ((DuConsObject *)cons)->car;
 }
 
 DuObject *_DuCons_NEXT(DuObject *cons)
 {
     assert(DuCons_Check(cons));
-    _du_read1(cons);
     DuObject *result = ((DuConsObject *)cons)->cdr;
     if (result != Du_None && !DuCons_Check(cons))
         Du_FatalError("_DuCons_NEXT: not a well-formed cons list");
diff --git a/duhton/containerobject.c b/duhton/containerobject.c
--- a/duhton/containerobject.c
+++ b/duhton/containerobject.c
@@ -5,16 +5,6 @@
     DuObject *ob_reference;
 } DuContainerObject;
 
-void container_free(DuContainerObject *ob)
-{
-    DuObject *x = ob->ob_reference;
-#ifdef Du_DEBUG
-    ob->ob_reference = (DuObject *)0xDD;
-#endif
-    free(ob);
-    Du_DECREF(x);
-}
-
 void container_print(DuContainerObject *ob)
 {
     printf("<container ");
@@ -24,24 +14,18 @@
 
 DuObject *DuContainer_GetRef(DuObject *ob)
 {
-    DuObject *result;
     DuContainer_Ensure("DuContainer_GetRef", ob);
 
-    Du_AME_READ(ob, (result = ((DuContainerObject *)ob)->ob_reference));
-
-    Du_INCREF(result);
-    return result;
+    _du_read1(ob);
+    return ((DuContainerObject *)ob)->ob_reference;
 }
 
 void DuContainer_SetRef(DuObject *ob, DuObject *x)
 {
     DuContainer_Ensure("DuContainer_SetRef", ob);
 
-    Du_AME_WRITE(ob);
-    DuObject *prev = ((DuContainerObject *)ob)->ob_reference;
-    Du_INCREF(x);
+    _du_write1(ob);
     ((DuContainerObject *)ob)->ob_reference = x;
-    Du_DECREF(prev);
 }
 
 DuType DuContainer_Type = {
@@ -51,12 +35,14 @@
     (print_fn)container_print,
 };
 
-DuObject *DuContainer_New()
+DuObject *DuContainer_New(DuObject *x)
 {
+    _du_save1(x);
     DuContainerObject *ob =                                     \
         (DuContainerObject *)DuObject_New(&DuContainer_Type);
-    Du_INCREF(Du_None);
-    ob->ob_reference = Du_None;
+    _du_restore1(x);
+
+    ob->ob_reference = x;
     return (DuObject *)ob;
 }
 
@@ -64,5 +50,5 @@
 {
     if (!DuContainer_Check(ob))
         Du_FatalError("%s: expected 'container' argument, got '%s'",
-                      where, ob->ob_type->dt_name);
+                      where, Du_TYPE(ob)->dt_name);
 }
diff --git a/duhton/duhton.c b/duhton/duhton.c
--- a/duhton/duhton.c
+++ b/duhton/duhton.c
@@ -32,8 +32,6 @@
         if (interactive) {
             Du_Print(res, 1);
         }
-        Du_DECREF(res);
-        Du_DECREF(code);
         Du_TransactionRun();
         if (!interactive)
             break;
diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -93,7 +93,7 @@
 void DuList_SetItem(DuObject *list, int index, DuObject *newobj);
 DuObject *DuList_Pop(DuObject *list, int index);
 
-DuObject *DuContainer_New(void);
+DuObject *DuContainer_New(DuObject *obj);
 DuObject *DuContainer_GetRef(DuObject *container);
 void DuContainer_SetRef(DuObject *container, DuObject *newobj);
 
diff --git a/duhton/glob.c b/duhton/glob.c
--- a/duhton/glob.c
+++ b/duhton/glob.c
@@ -2,15 +2,79 @@
 #include <sys/select.h>
 
 
+static void _du_getargs0(const char *name, DuObject *cons, DuObject *locals)
+{
+    if (cons != Du_None)
+        Du_FatalError("%s: expected no argument", name);
+}
+
+static void _du_getargs1(const char *name, DuObject *cons, DuObject *locals,
+                         DuObject **a)
+{
+    DuObject *expr1, *obj1;
+
+    if (cons == Du_None) goto error;
+
+    _du_read1(cons);
+    expr1 = _DuCons_CAR(cons);
+    cons = _DuCons_NEXT(cons);
+    if (cons != Du_None) goto error;
+
+    obj1 = Du_Eval(expr1, locals);
+    *a = obj1;
+    return;
+
+ error:
+    Du_FatalError("%s: expected one argument", name);
+}
+
+static void _du_getargs2(const char *name, DuObject *cons, DuObject *locals,
+                         DuObject **a, DuObject **b)
+{
+    DuObject *expr1, *expr2, *obj1, *obj2;
+
+    if (cons == Du_None) goto error;
+
+    _du_read1(cons);
+    expr1 = _DuCons_CAR(cons);
+    cons = _DuCons_NEXT(cons);
+    if (cons == Du_None) goto error;
+
+    _du_read1(cons);
+    expr2 = _DuCons_CAR(cons);
+    cons = _DuCons_NEXT(cons);
+    if (cons != Du_None) goto error;
+
+    _du_save2(expr2, locals);
+    obj1 = Du_Eval(expr1, locals);
+    _du_restore2(expr2, locals);
+
+    _du_save1(obj1);
+    obj2 = Du_Eval(expr2, locals);
+    _du_restore1(obj1);
+
+    *a = obj1;
+    *b = obj2;
+    return;
+
+ error:
+    Du_FatalError("%s: expected two arguments", name);
+}
+
+/************************************************************/
+
+
 DuObject *Du_Progn(DuObject *cons, DuObject *locals)
 {
     DuObject *result = Du_None;
-    Du_INCREF(Du_None);
     while (cons != Du_None) {
-        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
-        Du_DECREF(result);
-        result = obj;
-        cons = _DuCons_NEXT(cons);
+        _du_read1(cons);
+        DuObject *expr = _DuCons_CAR(cons);
+        DuObject *next = _DuCons_NEXT(cons);
+        _du_save2(next, locals);
+        result = Du_Eval(expr, locals);
+        _du_restore2(next, locals);
+        cons = next;
     }
     return result;
 }
@@ -18,18 +82,25 @@
 DuObject *du_setq(DuObject *cons, DuObject *locals)
 {
     DuObject *result = Du_None;
-    Du_INCREF(Du_None);
     while (cons != Du_None) {
+        _du_read1(cons);
         DuObject *symbol = _DuCons_CAR(cons);
         cons = _DuCons_NEXT(cons);
         if (cons == Du_None)
             Du_FatalError("setq: number of arguments is odd");
+        _du_read1(cons);
         DuObject *expr = _DuCons_CAR(cons);
-        DuObject *obj = Du_Eval(expr, locals);
+        DuObject *next = _DuCons_NEXT(cons);
+
+        _du_save3(symbol, next, locals);
+        result = Du_Eval(expr, locals);
+        _du_restore3(symbol, next, locals);
+
+        _du_save2(next, locals);
         DuFrame_SetSymbol(locals, symbol, obj);
-        Du_DECREF(result);
-        result = obj;
-        cons = _DuCons_NEXT(cons);
+        _du_restore2(next, locals);
+
+        cons = next;
     }
     return result;
 }
@@ -39,25 +110,34 @@
     DuObject *lst = DuList_New();
 
     while (cons != Du_None) {
-        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        _du_read1(cons);
+        DuObject *expr = _DuCons_CAR(cons);
+        DuObject *next = _DuCons_NEXT(cons);
+
+        _du_save3(lst, next, locals);
+        DuObject *obj = Du_Eval(expr, locals);
+        _du_restore3(lst, next, locals);
+
+        _du_save3(lst, next, locals);
         DuList_Append(lst, obj);
-        Du_DECREF(obj);
-        cons = _DuCons_NEXT(cons);
+        _du_restore3(lst, next, locals);
+
+        cons = next;
     }
 
-    Du_AME_TryInevitable();
+    _du_save1(lst);
+    stm_become_inevitable("print");
+    _du_restore1(lst);
 
     int i;
     for (i=0; i<DuList_Size(lst); i++) {
         if (i > 0) printf(" ");
-        DuObject *obj = DuList_GetItem(lst, i);
-        Du_Print(obj, 0);
-        Du_DECREF(obj);
+        _du_save1(lst);
+        Du_Print(DuList_GetItem(lst, i), 0);
+        _du_restore1(lst);
     }
-    Du_DECREF(lst);
 
     printf("\n");
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
@@ -65,10 +145,16 @@
 {
     int result = 0;
     while (cons != Du_None) {
-        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        _du_read1(cons);
+        DuObject *expr = _DuCons_CAR(cons);
+        DuObject *next = _DuCons_NEXT(cons);
+
+        _du_save2(next, locals);
+        DuObject *obj = Du_Eval(expr, locals);
         result += DuInt_AsInt(obj);
-        Du_DECREF(obj);
-        cons = _DuCons_NEXT(cons);
+        _du_restore2(next, locals);
+
+        cons = next;
     }
     return DuInt_FromInt(result);
 }
@@ -78,11 +164,17 @@
     int result = 0;
     int sign = 1;
     while (cons != Du_None) {
-        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        _du_read1(cons);
+        DuObject *expr = _DuCons_CAR(cons);
+        DuObject *next = _DuCons_NEXT(cons);
+
+        _du_save2(next, locals);
+        DuObject *obj = Du_Eval(next, locals);
         result += sign * DuInt_AsInt(obj);
+        _du_restore2(next, locals);
+
         sign = -1;
-        Du_DECREF(obj);
-        cons = _DuCons_NEXT(cons);
+        cons = next;
     }
     return DuInt_FromInt(result);
 }
@@ -91,25 +183,27 @@
 {
     int result = 1;
     while (cons != Du_None) {
-        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        _du_read1(cons);
+        DuObject *expr = _DuCons_CAR(cons);
+        DuObject *next = _DuCons_NEXT(cons);
+
+        _du_save2(next, locals);
+        DuObject *obj = Du_Eval(expr, locals);
         result *= DuInt_AsInt(obj);
-        Du_DECREF(obj);
-        cons = _DuCons_NEXT(cons);
+        _du_restore2(next, locals);
+
+        cons = next;
     }
     return DuInt_FromInt(result);
 }
 
 static DuObject *_du_intcmp(DuObject *cons, DuObject *locals, int mode)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None ||
-        _DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
-        Du_FatalError("get: expected two arguments");
-    DuObject *obj_a = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *obj_b = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    DuObject *obj_a, *obj_b;
+    _du_getargs2("comparison", cons, locals, &obj_a, &obj_b);
+
     int a = DuInt_AsInt(obj_a);
     int b = DuInt_AsInt(obj_b);
-    Du_DECREF(obj_a);
-    Du_DECREF(obj_b);
     int r = 0;
     switch (mode) {
     case 0: r = a < b; break;
@@ -137,74 +231,86 @@
 
 DuObject *du_type(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("type: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *res = (DuObject *)(obj->ob_type);
-    Du_INCREF(res);
-    Du_DECREF(obj);
-    return res;
+    DuObject *obj;
+    _du_getarg1("type", cons, locals, &obj);
+
+    return DuSymbol_FromString(Du_TYPE(obj)->dt_name);
 }
 
 DuObject *du_quote(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
         Du_FatalError("quote: expected one argument");
-    DuObject *obj = _DuCons_CAR(cons);
-    Du_INCREF(obj);
-    return obj;
+    return _DuCons_CAR(cons);
 }
 
 DuObject *du_list(DuObject *cons, DuObject *locals)
 {
     DuObject *list = DuList_New();
     while (cons != Du_None) {
-        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+        _du_read1(cons);
+        DuObject *expr = _DuCons_CAR(cons);
+        DuObject *next = _DuCons_NEXT(cons);
+
+        _du_save3(list, next, locals);
+        DuObject *obj = Du_Eval(expr, locals);
+        _du_restore3(list, next, locals);
+
+        _du_save3(list, next, locals);
         DuList_Append(list, obj);
-        Du_DECREF(obj);
-        cons = _DuCons_NEXT(cons);
+        _du_restore3(list, next, locals);
+
+        cons = next;
     }
     return list;
 }
 
 DuObject *du_container(DuObject *cons, DuObject *locals)
 {
-    if (cons != Du_None && _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("container: expected at most one argument");
-    DuObject *container = DuContainer_New();
-    if (cons != Du_None) {
-        DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
-        DuContainer_SetRef(container, obj);
-        Du_DECREF(obj);
-    }
-    return container;
+    DuObject *container;
+    DuObject *obj;
+
+    if (cons == Du_None)
+        obj = Du_None;
+    else
+        _du_getarg1("container", cons, locals, &obj);
+
+    return DuContainer_New(obj);
 }
 
 DuObject *du_get(DuObject *cons, DuObject *locals)
 {
     if (cons == Du_None)
         Du_FatalError("get: expected at least one argument");
-    DuObject *res;
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+
+    _du_read1(cons);
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_save2(next, locals);
+    DuObject *obj = Du_Eval(expr, locals);
+    _du_restore2(next, locals);
 
     if (DuList_Check(obj)) {
-        if (_DuCons_NEXT(cons) == Du_None ||
-            _DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
+        _du_read1(next);
+        if (next == Du_None || _DuCons_NEXT(next) != Du_None)
             Du_FatalError("get with a list: expected two arguments");
-        DuObject *index = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
-        res = DuList_GetItem(obj, DuInt_AsInt(index));
-        Du_DECREF(index);
+
+        _du_save1(obj);
+        DuObject *index = Du_Eval(_DuCons_CAR(next), locals);
+        _du_restore1(obj);
+
+        return DuList_GetItem(obj, DuInt_AsInt(index));
     }
     else if (DuContainer_Check(obj)) {
-        if (_DuCons_NEXT(cons) != Du_None)
+        if (next != Du_None)
             Du_FatalError("get with a container: expected one argument");
-        res = DuContainer_GetRef(obj);
+
+        return DuContainer_GetRef(obj);
     }
     else
-        Du_FatalError("get: bad argument type '%s'", obj->ob_type->dt_name);
-
-    Du_DECREF(obj);
-    return res;
+        Du_FatalError("get: bad argument type '%s'", Du_TYPE(obj)->dt_name);
 }
 
 DuObject *du_set(DuObject *cons, DuObject *locals)
diff --git a/duhton/transaction.c b/duhton/transaction.c
--- a/duhton/transaction.c
+++ b/duhton/transaction.c
@@ -1,29 +1,12 @@
 #include "duhton.h"
 
 
-#ifndef Du_AME
-
-static DuObject *list_transactions = NULL;
-
 void Du_TransactionAdd(DuObject *frame)
 {
-    if (list_transactions == NULL) {
-        list_transactions = DuList_New();
-        _Du_ForgetReference(list_transactions);
-    }
-    DuList_Append(list_transactions, frame);
+    /* XXX */
 }
 
 void Du_TransactionRun(void)
 {
-    if (list_transactions == NULL)
-        return;
-    while (DuList_Size(list_transactions) > 0) {
-        DuObject *frame = DuList_Pop(list_transactions, 0);
-        DuObject *res = DuFrame_Execute(frame);
-        Du_DECREF(res);
-        Du_DECREF(frame);
-    }
+    /* XXX */
 }
-
-#endif

From noreply at buildbot.pypy.org  Sun Jun 23 23:05:21 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun, 23 Jun 2013 23:05:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: rpython's int() doesn't allow any
 trailing non-integer characters (such as a newline)
Message-ID: <20130623210521.36D8C1C1067@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r64959:7bf8c506fa12
Date: 2013-06-23 23:04 +0200
http://bitbucket.org/pypy/pypy/changeset/7bf8c506fa12/

Log:	rpython's int() doesn't allow any trailing non-integer characters
	(such as a newline)

diff --git a/rpython/memory/gc/env.py b/rpython/memory/gc/env.py
--- a/rpython/memory/gc/env.py
+++ b/rpython/memory/gc/env.py
@@ -215,9 +215,12 @@
             fd = os.open('/sys/devices/system/cpu/cpu' + assert_str0(str(cpu))
                          + '/l2_cache_size', os.O_RDONLY, 0644)
             try:
-                number = int(os.read(fd, 4096))
+                line = os.read(fd, 4096)
             finally:
                 os.close(fd)
+            end = len(line) - 1
+            assert end > 0
+            number = int(line[:end])
         except OSError:
             break
         if number < L2cache:

From noreply at buildbot.pypy.org  Mon Jun 24 08:26:39 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Jun 2013 08:26:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: add some docstrings
Message-ID: <20130624062639.D12CF1C0619@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r64960:f53356f4ff92
Date: 2013-06-24 08:25 +0200
http://bitbucket.org/pypy/pypy/changeset/f53356f4ff92/

Log:	add some docstrings

diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -5,6 +5,22 @@
 
 @unwrap_spec(type=str)
 def newdict(space, type):
+    """ newdict(type)
+
+    Create a normal dict with a special implementation strategy.
+
+    type is a string and can be:
+
+    * "module" - equivalent to some_module.__dict__
+
+    * "instance" - equivalent to an instance dict with a not-changing-much
+                   set of keys
+
+    * "kwargs" - keyword args dict equivalent of what you get from **kwargs
+                 in a function, optimized for passing around
+
+    * "strdict" - string-key only dict. This one should be chosen automatically
+    """
     if type == 'module':
         return space.newdict(module=True)
     elif type == 'instance':
@@ -18,6 +34,10 @@
                               type)
 
 def dictstrategy(space, w_obj):
+    """ dictstrategy(dict)
+
+    show the underlaying strategy used by a dict object
+    """
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError,
                              space.wrap("expecting dict object"))

From noreply at buildbot.pypy.org  Mon Jun 24 08:26:42 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Jun 2013 08:26:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <20130624062642.405FD1C0619@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r64961:9ff93976dbad
Date: 2013-06-24 08:26 +0200
http://bitbucket.org/pypy/pypy/changeset/9ff93976dbad/

Log:	merge default

diff too long, truncating to 2000 out of 4277 lines

diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,6 +134,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -168,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -278,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -8,7 +8,12 @@
 class TclError(Exception):
     pass
 
-from .tklib import tklib, tkffi
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
 from .app import TkApp
 
 TK_VERSION = tkffi.string(tklib.get_tk_version())
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -707,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -934,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -76,4 +76,4 @@
 The rest of directories serve specific niche goal and are unlikely a good
 entry point.
 
-.. _`introduction to RPython`: getting-started-dev.rst
+.. _`introduction to RPython`: getting-started-dev.html
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -56,3 +56,4 @@
 .. branch: argsort-segfault
 Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
 
+.. branch: dtype-isnative
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -904,6 +904,9 @@
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
+    def newlist_unicode(self, list_u):
+        return self.newlist([self.wrap(u) for u in list_u])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -13,7 +13,6 @@
     _mixin_ = True
 
     def __init__(self, space, name, size, align, enumerators, enumvalues):
-        name = "enum " + name
         self._super.__init__(self, space, size, name, len(name), align)
         self.enumerators2values = {}   # str -> int
         self.enumvalues2erators = {}   # int -> str
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         else:
             extra = " *"
         self.is_file = (ctitem.name == "struct _IO_FILE" or
-                        ctitem.name == "struct $FILE")
+                        ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -25,7 +25,6 @@
     custom_field_pos = False
 
     def __init__(self, space, name):
-        name = '%s %s' % (self.kind, name)
         W_CType.__init__(self, space, -1, name, len(name))
 
     def check_complete(self, w_errorcls=None):
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 
 from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr,
     ctypearray, ctypestruct, ctypevoid, ctypeenum)
@@ -115,13 +116,15 @@
 # ____________________________________________________________
 
 SF_MSVC_BITFIELDS = 1
+SF_GCC_ARM_BITFIELDS = 2
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+elif rffi_platform.getdefined('__arm__', ''):
+    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
     DEFAULT_SFLAGS = 0
 
-
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -180,7 +183,7 @@
         # field is an anonymous bitfield
         falign = ftype.alignof()
         do_align = True
-        if fbitsize >= 0:
+        if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
                 # GCC: anonymous bitfields (of any size) don't cause alignment
                 do_align = (fname != '')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -641,6 +641,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -648,7 +650,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -657,7 +659,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -689,7 +691,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -708,7 +710,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -728,7 +730,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -744,7 +746,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -758,7 +760,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -792,7 +794,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -803,7 +805,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -832,7 +834,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -937,7 +939,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -953,7 +955,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -965,7 +967,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -985,7 +987,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1023,7 +1025,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1048,7 +1050,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1221,7 +1223,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1238,7 +1240,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1275,26 +1277,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1308,27 +1311,27 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1432,7 +1435,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1455,7 +1458,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1485,14 +1488,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1561,7 +1564,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1601,7 +1604,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1619,7 +1622,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1697,7 +1700,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1714,7 +1717,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1725,7 +1728,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1735,7 +1738,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1745,7 +1748,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1754,7 +1757,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1766,7 +1769,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1775,7 +1778,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1785,7 +1788,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1795,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1818,7 +1821,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1845,7 +1848,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1957,7 +1960,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1977,7 +1980,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2164,7 +2167,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2183,7 +2186,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2192,7 +2195,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2308,7 +2311,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2320,8 +2323,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2353,12 +2356,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2366,7 +2369,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2424,7 +2427,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2440,14 +2443,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2508,7 +2511,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2540,7 +2543,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2563,7 +2566,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2762,20 +2765,20 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2783,11 +2786,14 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
@@ -2795,10 +2801,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2807,6 +2818,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -13,19 +13,9 @@
 from pypy.module._rawffi.interp_rawffi import TYPEMAP
 from pypy.module._rawffi.interp_rawffi import size_alignment
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
 
-def push_elem(ll_array, pos, value):
-    TP = lltype.typeOf(value)
-    ll_array = rffi.cast(rffi.CArrayPtr(TP), ll_array)
-    ll_array[pos] = value
-push_elem._annspecialcase_ = 'specialize:argtype(2)'
-
-def get_elem(ll_array, pos, ll_t):
-    ll_array = rffi.cast(rffi.CArrayPtr(ll_t), ll_array)
-    return ll_array[pos]
-get_elem._annspecialcase_ = 'specialize:arg(2)'
-
 
 class W_Array(W_DataShape):
     def __init__(self, basicffitype, size):
@@ -57,7 +47,7 @@
                                                 " array length"))
             for num in range(iterlength):
                 w_item = items_w[num]
-                unwrap_value(space, push_elem, result.ll_buffer, num,
+                unwrap_value(space, write_ptr, result.ll_buffer, num,
                              self.itemcode, w_item)
         return space.wrap(result)
 
@@ -118,7 +108,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, push_elem, self.ll_buffer, num,
+        unwrap_value(space, write_ptr, self.ll_buffer, num,
                      self.shape.itemcode, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
@@ -136,7 +126,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, get_elem, self.ll_buffer, num,
+        return wrap_value(space, read_ptr, self.ll_buffer, num,
                           self.shape.itemcode)
 
     def descr_getitem(self, space, w_index):
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._rawffi.array import push_elem
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from pypy.module._rawffi.structure import W_Structure
 from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp,
      unwrap_value, unpack_argshapes, got_libffi_error)
@@ -40,7 +40,7 @@
                 args_w[i] = space.wrap(rffi.cast(rffi.ULONG, ll_args[i]))
         w_res = space.call(w_callable, space.newtuple(args_w))
         if callback_ptr.result is not None: # don't return void
-            unwrap_value(space, push_elem, ll_res, 0,
+            unwrap_value(space, write_ptr, ll_res, 0,
                          callback_ptr.result, w_res)
     except OperationError, e:
         tbprint(space, space.wrap(e.get_traceback()),
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,7 @@
 
 from rpython.rlib.clibffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
 import rpython.rlib.rposix as rposix
 
@@ -43,6 +44,8 @@
 }
 TYPEMAP_PTR_LETTERS = "POszZ"
 TYPEMAP_NUMBER_LETTERS = "bBhHiIlLqQ?"
+TYPEMAP_FLOAT_LETTERS = "fd" # XXX long doubles are not propperly supported in
+                             # rpython, so we ignore then here
 
 if _MS_WINDOWS:
     TYPEMAP['X'] = ffi_type_pointer
@@ -239,6 +242,52 @@
 )
 
 unroll_letters_for_numbers = unrolling_iterable(TYPEMAP_NUMBER_LETTERS)
+unroll_letters_for_floats = unrolling_iterable(TYPEMAP_FLOAT_LETTERS)
+
+_ARM = rffi_platform.getdefined('__arm__', '')
+
+def read_ptr(ptr, ofs, TP):
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as t_array:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, t_array),
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.sizeof(TP))
+                ptr_val = t_array[0]
+                return ptr_val
+    else:
+        return rffi.cast(T, ptr)[ofs]
+read_ptr._annspecialcase_ = 'specialize:arg(2)'
+
+def write_ptr(ptr, ofs, value):
+    TP = lltype.typeOf(value)
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as s_array:
+                s_array[0] = value
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.cast(rffi.VOIDP, s_array),
+                    rffi.sizeof(TP))
+                return
+    else:
+        rffi.cast(T, ptr)[ofs] = value
+write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -12,10 +12,13 @@
 from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance
 from pypy.module._rawffi.interp_rawffi import wrap_value, unwrap_value
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
-from pypy.module._rawffi.interp_rawffi import size_alignment, LL_TYPEMAP
+from pypy.module._rawffi.interp_rawffi import LL_TYPEMAP
 from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers
+from pypy.module._rawffi.interp_rawffi import size_alignment
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib import clibffi
-from rpython.rlib.rarithmetic import intmask, r_uint, signedtype, widen
+from rpython.rlib.rarithmetic import intmask, signedtype, widen
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong
 
 def unpack_fields(space, w_fields):
     fields_w = space.unpackiterable(w_fields)
@@ -260,10 +263,17 @@
 
 def LOW_BIT(x):
     return x & 0xFFFF
+
 def NUM_BITS(x):
     return x >> 16
-def BIT_MASK(x):
-    return (1 << x) - 1
+
+def BIT_MASK(x, ll_t):
+    if ll_t is lltype.SignedLongLong:
+        return (r_longlong(1) << x) - 1
+    elif ll_t is lltype.UnsignedLongLong:
+        return (r_ulonglong(1) << x) - 1
+    return (1  << x) -1
+BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
@@ -279,20 +289,20 @@
             lowbit = LOW_BIT(bitsize)
             if numbits:
                 value = widen(value)
-                current = widen(rffi.cast(T, ptr)[0])
-                bitmask = BIT_MASK(numbits)
+                bitmask = BIT_MASK(numbits, TP)
+                #
+                current = widen(read_ptr(ptr, 0, TP))
                 current &= ~ (bitmask << lowbit)
                 current |= (value & bitmask) << lowbit
                 value = rffi.cast(TP, current)
             break
-
-    rffi.cast(T, ptr)[0] = value
+    write_ptr(ptr, 0, value)
 push_field._annspecialcase_ = 'specialize:argtype(2)'
 
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     TP = lltype.Ptr(rffi.CArray(ll_t))
-    value = rffi.cast(TP, pos)[0]
+    value = read_ptr(pos, 0, ll_t)
 
     # Handle bitfields
     for c in unroll_letters_for_numbers:
@@ -301,13 +311,16 @@
             numbits = NUM_BITS(bitsize)
             lowbit = LOW_BIT(bitsize)
             if numbits:
-                value = widen(value)
+                value = widen(rffi.cast(ll_t, value))
+                bitmask = BIT_MASK(numbits, ll_t)
+                #
                 value >>= lowbit
-                value &= BIT_MASK(numbits)
+                value &= bitmask
                 if ll_t is lltype.Bool or signedtype(ll_t._type):
                     sign = (value >> (numbits - 1)) & 1
                     if sign:
-                        value = value - (1 << numbits)
+                        one = r_longlong(1) if ll_t is lltype.SignedLongLong else 1
+                        value = value - (one << numbits)
                 value = rffi.cast(ll_t, value)
             break
 
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -48,12 +48,12 @@
             free(x1->next);
             free(x1);
         }
-        
+
         const char *static_str = "xxxxxx";
         long static_int = 42;
         double static_double = 42.42;
         long double static_longdouble = 42.42;
-        
+
         unsigned short add_shorts(short one, short two)
         {
            return one + two;
@@ -175,7 +175,7 @@
             inp.y = inp.x * 100;
             return inp;
         }
-        
+
         '''))
         symbols = """get_char char_check get_raw_pointer
                      add_shorts
@@ -194,7 +194,7 @@
         eci = ExternalCompilationInfo(export_symbols=symbols)
         return str(platform.compile([c_file], eci, 'x', standalone=False))
     prepare_c_example = staticmethod(prepare_c_example)
-    
+
     def setup_class(cls):
         space = cls.space
         from rpython.rlib.clibffi import get_libc_name
@@ -211,6 +211,7 @@
         cls.w_platform = space.wrap(platform.name)
         cls.w_sizes_and_alignments = space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
+        cls.w_typemap = space.wrap(TYPEMAP.keys())
 
     def test_libload(self):
         import _rawffi
@@ -438,7 +439,7 @@
         arg.free()
         assert t.tm_year == 70
         assert t.tm_sec == 1
-        assert t.tm_min == 2      
+        assert t.tm_min == 2
         x.free()
 
     def test_nested_structures(self):
@@ -464,6 +465,20 @@
         free_double_struct = lib.ptr("free_double_struct", ['P'], None)
         free_double_struct(res)
 
+    def test_structure_bitfields_char(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'B', 1),
+                               ('B', 'B', 6),
+                               ('C', 'B', 1)])
+        x = X()
+        x.A = 0xf
+        x.B = 0xff
+        x.C = 0xf
+        assert x.A == 1
+        assert x.B == 63
+        assert x.C == 1
+        x.free()
+
     def test_structure_bitfields(self):
         import _rawffi
         X = _rawffi.Structure([('A', 'I', 1),
@@ -486,6 +501,60 @@
         assert (y.a, y.b, y.c) == (-1, -7, 0)
         y.free()
 
+    def test_structure_bitfields_longlong(self):
+        import _rawffi
+        Z = _rawffi.Structure([('a', 'Q', 1),
+                               ('b', 'Q', 62),
+                               ('c', 'Q', 1)])
+        z = Z()
+        z.a, z.b, z.c = 7, 0x1000000000000001, 7
+        assert (z.a, z.b, z.c) == (1, 0x1000000000000001, 1)
+        z.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
     def test_invalid_bitfields(self):
         import _rawffi
         raises(TypeError, _rawffi.Structure, [('A', 'c', 1)])
@@ -571,7 +640,7 @@
         res = pass_ll(arg1)
         assert res[0] == 1<<42
         arg1.free()
-    
+
     def test_callback(self):
         import _rawffi
         import struct
@@ -683,6 +752,18 @@
             assert _rawffi.sizeof(k) == s
             assert _rawffi.alignment(k) == a
 
+    def test_unaligned(self):
+        import _rawffi
+        for k in self.typemap:
+            if k not in 'fdg':
+                continue
+            S = _rawffi.Structure([('pad', 'c'), ('value', k)], pack=1)
+            s = S()
+            s.value = 4
+            assert s.value == 4
+            s.free()
+
+
     def test_array_addressof(self):
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
@@ -970,7 +1051,7 @@
         assert res.y == 33
         assert s2h.x == 7
         assert s2h.y == 11
-        
+
         s2h.free()
 
     def test_ret_struct_containing_array(self):
@@ -1029,7 +1110,7 @@
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
-    
+
     def setup_class(cls):
         cls.w_sizes_and_alignments = cls.space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -12,8 +12,8 @@
 from pypy.module.cpyext.stringobject import PyString_Check
 from pypy.module.sys.interp_encoding import setdefaultencoding
 from pypy.module._codecs.interp_codecs import CodecState
-from pypy.objspace.std import unicodeobject, unicodetype, stringtype
-from rpython.rlib import runicode
+from pypy.objspace.std import unicodeobject, unicodetype
+from rpython.rlib import rstring, runicode
 from rpython.tool.sourcetools import func_renamer
 import sys
 
@@ -684,9 +684,9 @@
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
     if rffi.cast(lltype.Signed, direction) <= 0:
-        return stringtype.stringstartswith(str, substr, start, end)
+        return rstring.startswith(str, substr, start, end)
     else:
-        return stringtype.stringendswith(str, substr, start, end)
+        return rstring.endswith(str, substr, start, end)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_Count(space, w_str, w_substr, start, end):
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,8 +2,9 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.buffer import RWBuffer
 from rpython.rlib import rmmap, rarithmetic
-from rpython.rlib.rmmap import RValueError, RTypeError
+from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -20,13 +21,14 @@
         self.mmap.close()
 
     def read_byte(self):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.read_byte())
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def readline(self):
+        self.check_valid()
         return self.space.wrap(self.mmap.readline())
 
     @unwrap_spec(num=int)
@@ -36,6 +38,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def find(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -49,6 +52,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def rfind(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -62,16 +66,18 @@
 
     @unwrap_spec(pos=OFF_T, whence=int)
     def seek(self, pos, whence=0):
+        self.check_valid()
         try:
             self.mmap.seek(pos, whence)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def tell(self):
+        self.check_valid()
         return self.space.wrap(self.mmap.tell())
 
     def descr_size(self):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.file_size())
         except OSError, e:
@@ -79,41 +85,40 @@
 
     @unwrap_spec(data='bufferstr')
     def write(self, data):
+        self.check_valid()
         self.check_writeable()
         try:
             self.mmap.write(data)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(byte=str)
     def write_byte(self, byte):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.write_byte(byte)
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(offset=int, size=int)
     def flush(self, offset=0, size=0):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.flush(offset, size))
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
         except OSError, e:
             raise mmap_error(self.space, e)
 
     @unwrap_spec(dest=int, src=int, count=int)
     def move(self, dest, src, count):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.move(dest, src, count)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(newsize=int)
     def resize(self, newsize):
@@ -136,28 +141,19 @@
         try:
             self.mmap.check_valid()
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def check_writeable(self):
         try:
             self.mmap.check_writeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def check_resizeable(self):
         try:
             self.mmap.check_resizeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def descr_getitem(self, w_index):
         self.check_valid()
@@ -166,6 +162,8 @@
         start, stop, step = space.decode_index(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(self.mmap.getitem(start))
+        elif step == 1:
+            return space.wrap(self.mmap.getslice(start, stop - start))
         else:
             res = "".join([self.mmap.getitem(i)
                            for i in range(start, stop, step)])
@@ -189,15 +187,16 @@
             if len(value) != length:
                 raise OperationError(space.w_ValueError,
                           space.wrap("mmap slice assignment is wrong size"))
-            for i in range(length):
-                self.mmap.setitem(start, value[i])
-                start += step
+            if step == 1:
+                self.mmap.setslice(start, value)
+            else:
+                for i in range(length):
+                    self.mmap.setitem(start, value[i])
+                    start += step
 
     def descr_buffer(self):
-        # XXX improve to work directly on the low-level address
-        from pypy.interpreter.buffer import StringLikeBuffer
-        space = self.space
-        return space.wrap(StringLikeBuffer(space, space.wrap(self)))
+        self.check_valid()
+        return self.space.wrap(MMapBuffer(self.space, self.mmap))
 
 if rmmap._POSIX:
 
@@ -213,10 +212,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 elif rmmap._MS_WINDOWS:
@@ -232,10 +229,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 W_MMap.typedef = TypeDef("mmap",
@@ -275,5 +270,64 @@
                                                  space.w_EnvironmentError)
 
 def mmap_error(space, e):
-    w_error = space.fromcache(Cache).w_error
-    return wrap_oserror(space, e, w_exception_class=w_error)
+    if isinstance(e, RValueError):
+        return OperationError(space.w_ValueError,
+                              space.wrap(e.message))
+    elif isinstance(e, RTypeError):
+        return OperationError(space.w_TypeError,
+                              space.wrap(e.message))
+    elif isinstance(e, OSError):
+        w_error = space.fromcache(Cache).w_error
+        return wrap_oserror(space, e, w_exception_class=w_error)
+    else:
+        # bogus 'e'?
+        return OperationError(space.w_SystemError, space.wrap('%s' % e))
+mmap_error._dont_inline_ = True
+
+
+class MMapBuffer(RWBuffer):
+    def __init__(self, space, mmap):
+        self.space = space
+        self.mmap = mmap
+
+    def get_raw_address(self):
+        return self.mmap.data
+
+    def getlength(self):
+        return self.mmap.size
+
+    def getitem(self, index):
+        self.check_valid()
+        return self.mmap.data[index]
+
+    def getslice(self, start, stop, step, size):
+        self.check_valid()
+        if step == 1:
+            return self.mmap.getslice(start, size)
+        else:
+            return RWBuffer.getslice(self, start, stop, step, size)
+
+    def setitem(self, index, char):
+        self.check_valid_writeable()
+        self.mmap.data[index] = char
+
+    def setslice(self, start, string):
+        self.check_valid_writeable()
+        self.mmap.setslice(start, string)
+
+    def get_raw_address(self):
+        self.check_valid()
+        return self.mmap.data
+
+    def check_valid(self):
+        try:
+            self.mmap.check_valid()
+        except RValueError, v:
+            raise mmap_error(self.space, v)
+
+    def check_valid_writeable(self):
+        try:
+            self.mmap.check_valid()
+            self.mmap.check_writeable()
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -550,6 +550,24 @@
         m.close()
         f.close()
 
+    def test_buffer_write(self):
+        from mmap import mmap
+        f = open(self.tmpname + "y", "w+")
+        f.write("foobar")
+        f.flush()
+        m = mmap(f.fileno(), 6)
+        m[5] = '?'
+        b = buffer(m)
+        try:
+            b[:3] = "FOO"
+        except TypeError:     # on CPython: "buffer is read-only" :-/
+            skip("on CPython: buffer is read-only")
+        m.close()
+        f.seek(0)
+        got = f.read()
+        assert got == "FOOba?"
+        f.close()
+
     def test_offset(self):
         from mmap import mmap, ALLOCATIONGRANULARITY
         f = open(self.tmpname + "y", "w+")
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -45,10 +45,7 @@
     raise NotImplementedError
 
 def resolvedirof(filename):
-    try:
-        filename = rpath.rabspath(filename)
-    except OSError:
-        pass
+    filename = rpath.rabspath(filename)
     dirname = rpath.rabspath(os.path.join(filename, '..'))
     if os.path.islink(filename):
         try:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
@@ -990,7 +990,7 @@
         assert f.a == 12345
         assert b.b == b"B"
         assert b.c == b"C"
-        assert repr(b).startswith("<cdata 'struct $bar_t *'")
+        assert repr(b).startswith("<cdata 'bar_t *'")
 
     def test_struct_with_two_usages(self):
         for name in ['foo_s', '']:    # anonymous or not
@@ -1300,9 +1300,9 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("typedef enum { Value0 = 0 } e, *pe;\n"
                  "typedef enum { Value1 = 1 } e1;")
-        assert ffi.getctype("e*") == 'enum $e *'
-        assert ffi.getctype("pe") == 'enum $e *'
-        assert ffi.getctype("e1*") == 'enum $e1 *'
+        assert ffi.getctype("e*") == 'e *'
+        assert ffi.getctype("pe") == 'e *'
+        assert ffi.getctype("e1*") == 'e1 *'
 
     def test_new_ctype(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -1,5 +1,6 @@
 # Generated by pypy/tool/import_cffi.py
-import py, sys
+import py, sys, platform
+import pytest
 from pypy.module.test_lib_pypy.cffi_tests import backend_tests, test_function, test_ownlib
 from cffi import FFI
 import _cffi_backend
@@ -122,6 +123,7 @@
         self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8)
         self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4)
 
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_anonymous_no_align(self):
         L = FFI().alignof("long long")
         self.check("char y; int :1;", 0, 1, 2)
@@ -134,6 +136,20 @@
         self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
         self.check("char x; long long  :57; char y;", L + 8, 1, L + 9)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_anonymous_align_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :1;", 0, 4, 4)
+        self.check("char x; int z:1; char y;", 2, 4, 4)
+        self.check("char x; int  :1; char y;", 2, 4, 4)
+        self.check("char x; long long z:48; char y;", 7, L, 8)
+        self.check("char x; long long  :48; char y;", 7, 8, 8)
+        self.check("char x; long long z:56; char y;", 8, L, 8 + L)
+        self.check("char x; long long  :56; char y;", 8, L, 8 + L)
+        self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L)
+        self.check("char x; long long  :57; char y;", L + 8, L, L + 8 + L)
+
+    @pytest.mark.skipif("platform.machine().startswith('arm')")
     def test_bitfield_zero(self):
         L = FFI().alignof("long long")
         self.check("char y; int :0;", 0, 1, 4)
@@ -144,6 +160,17 @@
         self.check("char x; int :0; short b:1; char y;", 5, 2, 6)
         self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
 
+    @pytest.mark.skipif("not platform.machine().startswith('arm')")
+    def test_bitfield_zero_arm(self):
+        L = FFI().alignof("long long")
+        self.check("char y; int :0;", 0, 4, 4)
+        self.check("char x; int :0; char y;", 4, 4, 8)
+        self.check("char x; int :0; int :0; char y;", 4, 4, 8)
+        self.check("char x; long long :0; char y;", L, 8, L + 8)
+        self.check("short x, y; int :0; int :0;", 2, 4, 4)
+        self.check("char x; int :0; short b:1; char y;", 5, 4, 8)
+        self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8)
+
     def test_error_cases(self):
         ffi = FFI()
         py.test.raises(TypeError,
@@ -152,3 +179,9 @@
             'ffi.cdef("struct s2 { char x:0; };"); ffi.new("struct s2 *")')
         py.test.raises(TypeError,
             'ffi.cdef("struct s3 { char x:9; };"); ffi.new("struct s3 *")')
+
+    def test_struct_with_typedef(self):
+        ffi = FFI()
+        ffi.cdef("typedef struct { float x; } foo_t;")
+        p = ffi.new("foo_t *", [5.2])
+        assert repr(p).startswith("<cdata 'foo_t *' ")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
@@ -1442,7 +1442,7 @@
                        "foo_t myfunc(void) { foo_t x = { 42 }; return x; }")
     assert str(e.value) in [
         "function myfunc: 'foo_t' is used as result type, but is opaque",
-        "function myfunc: result type 'struct $foo_t' is opaque"]
+        "function myfunc: result type 'foo_t' is opaque"]
 
 def test_include():
     ffi1 = FFI()
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
@@ -66,6 +66,10 @@
         x.a, x.b, x.c = -1, 7, -1
         assert (x.a, x.b, x.c) == (-1, 7, -1)
 
+        x = X()
+        x.a, x.b, x.c = -1, -7, -1
+        assert (x.a, x.b, x.c) == (-1, -7, -1)
+
     def test_ulonglong(self):
         class X(Structure):
             _fields_ = [("a", c_ulonglong, 1),
@@ -75,8 +79,8 @@
         assert sizeof(X) == sizeof(c_longlong)
         x = X()
         assert (x.a, x.b, x.c) == (0, 0, 0)
-        x.a, x.b, x.c = 7, 7, 7
-        assert (x.a, x.b, x.c) == (1, 7, 1)
+        x.a, x.b, x.c = 7, 2305843009213693953, 7
+        assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1)
 
     def test_signed(self):
         for c_typ in signed_int_types:
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -50,10 +50,6 @@
 def delegate_Bool2IntObject(space, w_bool):
     return W_IntObject(int(w_bool.boolval))
 
-def delegate_Bool2SmallInt(space, w_bool):
-    from pypy.objspace.std.smallintobject import W_SmallIntObject
-    return W_SmallIntObject(int(w_bool.boolval))   # cannot overflow
-
 
 def nonzero__Bool(space, w_bool):
     return w_bool
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -59,11 +59,6 @@
 
 registerimplementation(W_IntObject)
 
-# NB: This code is shared by smallintobject.py, and thus no other Int
-# multimethods should be invoked from these implementations. Instead, add an
-# alias and then teach copy_multimethods in smallintobject.py to override
-# it. See int__Int for example.
-
 def repr__Int(space, w_int1):
     a = w_int1.intval
     res = str(a)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -166,6 +166,12 @@
         storage = strategy.erase(list_s)
         return W_ListObject.from_storage_and_strategy(space, storage, strategy)
 
+    @staticmethod
+    def newlist_unicode(space, list_u):
+        strategy = space.fromcache(UnicodeListStrategy)
+        storage = strategy.erase(list_u)
+        return W_ListObject.from_storage_and_strategy(space, storage, strategy)
+
     def __repr__(self):
         """ representation for debugging purposes """
         return "%s(%s, %s)" % (self.__class__.__name__, self.strategy,
@@ -1216,8 +1222,7 @@
     def _safe_find(self, w_list, obj, start, stop):
         l = self.unerase(w_list.lstorage)
         for i in range(start, min(stop, len(l))):
-            val = l[i]
-            if val == obj:
+            if l[i] == obj:
                 return i
         raise ValueError
 
@@ -1537,6 +1542,18 @@
         if reverse:
             l.reverse()
 
+    def _safe_find(self, w_list, obj, start, stop):
+        from rpython.rlib.rfloat import isnan
+        if not isnan(obj):
+            return AbstractUnwrappedStrategy._safe_find(self, w_list, obj,
+                                                        start, stop)
+        # unwrapped nan != nan, finding it requires more effort
+        l = self.unerase(w_list.lstorage)
+        for i in range(start, min(stop, len(l))):
+            if isnan(l[i]):
+                return i
+        raise ValueError
+
 
 class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -294,6 +294,9 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
+    def newlist_unicode(self, list_u):
+        return W_ListObject.newlist_unicode(self, list_u)
+
     def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1062,14 +1062,22 @@
 
     def _intersect_base(self, w_set, w_other):
         if self is w_other.strategy:
-            strategy = w_set.strategy
-            storage = strategy._intersect_unwrapped(w_set, w_other)
+            strategy = self
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = self._intersect_unwrapped(w_other, w_set)
+            else:
+                storage = self._intersect_unwrapped(w_set, w_other)
         elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
             strategy = self.space.fromcache(EmptySetStrategy)
             storage = strategy.get_empty_storage()
         else:
             strategy = self.space.fromcache(ObjectSetStrategy)
-            storage = self._intersect_wrapped(w_set, w_other)
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = w_other.strategy._intersect_wrapped(w_other, w_set)
+            else:
+                storage = self._intersect_wrapped(w_set, w_other)
         return storage, strategy
 
     def _intersect_wrapped(self, w_set, w_other):
@@ -1092,9 +1100,6 @@
         return self.erase(result)
 
     def intersect(self, w_set, w_other):
-        if w_set.length() > w_other.length():
-            return w_other.intersect(w_set)
-
         storage, strategy = self._intersect_base(w_set, w_other)
         return w_set.from_storage_and_strategy(storage, strategy)
 
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -11,12 +11,13 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
 from pypy.objspace.std.stringtype import (
-    joined2, sliced, stringendswith, stringstartswith, wrapchar, wrapstr)
+    joined2, sliced, wrapchar, wrapstr)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, specialize)
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import StringBuilder, split
+from rpython.rlib.rstring import (StringBuilder, split, rsplit, replace,
+    endswith, startswith)
 
 
 class W_AbstractStringObject(W_Object):
@@ -287,31 +288,12 @@
     bylen = len(by)
     if bylen == 0:
         raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-    if bylen == 1 and maxsplit < 0:
-        res = []
-        start = 0
-        # fast path: uses str.rfind(character) and str.count(character)
-        by = by[0]    # annotator hack: string -> char
-        count = value.count(by)
-        res = [None] * (count + 1)
-        end = len(value)
-        while count >= 0:
-            assert end >= 0
-            prev = value.rfind(by, 0, end)
-            start = prev + 1
-            assert start >= 0
-            res[count] = value[start:end]
-            count -= 1
-            end = prev
-    else:
-        res = split(value, by, maxsplit)
-
+    res = split(value, by, maxsplit)
     return space.newlist_str(res)
 
 def str_rsplit__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     i = len(value)-1
     while True:
@@ -336,43 +318,21 @@
         # the word is value[j+1:i+1]
         j1 = j + 1
         assert j1 >= 0
-        res_w.append(sliced(space, value, j1, i+1, w_self))
+        res.append(value[j1:i+1])
 
         # continue to look from the character before the space before the word
         i = j - 1
 
-    res_w.reverse()
-    return space.newlist(res_w)
+    res.reverse()
+    return space.newlist_str(res)
 
-def make_rsplit_with_delim(funcname, sliced):
-    from rpython.tool.sourcetools import func_with_new_name
-
-    def fn(space, w_self, w_by, w_maxsplit=-1):
-        maxsplit = space.int_w(w_maxsplit)
-        res_w = []
-        value = w_self._value
-        end = len(value)
-        by = w_by._value
-        bylen = len(by)
-        if bylen == 0:
-            raise OperationError(space.w_ValueError, space.wrap("empty separator"))
-
-        while maxsplit != 0:
-            next = value.rfind(by, 0, end)
-            if next < 0:
-                break
-            res_w.append(sliced(space, value, next+bylen, end, w_self))
-            end = next
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-
-        res_w.append(sliced(space, value, 0, end, w_self))
-        res_w.reverse()
-        return space.newlist(res_w)
-
-    return func_with_new_name(fn, funcname)
-
-str_rsplit__String_String_ANY = make_rsplit_with_delim('str_rsplit__String_String_ANY',
-                                                       sliced)
+def str_rsplit__String_String_ANY(space, w_self, w_by, w_maxsplit=-1):
+    maxsplit = space.int_w(w_maxsplit)
+    value = w_self._value
+    by = w_by._value
+    if not by:
+        raise OperationError(space.w_ValueError, space.wrap("empty separator"))
+    return space.newlist_str(rsplit(value, by, maxsplit))
 
 def str_join__String_ANY(space, w_self, w_list):
     l = space.listview_str(w_list)
@@ -524,75 +484,30 @@
 

From noreply at buildbot.pypy.org  Mon Jun 24 09:10:56 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 09:10:56 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add some comments; add hash_mangling
Message-ID: <20130624071056.1506A1C034D@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r248:78c58b8aabfa
Date: 2013-06-24 09:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/78c58b8aabfa/

Log:	add some comments; add hash_mangling

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -88,13 +88,45 @@
 }
 
 /************************************************************/
-
+/* Each object has a h_original pointer to an old copy of 
+   the same object (e.g. an old revision), the "original". 
+   The memory location of this old object is used as the ID 
+   for this object. If h_original is NULL *and* it is an
+   old object copy, it itself is the original. This invariant
+   must be upheld by all code dealing with h_original.
+   The original copy must never be moved again. Also, it may
+   be just a stub-object.
+   
+   If we want the ID of an object which is still young,
+   we must preallocate an old shadow-original that is used
+   as the target of the young object in a minor collection.
+   In this case, we set the HAS_ID flag on the young obj
+   to notify minor_collect.
+   This flag can be lost if the young obj is stolen. Then
+   the stealing thread uses the shadow-original itself and
+   minor_collect must not overwrite it again.
+   Also, if there is already a backup-copy around, we use
+   this instead of allocating another old object to use as 
+   the shadow-original.
+ */
 
 revision_t stm_hash(gcptr p)
 {
     return stm_id(p);
 }
 
+
+static revision_t mangle_hash(revision_t n)
+{
+    /* To hash pointers in dictionaries.  Assumes that i shows some
+       alignment (to 4, 8, maybe 16 bytes), so we use the following
+       formula to avoid the trailing bits being always 0.
+       This formula is reversible: two different values of 'i' will
+       always give two different results.
+    */
+    return n ^ (n >> 4);
+}
+
 revision_t stm_id(gcptr p)
 {
     struct tx_descriptor *d = thread_descriptor;
@@ -104,15 +136,23 @@
     if (p->h_original) { /* fast path */
         fprintf(stderr, "stm_id(%p) has orig fst: %p\n", 
                 p, (gcptr)p->h_original);
-        return p->h_original;
+        return mangle_hash(p->h_original);
     } 
     else if (!(p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
                && (p->h_tid & GCFLAG_OLD)) {
         /* we can be sure that p->h_original doesn't
-           get set during the if and the else-if */
+           get set during the if and the else-if 
+           
+           XXX: check for priv_from_protected may not be 
+           necessary. only if this func may be called on 
+           another thread's young objects that are made 
+           old at the same time, and we see the OLD flag 
+           before h_original has been set.
+        */
         fprintf(stderr, "stm_id(%p) is old, orig=0 fst: %p\n", p, p);
-        return (revision_t)p;
+        return mangle_hash((revision_t)p);
     }
+
     
     spinlock_acquire(d->public_descriptor->collection_lock, 'I');
     /* old objects must have an h_original xOR be
@@ -137,7 +177,6 @@
         if (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
             gcptr B = (gcptr)p->h_revision;
             /* don't set, otherwise nursery will copy over backup */
-            //p->h_tid |= GCFLAG_HAS_ID; // see AbortPrivateFromProtected
             p->h_original = (revision_t)B;
             // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
             
@@ -157,7 +196,7 @@
     }
     
     spinlock_release(d->public_descriptor->collection_lock);
-    return result;
+    return mangle_hash(result);
 }
 
 revision_t stm_pointer_equal(gcptr p1, gcptr p2)
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -93,7 +93,6 @@
             assert(!(L->h_tid & GCFLAG_HAS_ID));
             /* original must be L */
             B->h_original = (revision_t)L;
-            assert(0);
         }
         else {
             /* we can make the backup the "original"
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -216,12 +216,10 @@
 def test_id_private_from_protected():
     # read and write from protected
     p = oalloc(HDR)
-    pid = lib.stm_id(p)
     porig = follow_original(p)
     # impl detail {
     # old objects have id==itself, if not set differently
     assert porig == ffi.NULL
-    assert ffi.cast("gcptr", pid) == p
     # }
 
     p1 = oalloc(HDR)
@@ -237,7 +235,6 @@
     assert p2id == lib.stm_id(p2w)
     # impl detail {
     assert p2w.h_original == 0
-    assert follow_revision(p2w).h_original == lib.stm_id(p2w)
     # }
     
 

From noreply at buildbot.pypy.org  Mon Jun 24 09:41:25 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 09:41:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add multiple thread start/finish to
	demo_random
Message-ID: <20130624074125.5C7361C0619@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r249:c73629c0bb3c
Date: 2013-06-24 09:41 +0200
http://bitbucket.org/pypy/stmgc/changeset/c73629c0bb3c/

Log:	add multiple thread start/finish to demo_random

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -13,7 +13,8 @@
 
 
 #define NUMTHREADS 4
-#define STEPS 1000000
+#define STEPS_PER_THREAD 5000
+#define THREAD_STARTS 100 // how many restarts of threads
 #define NUMROOTS 10 // per thread
 #define PREBUILT 3 // per thread
 #define MAXROOTS 1000
@@ -234,7 +235,7 @@
 {
     int i;
     td.thread_seed = default_seed;
-    td.steps_left = STEPS;
+    td.steps_left = STEPS_PER_THREAD;
     td.interruptible = 0;
     
     td.num_roots = PREBUILT + NUMROOTS;
@@ -250,8 +251,6 @@
 
 gcptr do_step(gcptr p)
 {
-    fprintf(stdout, "#");
-    
     nodeptr w_r, w_sr;
     gcptr _r, _sr;
     int num, k;
@@ -394,6 +393,10 @@
     gcptr p = NULL;
     while (td.steps_left) {
         td.steps_left--;
+
+        if (td.steps_left % 8 == 0)
+            fprintf(stdout, "#");
+
         p = do_step(p);
 
         if (p == (gcptr)-1)
@@ -453,13 +456,20 @@
     status = sem_init(&done, 0, 0);
     assert(status == 0);
     
-    for (i = 0; i < NUMTHREADS; i++)
+    int thread_starts = NUMTHREADS * THREAD_STARTS;
+    for (i = 0; i < NUMTHREADS; i++) {
         newthread(demo, NULL);
+        thread_starts--;
+    }
     
-    for (i=0; i < NUMTHREADS; i++) {
+    for (i=0; i < NUMTHREADS * THREAD_STARTS; i++) {
         status = sem_wait(&done);
         assert(status == 0);
         printf("thread finished\n");
+        if (thread_starts) {
+            thread_starts--;
+            newthread(demo, NULL);
+        }
     }
     
     return 0;

From noreply at buildbot.pypy.org  Mon Jun 24 10:14:36 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Jun 2013 10:14:36 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: merge default
Message-ID: <20130624081436.316351C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r64962:da084e4488d7
Date: 2013-06-24 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/da084e4488d7/

Log:	merge default

diff too long, truncating to 2000 out of 21147 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -58,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -80,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,21 +134,30 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
 }
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
+}
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
@@ -164,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
@@ -174,8 +187,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -183,9 +196,9 @@
     if isinstance(level, int):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
@@ -277,7 +290,7 @@
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - long(ct)) * 1000
+        self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads and thread:
             self.thread = thread.get_ident()
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -156,7 +156,7 @@
         h = klass(*args)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -187,7 +187,7 @@
     opts = cp.options(sectname)
     if "level" in opts:
         level = cp.get(sectname, "level")
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = cp.get(sectname, "handlers")
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,16 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    try:
-                        self._sock._decref_socketios()
-                    except AttributeError:
-                        pass  # bah, someone built a _fileobject manually
-                              # with some unexpected replacement of the
-                              # _socketobject class
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -65,7 +65,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
         finally:
             logging._releaseLock()
 
@@ -97,8 +98,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
@@ -275,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
@@ -166,8 +166,7 @@
         if self is StructOrUnion:
             return
         if '_fields_' not in self.__dict__:
-            self._fields_ = []
-            _set_shape(self, [], self._is_union)
+            self._fields_ = []  # As a side-effet, this also sets the ffishape.
 
     __setattr__ = struct_setattr
 
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_pypy_testcapi.py
copy from lib_pypy/_testcapi.py
copy to lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,14 +1,20 @@
-import os, sys
+import os, sys, imp
 import tempfile
 
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
     """
     thisdir = os.path.dirname(__file__)
     output_dir = tempfile.mkdtemp()
 
     from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -19,13 +25,13 @@
         ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
+    res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
                            extra_preargs=ccflags)
     object_filename = res[0]
 
     # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
+    output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
@@ -37,7 +43,7 @@
             library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
+                        '/EXPORT:init' + modulename]
     else:
         libraries = []
         extra_ldargs = []
@@ -49,14 +55,7 @@
         libraries=libraries,
         extra_preargs=extra_ldargs)
 
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-try:
-    import cpyext
-except ImportError:
-    raise ImportError("No module named '_testcapi'")
-else:
-    compile_shared()
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,62 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
 try:
     import cpyext
 except ImportError:
     raise ImportError("No module named '_testcapi'")
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,48 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.6"
-__version_info__ = (0, 6)
+__version__ = "0.7"
+__version_info__ = (0, 7)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -73,15 +73,15 @@
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        BVoidP = self._get_cached_btype(model.voidp_type)
+        self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(BVoidP, 0)
+                FFI.NULL = self.cast(self.BVoidP, 0)
                 FFI.CData, FFI.CType = backend._get_types()
         else:
             # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(BVoidP, 0)
+            self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False):
@@ -346,6 +346,12 @@
         self._cdefsources.extend(ffi_to_include._cdefsources)
         self._cdefsources.append(']')
 
+    def new_handle(self, x):
+        return self._backend.newp_handle(self.BVoidP, x)
+
+    def from_handle(self, x):
+        return self._backend.from_handle(x)
+
 
 def _make_ffi_library(ffi, libname, flags):
     import os
@@ -355,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
@@ -372,8 +378,8 @@
             BType = ffi._get_cached_btype(tp)
             try:
                 value = backendlib.load_function(BType, name)
-            except KeyError:
-                raise AttributeError(name)
+            except KeyError as e:
+                raise AttributeError('%s: %s' % (name, e))
             library.__dict__[name] = value
             return
         #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -16,6 +16,7 @@
 class CTypesData(object):
     __metaclass__ = CTypesType
     __slots__ = ['__weakref__']
+    __name__ = '<cdata>'
 
     def __init__(self, *args):
         raise TypeError("cannot instantiate %r" % (self.__class__,))
@@ -491,6 +492,8 @@
         elif BItem in (getbtype(model.PrimitiveType('signed char')),
                        getbtype(model.PrimitiveType('unsigned char'))):
             kind = 'bytep'
+        elif BItem is getbtype(model.void_type):
+            kind = 'voidp'
         else:
             kind = 'generic'
         #
@@ -546,13 +549,13 @@
             def __setitem__(self, index, value):
                 self._as_ctype_ptr[index] = BItem._to_ctypes(value)
 
-            if kind == 'charp':
+            if kind == 'charp' or kind == 'voidp':
                 @classmethod
-                def _arg_to_ctypes(cls, value):
-                    if isinstance(value, bytes):
-                        return ctypes.c_char_p(value)
+                def _arg_to_ctypes(cls, *value):
+                    if value and isinstance(value[0], bytes):
+                        return ctypes.c_char_p(value[0])
                     else:
-                        return super(CTypesPtr, cls)._arg_to_ctypes(value)
+                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
 
             if kind == 'charp' or kind == 'bytep':
                 def _to_string(self, maxlen):
@@ -704,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -931,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -15,6 +15,20 @@
     def patch_extension_kwds(self, kwds):
         pass
 
+    def find_module(self, module_name, path, so_suffix):
+        try:
+            f, filename, descr = imp.find_module(module_name, path)
+        except ImportError:
+            return None
+        if f is not None:
+            f.close()
+        # Note that after a setuptools installation, there are both .py
+        # and .so files with the same basename.  The code here relies on
+        # imp.find_module() locating the .so in priority.
+        if descr[0] != so_suffix:
+            return None
+        return filename
+
     def collect_types(self):
         self._typesdict = {}
         self._generate("collecttype")
@@ -142,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -427,9 +444,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -687,7 +704,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
     # ----------
 
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -1,4 +1,4 @@
-import sys
+import sys, os
 import types
 
 from . import model, ffiplatform
@@ -20,6 +20,16 @@
         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
+    def find_module(self, module_name, path, so_suffix):
+        basename = module_name + so_suffix
+        if path is None:
+            path = sys.path
+        for dirname in path:
+            filename = os.path.join(dirname, basename)
+            if os.path.isfile(filename):
+                return filename
+        return None
+
     def collect_types(self):
         pass      # not needed in the generic engine
 
@@ -64,6 +74,9 @@
         class FFILibrary(types.ModuleType):
             _cffi_generic_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir
         library = FFILibrary("")
         #
         # finally, call the loaded_gen_xxx() functions.  This will set
@@ -158,21 +171,22 @@
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
-            if any(isinstance(type, model.StructOrUnion) for type in tp.args):
+            if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
-                for i, type in enumerate(tp.args):
-                    if isinstance(type, model.StructOrUnion):
-                        type = model.PointerType(type)
-                        indirections.append((i, type))
-                    indirect_args.append(type)
+                for i, typ in enumerate(tp.args):
+                    if isinstance(typ, model.StructOrUnion):
+                        typ = model.PointerType(typ)
+                        indirections.append((i, typ))
+                    indirect_args.append(typ)
                 tp = model.FunctionPtrType(tuple(indirect_args),
                                            tp.result, tp.ellipsis)
             BFunc = self.ffi._get_cached_btype(tp)
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
-            for i, type in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, type)
+            for i, typ in indirections:
+                newfunction = self._make_struct_wrapper(newfunction, i, typ)
         setattr(library, name, newfunction)
+        type(library)._cffi_dir.append(name)
 
     def _make_struct_wrapper(self, oldfunc, i, tp):
         backend = self.ffi._backend
@@ -216,9 +230,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
-                and ftype.is_integer_type()):
+                and ftype.is_integer_type()) or fbitsize >= 0:
                 # accept all integers, but complain on float or double
                 prnt('  (void)((p->%s) << 1);' % fname)
             else:
@@ -380,6 +394,7 @@
         is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
         value = self._load_constant(is_int, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # enums
@@ -427,6 +442,7 @@
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
             setattr(library, enumerator, enumvalue)
+            type(library)._cffi_dir.append(enumerator)
 
     # ----------
     # macros: for now only for integers
@@ -440,6 +456,7 @@
     def _loaded_gen_macro(self, tp, name, module, library):
         value = self._load_constant(True, tp, name, module)
         setattr(library, name, value)
+        type(library)._cffi_dir.append(name)
 
     # ----------
     # global variables
@@ -465,6 +482,7 @@
                 BArray = self.ffi._get_cached_btype(tp)
                 value = self.ffi.cast(BArray, value)
             setattr(library, name, value)
+            type(library)._cffi_dir.append(name)
             return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
@@ -476,7 +494,8 @@
             return ptr[0]
         def setter(library, value):
             ptr[0] = value
-        setattr(library.__class__, name, property(getter, setter))
+        setattr(type(library), name, property(getter, setter))
+        type(library)._cffi_dir.append(name)
 
 cffimod_header = r'''
 #include <stdio.h>
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -102,21 +102,10 @@
                 path = pkg.__path__
             else:
                 path = None
-            try:
-                f, filename, descr = imp.find_module(self.get_module_name(),
-                                                     path)
-            except ImportError:
+            filename = self._vengine.find_module(self.get_module_name(), path,
+                                                 _get_so_suffix())
+            if filename is None:
                 return
-            if f is not None:
-                f.close()
-            if filename.lower().endswith('.py'):
-                # on PyPy, if there are both .py and .pypy-19.so files in
-                # the same directory, the .py file is returned.  That's the
-                # case after a setuptools installation.  We never want to
-                # load the .py file here...
-                filename = filename[:-3] + _get_so_suffix()
-                if not os.path.isfile(filename):
-                    return
             self.modulefilename = filename
         self._vengine.collect_types()
         self._has_module = True
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
new file mode 100644
--- /dev/null
+++ b/lib_pypy/greenlet.egg-info
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: greenlet
+Version: 0.4.0
+Summary: Lightweight in-process concurrent programming
+Home-page: https://github.com/python-greenlet/greenlet
+Author: Ralf Schmitt (for CPython), PyPy team
+Author-email: pypy-dev at python.org
+License: MIT License
+Description: UNKNOWN
+Platform: UNKNOWN
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -27,7 +27,8 @@
     BoolOption("completer", "use readline commandline completer",
                default=False, cmdline="-C"),
     BoolOption("optimize",
-               "dummy optimization flag for compatibility with CPython",
+               "skip assert statements and remove docstrings when importing modules"
+               " (this is -OO in regular CPython)",
                default=False, cmdline="-O"),
     BoolOption("no_site_import", "do not 'import site' on initialization",
                default=False, cmdline="-S"),
@@ -94,6 +95,17 @@
     space.setitem(space.sys.w_dict, space.wrap('executable'),
                   space.wrap(argv[0]))
 
+    if interactiveconfig.optimize:
+        #change the optimize flag's value and set __debug__ to False
+        space.appexec([], """():
+            import sys
+            flags = list(sys.flags)
+            flags[6] = 2
+            sys.flags = type(sys.flags)(flags)
+            import __pypy__
+            __pypy__.set_debug(False)
+        """)
+
     # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
     # sys.exec_prefix
     executable = argv[0]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -32,11 +32,10 @@
      "rctime" , "select", "zipimport", "_lsprof",
      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
-     "thread", "itertools", "pyexpat", "_ssl", "array",
+     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
      "binascii", "_multiprocessing", '_warnings',
      "_collections", "_multibytecodec", "micronumpy", "_ffi",
-     "_continuation", "_cffi_backend", "_csv"] # "cpyext", "cppyy"]
-# disabled until problems are fixed
+     "_continuation", "_cffi_backend", "_csv", "cppyy"]
 ))
 
 translation_modules = default_modules.copy()
@@ -65,7 +64,8 @@
     del working_modules["termios"]
     del working_modules["_minimal_curses"]
 
-    del working_modules["cppyy"]  # not tested on win32
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # not tested on win32
 
     # The _locale module is needed by site.py on Windows
     default_modules["_locale"] = None
@@ -78,7 +78,8 @@
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
     del working_modules["_multiprocessing"]   # depends on rctime
-    del working_modules["cppyy"]  # depends on ctypes
+    if "cppyy" in working_modules:
+        del working_modules["cppyy"]  # depends on ctypes
 
 
 module_dependencies = {
@@ -215,10 +216,6 @@
                    "(the empty string and potentially single-char strings)",
                    default=False),
 
-        BoolOption("withsmalltuple",
-                   "use small tuples",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,3 +1,7 @@
+
+.. comment: this document is very incomplete, should we generate
+            it automatically?
+
 =======================
 The ``__pypy__`` module
 =======================
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -163,6 +163,9 @@
     $ genreflex MyClass.h
     $ g++ -fPIC -rdynamic -O2 -shared -I$REFLEXHOME/include MyClass_rflx.cpp -o libMyClassDict.so -L$REFLEXHOME/lib -lReflex
 
+Next, make sure that the library can be found through the dynamic lookup path
+(the ``LD_LIBRARY_PATH`` environment variable on Linux, ``PATH`` on Windows),
+for example by adding ".".
 Now you're ready to use the bindings.
 Since the bindings are designed to look pythonistic, it should be
 straightforward::
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:

From noreply at buildbot.pypy.org  Mon Jun 24 10:28:39 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Mon, 24 Jun 2013 10:28:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: clean up test
Message-ID: <20130624082839.B8AD71C0F10@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r64963:10f6357386ef
Date: 2013-06-24 03:21 -0500
http://bitbucket.org/pypy/pypy/changeset/10f6357386ef/

Log:	clean up test

diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1,6 +1,6 @@
 from rpython.translator.platform import platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.module._rawffi.interp_rawffi import TYPEMAP
+from pypy.module._rawffi.interp_rawffi import TYPEMAP, TYPEMAP_FLOAT_LETTERS
 from pypy.module._rawffi.tracker import Tracker
 
 import os, sys, py
@@ -211,7 +211,7 @@
         cls.w_platform = space.wrap(platform.name)
         cls.w_sizes_and_alignments = space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
-        cls.w_typemap = space.wrap(TYPEMAP.keys())
+        cls.w_float_typemap = space.wrap(TYPEMAP_FLOAT_LETTERS)
 
     def test_libload(self):
         import _rawffi
@@ -749,14 +749,13 @@
     def test_sizes_and_alignments(self):
         import _rawffi
         for k, (s, a) in self.sizes_and_alignments.iteritems():
+            print k,s,a
             assert _rawffi.sizeof(k) == s
             assert _rawffi.alignment(k) == a
 
     def test_unaligned(self):
         import _rawffi
-        for k in self.typemap:
-            if k not in 'fdg':
-                continue
+        for k in self.float_typemap:
             S = _rawffi.Structure([('pad', 'c'), ('value', k)], pack=1)
             s = S()
             s.value = 4

From noreply at buildbot.pypy.org  Mon Jun 24 10:43:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 10:43:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add a comment about dead threads.
Message-ID: <20130624084341.C448A1C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r250:83d82fca9ee6
Date: 2013-06-24 10:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/83d82fca9ee6/

Log:	Add a comment about dead threads.

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -69,6 +69,15 @@
 void stm_steal_stub(gcptr P)
 {
     struct tx_public_descriptor *foreign_pd = STUB_THREAD(P);
+    /* A note about dead threads: 'foreign_pd' might belong to a thread
+     * that finished and is no longer around.  In this case, note that
+     * this function will not add anything to 'foreign_pd->stolen_objects':
+     * 
+     * 1. the dead thread is not running a transaction, so there are no
+     *    private objects (so no PRIVATE_FROM_PROTECTED)
+     *
+     * 2. the dead thread has no nursery, so all of its objects are old
+     */
 
     spinlock_acquire(foreign_pd->collection_lock, 'S');   /*stealing*/
 

From noreply at buildbot.pypy.org  Mon Jun 24 10:43:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 10:43:42 +0200 (CEST)
Subject: [pypy-commit] stmgc default: move to the current issue
Message-ID: <20130624084342.F3BB41C0619@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r251:1ff1701e5e03
Date: 2013-06-24 10:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/1ff1701e5e03/

Log:	move to the current issue

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -522,4 +522,4 @@
 def test_more_multi_thread():
     #py.test.skip("more random tests")
     for i in range(200):
-        yield test_multi_thread, (3311//2) + i
+        yield test_multi_thread, 1690 + i

From noreply at buildbot.pypy.org  Mon Jun 24 10:48:37 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 10:48:37 +0200 (CEST)
Subject: [pypy-commit] stmgc default: remove assert
Message-ID: <20130624084837.B9F1B1C0F10@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r252:6effbf62467f
Date: 2013-06-24 10:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/6effbf62467f/

Log:	remove assert

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1501,8 +1501,11 @@
           assert(descriptor_array_free_list >= 0);
           assert(pd->stolen_objects.size == 0);
           assert(pd->stolen_young_stubs.size == 0);
-          assert(pd->collection_lock == 0);
           pd->shutdown = 0;
+          /* there may be a thread holding the collection lock
+             because it steals a stub belonging to the thread
+             that previously owned this descriptor.
+          */
       }
       else {
           /* no item in the free list */

From noreply at buildbot.pypy.org  Mon Jun 24 10:52:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 10:52:30 +0200 (CEST)
Subject: [pypy-commit] buildbot default: Kill this anti-feature of checking
 if the category name ends with "-run".
Message-ID: <20130624085230.871CC1C0F10@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r827:023a2b6ef5b8
Date: 2013-06-24 10:52 +0200
http://bitbucket.org/pypy/buildbot/changeset/023a2b6ef5b8/

Log:	Kill this anti-feature of checking if the category name ends with
	"-run".

diff --git a/bot2/pypybuildbot/summary.py b/bot2/pypybuildbot/summary.py
--- a/bot2/pypybuildbot/summary.py
+++ b/bot2/pypybuildbot/summary.py
@@ -179,14 +179,14 @@
         run_info = {'URL': run_url, 'elapsed': pytest_elapsed or None,
                     'times': build.getTimes()}
         outcome_set = RevisionOutcomeSet(rev, key, run_info)
-        someresult = False
-        # "*-run" categories mean the build is not a test build!
-        if builderStatus.category:
-            someresult = builderStatus.category.endswith("-run")
+        #someresult = False
+        ## "*-run" categories mean the build is not a test build!
+        #if builderStatus.category:
+        #    someresult = builderStatus.category.endswith("-run")
         if pytest_logs:
             for stepName, resultLog in pytest_logs:
                 if resultLog.hasContents():
-                    someresult = True
+                    #someresult = True
                     outcome_set.populate(resultLog)
 
         failedtests = not not outcome_set.failed
@@ -201,11 +201,12 @@
                 failure = text
                 break
 
-        if not someresult or failure is not None:
-            if failure:
-                name = '"%s"' % failure # quote
-            else:
-                name = '<run>'
+        #if not someresult or failure is not None:
+        if failure is not None:
+            #if failure:
+            name = '"%s"' % failure # quote
+            #else:
+            #    name = '<run>'
             outcome_set.populate_one(name, '!')
 
         return outcome_set

From noreply at buildbot.pypy.org  Mon Jun 24 12:30:29 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 12:30:29 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fix
Message-ID: <20130624103029.AB5D81C06BD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r253:500b2ca07505
Date: 2013-06-24 12:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/500b2ca07505/

Log:	fix

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -160,19 +160,18 @@
                 /* use id-copy for us */
                 O = (gcptr)L->h_original;
                 L->h_tid &= ~GCFLAG_HAS_ID;
-                L->h_revision = (revision_t)O;
-                copy_to_old_id_copy(L, (gcptr)L->h_original);
+                copy_to_old_id_copy(L, O);
+                O->h_original = 0;
             } else {
                 /* Copy the object out of the other thread's nursery, 
                    if needed */
                 O = stmgc_duplicate_old(L);
-                L->h_revision = (revision_t)O;
 
-                /* young and without original?
-                   we may lose the HAS_ID flag like above */
+                /* young and without original? */
                 if (!(L->h_original))
                     L->h_original = (revision_t)O;
             }
+            L->h_revision = (revision_t)O;
 
             L->h_tid |= GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED;
             /* subtle: we need to remove L from the fxcache of the target
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -22,6 +22,7 @@
     assert p.h_revision == r
     assert p.h_tid == lib.gettid(p) | 0    # no GC flags
     assert classify(p) == "private"
+    assert lib.stm_id(p) != 0
 
 def test_write_barrier_private():
     p = nalloc(HDR)
@@ -84,17 +85,20 @@
                                        GCFLAG_PUBLIC |
                                        GCFLAG_PREBUILT_ORIGINAL)
     assert classify(p) == "public"
+    assert lib.stm_id(p) != 0
 
 def test_prebuilt_object_to_private():
     p = palloc(HDR)
     flags = p.h_tid
     assert (flags & GCFLAG_PUBLIC_TO_PRIVATE) == 0
+    pid = lib.stm_id(p)
     assert classify(p) == "public"
     p2 = lib.stm_write_barrier(p)
     assert p2 != p
     assert classify(p) == "public"
     assert classify(p2) == "private"
     assert p.h_tid == flags | GCFLAG_PUBLIC_TO_PRIVATE
+    assert pid == lib.stm_id(p2)
 
 def test_commit_change_to_prebuilt_object():
     p = palloc(HDR + WORD)
@@ -155,6 +159,7 @@
 
 def test_read_barrier_public_to_private():
     p = palloc(HDR)
+    pid = lib.stm_id(p)
     p2 = lib.stm_write_barrier(p)
     assert p2 != p
     assert classify(p) == "public"
@@ -165,6 +170,7 @@
     p3 = lib.stm_read_barrier(p)
     assert p3 == p2
     assert list_of_read_objects() == [p]
+    assert pid == lib.stm_id(p2)
 
 def test_read_barrier_handle_protected():
     p = palloc(HDR)
@@ -231,8 +237,8 @@
 
     p2 = oalloc(HDR)
     p2w = lib.stm_write_barrier(p2)
-    p2id = lib.stm_id(p2)
-    assert p2id == lib.stm_id(p2w)
+    p2id = lib.stm_id(p2w)
+    assert p2id == lib.stm_id(p2)
     # impl detail {
     assert p2w.h_original == 0
     # }
@@ -255,6 +261,8 @@
         lib.rawsetlong(p1, 0, 2782172)
         lib.stm_commit_transaction()
         lib.stm_begin_inevitable_transaction()
+        p1id = lib.stm_id(p1)
+        assert p1id == lib.stm_id(p)
         assert classify(p) == "public"
         assert classify(p1) == "protected"
         plist.append(p1)     # now p's most recent revision is protected
@@ -265,9 +273,11 @@
         assert classify(p1) == "public"
         assert lib.stm_read_barrier(p) == p1
         assert lib.stm_read_barrier(p1) == p1
+        assert lib.stm_id(p1) == p1id
     def f2(r):
         r.wait(2)
         p2 = lib.stm_read_barrier(p)    # steals
+        assert lib.stm_id(p) == lib.stm_id(p2)
         assert classify(p2) == "public"
         assert lib.rawgetlong(p2, 0) == 2782172
         assert p2 == lib.stm_read_barrier(p)    # short-circuit h_revision
@@ -297,8 +307,10 @@
         plist.append(p1)     # now p's most recent revision is protected
         assert classify(follow_revision(p)) == "stub"
         assert p1.h_revision & 1
+        p1id = lib.stm_id(p1)
         r.set(2)
         r.wait(3)
+        assert p1id == lib.stm_id(p1)
         assert classify(p1) == "public"
         assert (p1.h_revision & 1) == 0    # outdated
         p2 = ffi.cast("gcptr", p1.h_revision)
@@ -310,6 +322,7 @@
     def f2(r):
         r.wait(2)
         p2 = lib.stm_read_barrier(p)    # steals
+        assert lib.stm_id(p) == lib.stm_id(p2)
         assert classify(p2) == "public"
         assert lib.rawgetlong(p2, 0) == 2782172
         assert p2 == lib.stm_read_barrier(p)    # short-circuit h_revision
@@ -425,6 +438,8 @@
 def test_stub_for_refs_from_stolen(old=False):
     p = palloc_refs(1)
     qlist = []
+    qid = []
+    pid = []
     def f1(r):
         q1 = nalloc(HDR + WORD)
         if old:
@@ -446,6 +461,16 @@
         assert classify(p1) == "protected"
         assert classify(follow_revision(p)) == "stub"
         assert p1.h_revision & 1
+        pid.append(lib.stm_id(p1))
+        assert classify(q1) == "protected"
+        qid.append(lib.stm_id(q1))
+        if old:
+                assert q1.h_tid & GCFLAG_OLD
+                assert not (q1.h_tid & GCFLAG_HAS_ID)
+        else:
+                assert not (q1.h_tid & GCFLAG_OLD)
+                assert q1.h_tid & GCFLAG_HAS_ID
+        
         r.set(2)
         r.wait(3)     # wait until the other thread really started
     def f2(r):
@@ -454,11 +479,14 @@
         p2 = lib.stm_read_barrier(p)    # steals
         assert classify(p2) == "public"
         q2 = lib.getptr(p2, 0)
+        assert lib.stm_id(p2) == pid[0]
+        assert lib.stm_id(q2) == qid[0]
         assert q2 != ffi.NULL
         assert q2 != qlist[0]
         assert classify(q2) == "stub"
         assert q2.h_revision % 4 == 2
         q3 = lib.stm_read_barrier(q2)
+        assert lib.stm_id(q3) == qid[0]
         assert q3 != q2
         if old:
             assert q3 == qlist[0]

From noreply at buildbot.pypy.org  Mon Jun 24 13:43:04 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 13:43:04 +0200 (CEST)
Subject: [pypy-commit] stmgc default: more tests
Message-ID: <20130624114304.9439D1C0619@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r254:78560280729a
Date: 2013-06-24 13:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/78560280729a/

Log:	more tests

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -176,7 +176,7 @@
            backup, if exists */
         if (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
             gcptr B = (gcptr)p->h_revision;
-            /* don't set, otherwise nursery will copy over backup */
+            /* don't set HAS_ID, otherwise nursery will copy over backup */
             p->h_original = (revision_t)B;
             // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
             
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -496,3 +496,77 @@
 
 def test_stub_for_refs_from_stolen_old():
     test_stub_for_refs_from_stolen(old=True)
+
+
+def id_with_stealing(a=0, b=0):
+    p = palloc_refs(2)
+    qlist = []
+    rlist = []
+    qid = []
+    pid = []
+    rid = []
+    def f1(r):
+        r1 = nalloc(HDR)
+        q1 = nalloc(HDR)
+        p1 = lib.stm_write_barrier(p)   # private copy
+        qlist.append(q1)
+        rlist.append(r1)
+        if a:
+            # id on young priv
+            qid.append(lib.stm_id(q1))
+            assert q1.h_tid & GCFLAG_HAS_ID
+        if b:
+            # id on pub_to_priv
+            assert follow_original(p1) == p
+            pid.append(lib.stm_id(p1))
+            assert not (p1.h_tid & GCFLAG_HAS_ID)
+        lib.setptr(p1, 0, q1)
+        lib.setptr(p1, 1, r1)
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        # p old public -> stub -> p1 young prot
+        # q1 young prot, r1 young prot
+        if not a:
+            # id on young prot
+            qid.append(lib.stm_id(q1))
+            assert q1.h_tid & GCFLAG_HAS_ID
+        if not b:
+            # id on young prot
+            assert follow_original(p1) == p
+            pid.append(lib.stm_id(p1))
+            assert not (p1.h_tid & GCFLAG_HAS_ID)
+            
+        r1w = lib.stm_write_barrier(r1) # priv_from_prot
+        assert r1w.h_tid & GCFLAG_PRIVATE_FROM_PROTECTED
+        rid.append(lib.stm_id(r1w))
+        assert not (r1w.h_tid & GCFLAG_HAS_ID) # use backup
+        assert follow_original(r1w) == follow_revision(r1w)
+        
+        r.set(2)
+        r.wait(3)     # wait until the other thread really started
+    def f2(r):
+        r.wait(2)
+        r.set(3)
+
+        p2 = lib.stm_read_barrier(p)    # steals
+        assert pid[-1] == lib.stm_id(p2)
+
+        r2 = lib.getptr(p2, 1)
+        r3 = lib.stm_read_barrier(r2)
+        assert lib.stm_id(r3) == rid[-1]
+        
+        q2 = lib.getptr(p2, 0)
+        assert lib.stm_id(q2) == qid[-1]
+
+        q3 = lib.stm_read_barrier(q2)
+        assert lib.stm_id(q3) == qid[-1]
+
+    run_parallel(f1, f2)
+
+def test_id_with_stealing():
+    id_with_stealing(1, 1)
+    id_with_stealing(1, 0)
+    id_with_stealing(0, 1)
+    id_with_stealing(0, 0)
+
+    

From noreply at buildbot.pypy.org  Mon Jun 24 14:25:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 14:25:19 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130624122519.BE9011C2FAE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r255:314b7b686498
Date: 2013-06-24 14:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/314b7b686498/

Log:	Fix

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -350,8 +350,26 @@
 static void cleanup_for_thread(struct tx_descriptor *d)
 {
     long i;
-    gcptr *items = d->list_of_read_objects.items;
+    gcptr *items;
 
+    /* It can occur that 'private_from_protected' contains an object that
+     * has not been visited at all (maybe only in inevitable
+     * transactions). 
+     */
+    items = d->private_from_protected.items;
+    for (i = d->private_from_protected.size - 1; i >= 0; i--) {
+        gcptr obj = items[i];
+        assert(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+
+        if (!(obj->h_tid & GCFLAG_VISITED)) {
+            /* forget 'obj' */
+            items[i] = items[--d->private_from_protected.size];
+        }
+    }
+
+    /* If we're aborting this transaction anyway, we don't need to do
+     * more here.
+     */
     if (d->active < 0)
         return;       /* already "aborted" during forced minor collection */
 
@@ -360,6 +378,7 @@
         gcptrlist_clear(&d->list_of_read_objects);
     }
 
+    items = d->list_of_read_objects.items;
     for (i = d->list_of_read_objects.size - 1; i >= 0; --i) {
         gcptr obj = items[i];
         assert(!(obj->h_tid & GCFLAG_STUB));
@@ -424,21 +443,6 @@
         fprintf(stderr, "\tpublic_to_private: %p\n", item->addr);
 
     } G2L_LOOP_END;
-
-    /* It can occur that 'private_from_protected' contains an object that
-     * has not been visited at all (maybe only in inevitable
-     * transactions). 
-     */
-    items = d->private_from_protected.items;
-    for (i = d->private_from_protected.size - 1; i >= 0; i--) {
-        gcptr obj = items[i];
-        assert(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
-
-        if (!(obj->h_tid & GCFLAG_VISITED)) {
-            /* forget 'obj' */
-            items[i] = items[--d->private_from_protected.size];
-        }
-    }
 }
 
 static void clean_up_lists_of_read_objects_and_fix_outdated_flags(void)

From noreply at buildbot.pypy.org  Mon Jun 24 14:39:55 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 14:39:55 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add support for predefined hash in
	prebuilt objects
Message-ID: <20130624123955.1AAD21C14FE@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r256:67200f7bca96
Date: 2013-06-24 14:39 +0200
http://bitbucket.org/pypy/stmgc/changeset/67200f7bca96/

Log:	add support for predefined hash in prebuilt objects

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -489,9 +489,13 @@
   /* note that stmgc_duplicate() usually returns a young object, but may
      return an old one if the nursery is full at this moment. */
   gcptr L = stmgc_duplicate(R);
-  if (!(L->h_original)) {
-    /* if we don't have an original object yet,
-     it must be the old public R */
+  if (!(L->h_original) || L->h_tid & GCFLAG_PREBUILT_ORIGINAL) {
+    /* if we don't have an original object yet, it must be the 
+       old public R 
+       Also, prebuilt objects may have a predefined hash stored
+       in the h_original. -> point to the original copy on copies
+       of the prebuilt.
+    */
     assert(R->h_tid & GCFLAG_OLD); // if not, force stm_id??
     L->h_original = (revision_t)R;
   }
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -110,12 +110,6 @@
    the shadow-original.
  */
 
-revision_t stm_hash(gcptr p)
-{
-    return stm_id(p);
-}
-
-
 static revision_t mangle_hash(revision_t n)
 {
     /* To hash pointers in dictionaries.  Assumes that i shows some
@@ -127,13 +121,40 @@
     return n ^ (n >> 4);
 }
 
+
+revision_t stm_hash(gcptr p)
+{
+    /* Prebuilt objects may have a specific hash stored in an extra 
+       field. For now, we will simply always follow h_original and
+       see, if it is a prebuilt object (XXX: maybe propagate a flag
+       to all copies of a prebuilt to avoid this cache miss).
+     */
+    if (p->h_original) {
+        if (p->h_tid & GCFLAG_PREBUILT_ORIGINAL) {
+            return p->h_original;
+        }
+        gcptr orig = (gcptr)p->h_original;
+        if ((orig->h_tid & GCFLAG_PREBUILT_ORIGINAL) && orig->h_original) {
+            return orig->h_original;
+        }
+    }
+    return stm_id(p);
+}
+
+
 revision_t stm_id(gcptr p)
 {
     struct tx_descriptor *d = thread_descriptor;
     revision_t result;
 
-    
     if (p->h_original) { /* fast path */
+        if (p->h_tid & GCFLAG_PREBUILT_ORIGINAL) {
+            /* h_original may contain a specific hash value,
+               but in case of the prebuilt original version, 
+               its memory location is the id */
+            return mangle_hash((revision_t)p);
+        }
+
         fprintf(stderr, "stm_id(%p) has orig fst: %p\n", 
                 p, (gcptr)p->h_original);
         return mangle_hash(p->h_original);
@@ -152,6 +173,7 @@
         fprintf(stderr, "stm_id(%p) is old, orig=0 fst: %p\n", p, p);
         return mangle_hash((revision_t)p);
     }
+    
 
     
     spinlock_acquire(d->public_descriptor->collection_lock, 'I');
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -91,6 +91,7 @@
     void rawsetlong(gcptr, long, long);
 
     gcptr pseudoprebuilt(size_t size, int tid);
+    gcptr pseudoprebuilt_with_hash(size_t size, int tid, revision_t hash);
     revision_t get_private_rev_num(void);
     revision_t get_start_time(void);
     void *my_stub_thread(void);
@@ -217,6 +218,13 @@
         return x;
     }
 
+    gcptr pseudoprebuilt_with_hash(size_t size, int tid, revision_t hash)
+    {
+        gcptr x = pseudoprebuilt(size, tid);
+        x->h_original = hash;
+        return x;
+    }
+
     revision_t get_start_time(void)
     {
         return thread_descriptor->start_time;
@@ -473,15 +481,22 @@
         assert rawgetptr(p, i) == ffi.NULL   # must already be zero-filled
     return p
 
-def palloc(size):
+def palloc(size, prehash=None):
     "Get a ``prebuilt'' object."
-    p = lib.pseudoprebuilt(size, 42 + size)
+    if prehash is None:
+        p = lib.pseudoprebuilt(size, 42 + size)
+    else:
+        p = lib.pseudoprebuilt_with_hash(size, 42 + size, prehash)
     assert p.h_revision == 1
     return p
 
-def palloc_refs(nrefs):
+def palloc_refs(nrefs, prehash=None):
     "Get a ``prebuilt'' object with nrefs pointers."
-    p = lib.pseudoprebuilt(HDR + WORD * nrefs, 421 + nrefs)
+    if prehash is None:
+        p = lib.pseudoprebuilt(HDR + WORD * nrefs, 421 + nrefs)
+    else:
+        p = lib.pseudoprebuilt_with_hash(HDR + WORD * nrefs,
+                                         421 + nrefs, prehash)
     return p
 
 gettid = lib.gettid
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -569,4 +569,44 @@
     id_with_stealing(0, 1)
     id_with_stealing(0, 0)
 
+
+def test_prehash_simple():
+    p = palloc(HDR, 99)
+    assert lib.stm_hash(p) == 99
+    assert lib.stm_id(p) != lib.stm_hash(p)
+    pr = lib.stm_read_barrier(p)
+    assert lib.stm_hash(pr) == 99
+    assert lib.stm_id(p) == lib.stm_id(pr)
+    pw = lib.stm_write_barrier(p)
+    assert lib.stm_hash(pw) == 99
+    assert lib.stm_id(p) == lib.stm_id(pw)
+    lib.stm_push_root(pw)
+    minor_collect()
+    pw = lib.stm_pop_root()
+    assert lib.stm_hash(pw) == 99
+
+
+def test_prehash_with_stealing():
+    p = palloc(HDR, 99)
+    def f1(r):
+        lib.stm_write_barrier(p)   # private copy
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
+        # p old public -> stub -> p1 young prot
+        r.set(2)
+        r.wait(3)     # wait until the other thread really started
+    def f2(r):
+        r.wait(2)
+        r.set(3)
+
+        p2 = lib.stm_read_barrier(p)    # steals
+        assert lib.stm_hash(p) == 99
+        assert lib.stm_hash(p2) == 99
+        assert lib.stm_id(p) == lib.stm_id(p2)
+        assert lib.stm_hash(p) != lib.stm_id(p)
+    run_parallel(f1, f2)
+
+
+
     
+    

From noreply at buildbot.pypy.org  Mon Jun 24 14:58:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 14:58:30 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Two extras passing tests
Message-ID: <20130624125830.741A21C14FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r257:b9d9edc34feb
Date: 2013-06-24 14:58 +0200
http://bitbucket.org/pypy/stmgc/changeset/b9d9edc34feb/

Log:	Two extras passing tests

diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -383,3 +383,33 @@
     p1 = lib.stm_pop_root()
     check_not_free(p1)
     check_not_free(lib.getptr(p1, 0))
+
+def test_prebuilt_modified_during_transaction():
+    p1 = palloc(HDR)
+    p2 = nalloc_refs(1)
+    lib.setptr(p2, 0, p1)
+    lib.stm_push_root(p2)
+    major_collect()
+    major_collect()
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b != p1
+    major_collect()
+    lib.stm_pop_root()
+    p1b = lib.stm_read_barrier(p1)
+    check_not_free(p1b)
+
+def test_prebuilt_modified_later():
+    p1 = palloc(HDR)
+    p2 = nalloc_refs(1)
+    lib.setptr(p2, 0, p1)
+    lib.stm_push_root(p2)
+    major_collect()
+    major_collect()
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b != p1
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    major_collect()
+    lib.stm_pop_root()
+    p1b = lib.stm_read_barrier(p1)
+    check_not_free(p1b)
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -522,4 +522,4 @@
 def test_more_multi_thread():
     #py.test.skip("more random tests")
     for i in range(200):
-        yield test_multi_thread, 1690 + i
+        yield test_multi_thread, 1751 + i

From noreply at buildbot.pypy.org  Mon Jun 24 15:01:25 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 15:01:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add demo_random as a test
Message-ID: <20130624130125.8A0211C14FE@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r258:3e317d4ac80d
Date: 2013-06-24 15:00 +0200
http://bitbucket.org/pypy/stmgc/changeset/3e317d4ac80d/

Log:	add demo_random as a test

diff --git a/c4/test/test_zdemo_random.py b/c4/test/test_zdemo_random.py
new file mode 100644
--- /dev/null
+++ b/c4/test/test_zdemo_random.py
@@ -0,0 +1,13 @@
+import py
+import os
+import subprocess
+
+def test_and_run():
+    path = os.path.dirname(__file__)
+    path = os.path.dirname(path)
+    res = subprocess.call(["make", "debug-demo_random"], cwd=path)
+    assert not res
+    res = subprocess.call(["./debug-demo_random"], cwd=path)
+    assert not res
+    
+    

From noreply at buildbot.pypy.org  Mon Jun 24 15:01:26 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 15:01:26 +0200 (CEST)
Subject: [pypy-commit] stmgc default: merge
Message-ID: <20130624130126.9975C1C14FE@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r259:076863fce4d4
Date: 2013-06-24 15:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/076863fce4d4/

Log:	merge

diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -383,3 +383,33 @@
     p1 = lib.stm_pop_root()
     check_not_free(p1)
     check_not_free(lib.getptr(p1, 0))
+
+def test_prebuilt_modified_during_transaction():
+    p1 = palloc(HDR)
+    p2 = nalloc_refs(1)
+    lib.setptr(p2, 0, p1)
+    lib.stm_push_root(p2)
+    major_collect()
+    major_collect()
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b != p1
+    major_collect()
+    lib.stm_pop_root()
+    p1b = lib.stm_read_barrier(p1)
+    check_not_free(p1b)
+
+def test_prebuilt_modified_later():
+    p1 = palloc(HDR)
+    p2 = nalloc_refs(1)
+    lib.setptr(p2, 0, p1)
+    lib.stm_push_root(p2)
+    major_collect()
+    major_collect()
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b != p1
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()
+    major_collect()
+    lib.stm_pop_root()
+    p1b = lib.stm_read_barrier(p1)
+    check_not_free(p1b)
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -522,4 +522,4 @@
 def test_more_multi_thread():
     #py.test.skip("more random tests")
     for i in range(200):
-        yield test_multi_thread, 1690 + i
+        yield test_multi_thread, 1751 + i

From noreply at buildbot.pypy.org  Mon Jun 24 15:16:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 15:16:21 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Bah, it was only missing this
Message-ID: <20130624131621.AB7961C323C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r260:b12a5519539c
Date: 2013-06-24 15:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/b12a5519539c/

Log:	Bah, it was only missing this

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -43,6 +43,7 @@
         self.dump('run')
         #
         if sync.prebuilt_object is None:
+            lib.stm_clear_between_tests()
             lib.stm_initialize_tests(0)
             self.startrev()
             sync.prebuilt_object = self.alloc(palloc_refs)

From noreply at buildbot.pypy.org  Mon Jun 24 15:26:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 15:26:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Next small issue with the test
Message-ID: <20130624132638.9D14E1C323C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r261:5118271cff1a
Date: 2013-06-24 15:26 +0200
http://bitbucket.org/pypy/stmgc/changeset/5118271cff1a/

Log:	Next small issue with the test

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -150,19 +150,19 @@
         #
         return x, y
 
-    def do_check_can_still_commit(self):
+    def do_check_can_still_commit(self, real_operation):
         try:
             self.current_rev.check_can_still_commit()
         except (model.Deleted, model.Conflict), e:
             # the model says that we might get an abort
             self.dump('possible delayed abort!')
             self.expected_abort()
-            lib.AbortNowIfDelayed()
+            real_operation()
             # ok, it's fine if we don't actually get an abort
             self.cancel_expected_abort()
         else:
             # the model says that we must not get an abort
-            lib.AbortNowIfDelayed()
+            real_operation()
 
     def get_ref(self, r, index):
         self.check(r)
@@ -371,7 +371,7 @@
             if do_wait:
                 self.push_roots(extra=p)
                 do_wait()
-                self.do_check_can_still_commit()
+                self.do_check_can_still_commit(lib.AbortNowIfDelayed)
                 p = self.pop_roots(extra=p)
 
     def run_single_thread(self):
@@ -427,7 +427,7 @@
                 self.dump('major collect')
                 self.push_roots()
                 if DO_MAJOR_COLLECTS:
-                    major_collect()
+                    self.do_check_can_still_commit(major_collect)
                 self.pop_roots()
                 p = emptypair
             if k1 == 82 and self.interruptible_transaction:
@@ -523,4 +523,4 @@
 def test_more_multi_thread():
     #py.test.skip("more random tests")
     for i in range(200):
-        yield test_multi_thread, 1751 + i
+        yield test_multi_thread, 1858 + i

From noreply at buildbot.pypy.org  Mon Jun 24 15:38:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 15:38:30 +0200 (CEST)
Subject: [pypy-commit] stmgc default: 200 random tests pass again.
Message-ID: <20130624133830.0EE811C323D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r262:3a599531f404
Date: 2013-06-24 15:38 +0200
http://bitbucket.org/pypy/stmgc/changeset/3a599531f404/

Log:	200 random tests pass again.

diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -521,6 +521,6 @@
     test_multi_thread(1624)
 
 def test_more_multi_thread():
-    #py.test.skip("more random tests")
+    py.test.skip("more random tests")
     for i in range(200):
         yield test_multi_thread, 1858 + i

From noreply at buildbot.pypy.org  Mon Jun 24 16:14:37 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Jun 2013 16:14:37 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add hash to demo_random,
	crashes sometimes..
Message-ID: <20130624141437.EB01B1C034D@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r263:05dae094fa06
Date: 2013-06-24 16:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/05dae094fa06/

Log:	add hash to demo_random, crashes sometimes..

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -29,6 +29,7 @@
     struct stm_object_s hdr;
     long value;
     revision_t id;
+    revision_t hash;
     struct node *next;
 };
 typedef struct node * nodeptr;
@@ -84,7 +85,14 @@
     gcptr x = calloc(1, size);
     x->h_tid = PREBUILT_FLAGS | tid;
     x->h_revision = PREBUILT_REVISION;
-    x->h_original = 0;
+    return x;
+}
+
+gcptr allocate_pseudoprebuilt_with_hash(size_t size, int tid,
+                                        revision_t hash)
+{
+    gcptr x = allocate_pseudoprebuilt(size, tid);
+    x->h_original = hash;
     return x;
 }
 
@@ -240,8 +248,17 @@
     
     td.num_roots = PREBUILT + NUMROOTS;
     for (i = 0; i < PREBUILT; i++) {
-        td.roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
-                                              GCTID_STRUCT_NODE);
+        if (i % 3 == 0) {
+            td.roots[i] = allocate_pseudoprebuilt_with_hash(
+                                          sizeof(struct node), 
+                                          GCTID_STRUCT_NODE,
+                                          i);
+            ((nodeptr)td.roots[i])->hash = i;
+        }
+        else {
+            td.roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
+                                                  GCTID_STRUCT_NODE);
+        }
     }
     for (i = PREBUILT; i < PREBUILT + NUMROOTS; i++) {
         td.roots[i] = (gcptr)allocate_node();
@@ -251,17 +268,19 @@
 
 gcptr do_step(gcptr p)
 {
-    nodeptr w_r, w_sr;
-    gcptr _r, _sr;
+    nodeptr w_r, w_sr, w_t;
+    gcptr _r, _sr, _t;
     int num, k;
 
+    _t = NULL;
     num = get_rand(td.num_roots);
     _r = td.roots[num];
-
+    
     num = get_rand(SHARED_ROOTS);
     _sr = shared_roots[num];
 
-    k = get_rand(17);
+    k = get_rand(19);
+    check(p);
 
     switch (k) {
     case 0: // remove a root
@@ -323,28 +342,42 @@
         pop_roots();
         p = NULL;
         break;
-    case 15: /* test stm_id on non-shared roots */
-        w_r = (nodeptr)read_barrier(_r);
-        if (w_r->id) {
-            assert(w_r->id == stm_id((gcptr)w_r));
-            assert(w_r->id == stm_id((gcptr)_r));
+    case 15: /* test stm_id on (non-)shared roots */
+        _t = _r;
+    case 16:
+        if (!_t)
+            _t = _sr;
+        w_t = (nodeptr)read_barrier(_t);
+        if (w_t->id) {
+            assert(w_t->id == stm_id((gcptr)w_t));
+            assert(w_t->id == stm_id((gcptr)_t));
         } 
         else {
-            w_r = (nodeptr)write_barrier(_r);
-            w_r->id = stm_id((gcptr)w_r);
-            assert(w_r->id == stm_id((gcptr)_r));
+            w_t = (nodeptr)write_barrier(_t);
+            w_t->id = stm_id((gcptr)w_t);
+            assert(w_t->id == stm_id((gcptr)_t));
         }
-    case 16: /* test stm_id on shared roots */
-        w_sr = (nodeptr)read_barrier(_sr);
-        if (w_sr->id) {
-            assert(w_sr->id == stm_id((gcptr)w_sr));
-            assert(w_sr->id == stm_id((gcptr)_sr));
-        } 
+        break;
+    case 17: /* test stm_hash on (non-)shared roots */
+        _t = _r;
+    case 18:
+        if (!_t)
+            _t = _sr;
+        w_t = (nodeptr)read_barrier(_t);
+        if (w_t->hash) {
+            assert(w_t->hash == stm_hash((gcptr)w_t));
+            assert(w_t->hash == stm_hash((gcptr)_t));
+        }
         else {
-            w_sr = (nodeptr)write_barrier(_sr);
-            w_sr->id = stm_id((gcptr)w_sr);
-            assert(w_sr->id == stm_id((gcptr)_sr));
+            w_t = (nodeptr)write_barrier(_t);
+            w_t->hash = stm_hash((gcptr)w_t);
+            assert(w_t->hash == stm_hash((gcptr)_t));
         }
+        if (w_t->hash < PREBUILT || w_t->hash < SHARED_ROOTS) {
+            // should be with predefined hash
+            assert (stm_id((gcptr)w_t) != stm_hash((gcptr)w_t));
+        }
+        break;
     }
     return p;
 }
@@ -449,8 +482,17 @@
     default_seed = time(NULL) / 3600 / 24;
     
     for (i = 0; i < SHARED_ROOTS; i++) {
-        shared_roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
-                                                  GCTID_STRUCT_NODE);
+        if (i % 3 == 0) {
+            shared_roots[i] = allocate_pseudoprebuilt_with_hash(
+                                          sizeof(struct node), 
+                                          GCTID_STRUCT_NODE,
+                                          i);
+            ((nodeptr)shared_roots[i])->hash = i;
+        }
+        else {
+            shared_roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
+                                                      GCTID_STRUCT_NODE);
+        }
     }    
     
     status = sem_init(&done, 0, 0);

From noreply at buildbot.pypy.org  Mon Jun 24 17:05:55 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 17:05:55 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Replace fprintf(stderr,
 ...) with dprintf((...)), which is turned off if
Message-ID: <20130624150555.196501C2FAC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r264:104bd28d6f04
Date: 2013-06-24 17:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/104bd28d6f04/

Log:	Replace fprintf(stderr, ...) with dprintf((...)), which is turned
	off if _GC_DEBUG is not defined. Now the "build-demo*" executables
	are incredibly fast --- almost all the time was spent printing this
	debug stuff.

diff --git a/c4/Makefile b/c4/Makefile
--- a/c4/Makefile
+++ b/c4/Makefile
@@ -2,11 +2,14 @@
 # Makefile for the demos.
 #
 
-build: build-demo1 build-demo2 build-demo_random
-debug: debug-demo1 debug-demo2 debug-demo_random
+BUILD_EXE = build-demo1 build-demo2 build-demo_random
+DEBUG_EXE = debug-demo1 debug-demo2 debug-demo_random
+
+build: $(BUILD_EXE)
+debug: $(DEBUG_EXE)
 
 clean:
-	rm -f debug-demo1
+	rm -f $(BUILD_EXE) $(DEBUG_EXE)
 
 
 H_FILES = atomic_ops.h stmgc.h stmimpl.h \
@@ -16,9 +19,10 @@
 C_FILES = et.c lists.c steal.c nursery.c gcpage.c \
           stmsync.c dbgmem.c fprintcolor.c
 
-DEBUG = -g -DGC_NURSERY=0x10000  #-D_GC_DEBUG=1
+DEBUG = -g -DGC_NURSERY=0x10000 -D_GC_DEBUG=1
 
 
+# note that we don't say -DNDEBUG, so that asserts should still be compiled it
 build-%: %.c ${H_FILES} ${C_FILES} stmgc.c
 	gcc -lrt -pthread -O2 -g $< -o build-$* -Wall stmgc.c
 
diff --git a/c4/atomic_ops.h b/c4/atomic_ops.h
--- a/c4/atomic_ops.h
+++ b/c4/atomic_ops.h
@@ -99,13 +99,13 @@
 #if 0    /* fprinting versions */
 # define spinlock_acquire(lock, targetvalue)                            \
     do { if (bool_cas(&(lock), 0, (targetvalue))) {                     \
-             fprintf(stderr, "<<< locked %d\n", (int)targetvalue);      \
+             dprintf(("<<< locked %d\n", (int)targetvalue));            \
              break;                                                     \
          }                                                              \
          do { smp_spinloop(); } while (ACCESS_ONCE(lock));              \
     } while (1)
 # define spinlock_release(lock)                                         \
-    do { fprintf(stderr, "unlocked >>>\n"); smp_wmb();                  \
+    do { dprintf(("unlocked >>>\n")); smp_wmb();                        \
          assert((lock) != 0); (lock) = 0; } while (0)
 #else
 # define spinlock_acquire(lock, targetvalue)                    \
diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -22,7 +22,7 @@
     intptr_t align = ((intptr_t)p) & (PAGE_SIZE-1);
     p = ((char *)p) - align;
     sz += align;
-    fprintf(stderr, "dbgmem: %p, %ld, %d\n", p, (long)sz, prot);
+    dprintf(("dbgmem: %p, %ld, %d\n", p, (long)sz, prot));
     int err = mprotect(p, sz, prot);
     assert(err == 0);
 }
@@ -35,8 +35,7 @@
         zone_start = mmap(NULL, MMAP_TOTAL, PROT_NONE,
                           MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
         if (zone_start == NULL || zone_start == MAP_FAILED) {
-            fprintf(stderr, "not enough memory: mmap() failed\n");
-            abort();
+            stm_fatalerror("not enough memory: mmap() failed\n");
         }
         zone_current = zone_start;
         zone_end = zone_start + MMAP_TOTAL;
@@ -48,9 +47,8 @@
     char *result = zone_current;
     zone_current += nb_pages * PAGE_SIZE;
     if (zone_current > zone_end) {
-        fprintf(stderr, "dbgmem.c: %ld MB of memory have been exausted\n",
-                (long)(MMAP_TOTAL / (1024*1024)));
-        abort();
+        stm_fatalerror("dbgmem.c: %ld MB of memory have been exausted\n",
+                       (long)(MMAP_TOTAL / (1024*1024)));
     }
     pthread_mutex_unlock(&malloc_mutex);
 
diff --git a/c4/demo1.c b/c4/demo1.c
--- a/c4/demo1.c
+++ b/c4/demo1.c
@@ -66,7 +66,7 @@
 
       r_n = (struct node *)stm_read_barrier((gcptr)r_n->next);
       long v = r_n->value;
-      fprintf(stderr, "\t\t\t\t{ %ld, %p }\n", v, r_n->next);
+      dprintf(("\t\t\t\t{ %ld, %p }\n", v, r_n->next));
       assert(0 <= v && v < UPPER_BOUND * NUMTHREADS);
       int tn = v / UPPER_BOUND;
       assert(seen[tn] == v % UPPER_BOUND);
@@ -95,10 +95,10 @@
     assert(seen[nvalue / UPPER_BOUND] == nvalue % UPPER_BOUND);
     w_newnode->value = nvalue;
     w_newnode->next = NULL;
-    fprintf(stderr, "DEMO1: %p->value = %ld\n", w_newnode, nvalue);
+    dprintf(("DEMO1: %p->value = %ld\n", w_newnode, nvalue));
 
     struct node *w_last = (struct node *)stm_write_barrier((gcptr)last);
-    fprintf(stderr, "DEMO1:   %p->next = %p\n", w_last, w_newnode);
+    dprintf(("DEMO1:   %p->next = %p\n", w_last, w_newnode));
     w_last->next = w_newnode;
 
     return 0;   /* return from stm_perform_transaction() */
@@ -118,7 +118,7 @@
 
     start = thr_mynum++;   /* protected by being inevitable here */
     start *= UPPER_BOUND;
-    fprintf(stderr, "THREAD STARTING: start = %d\n", start);
+    dprintf(("THREAD STARTING: start = %d\n", start));
 
     w_node = (struct node *)stm_allocate(sizeof(struct node),
                                          GCTID_STRUCT_NODE);
diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -137,7 +137,7 @@
     stm_initialize();
     
     thr_mynum++;   /* protected by being inevitable here */
-    fprintf(stderr, "THREAD STARTING\n");
+    dprintf(("THREAD STARTING\n"));
     
     
     while (check_sorted() == -1) {
diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -417,7 +417,7 @@
     setup_thread();
     
     thr_mynum++;   /* protected by being inevitable here */
-    fprintf(stderr, "THREAD STARTING\n");
+    dprintf(("THREAD STARTING\n"));
 
     run_me();
     
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -144,7 +144,7 @@
           */
           if (FXCACHE_AT(P) == P)
             {
-              fprintf(stderr, "read_barrier: %p -> %p fxcache\n", G, P);
+              dprintf(("read_barrier: %p -> %p fxcache\n", G, P));
               return P;
             }
         }
@@ -164,7 +164,7 @@
           P = item->val;
           assert(!(P->h_tid & GCFLAG_PUBLIC));
           assert(is_private(P));
-          fprintf(stderr, "read_barrier: %p -> %p public_to_private\n", G, P);
+          dprintf(("read_barrier: %p -> %p public_to_private\n", G, P));
           return P;
 
         no_private_obj:
@@ -190,13 +190,13 @@
           ValidateNow(d);                  // try to move start_time forward
           goto restart_all_public;         // restart searching from P
         }
-      fprintf(stderr, "read_barrier: %p -> %p public\n", G, P);
+      dprintf(("read_barrier: %p -> %p public\n", G, P));
     }
   else
     {
       /* Not private and not public: it's a protected object
        */
-      fprintf(stderr, "read_barrier: %p -> %p protected\n", G, P);
+      dprintf(("read_barrier: %p -> %p protected\n", G, P));
 
       /* The risks are not high, but in parallel it's possible for the
          object to be stolen by another thread and become public, after
@@ -205,7 +205,7 @@
       /*assert(P->h_revision & 1);*/
     }
 
-  fprintf(stderr, "readobj: %p\n", P);
+  dprintf(("readobj: %p\n", P));
   assert(!(P->h_tid & GCFLAG_STUB));
   gcptrlist_insert(&d->list_of_read_objects, P);
 
@@ -227,7 +227,7 @@
          we reach any kind of object, even a public object, in case it
          was stolen.  So we just repeat the whole procedure. */
       P = (gcptr)(v - 2);
-      fprintf(stderr, "read_barrier: %p -> %p via stub\n  ", G, P);
+      dprintf(("read_barrier: %p -> %p via stub\n  ", G, P));
 
       if (UNLIKELY((P->h_revision != stm_private_rev_num) &&
                    (FXCACHE_AT(P) != P)))
@@ -238,7 +238,7 @@
   else
     {
       /* stealing */
-      fprintf(stderr, "read_barrier: %p -> stealing %p...\n  ", G, P);
+      dprintf(("read_barrier: %p -> stealing %p...\n  ", G, P));
       stm_steal_stub(P);
 
       assert(P->h_tid & GCFLAG_PUBLIC);
@@ -262,12 +262,12 @@
       assert(!(privobj->h_tid & GCFLAG_PUBLIC));
       assert(is_private(privobj));
       if (P != org_pubobj)
-        fprintf(stderr, "| actually %p ", org_pubobj);
+        dprintf(("| actually %p ", org_pubobj));
       if (from_stolen)
-        fprintf(stderr, "-stolen");
+        dprintf(("-stolen"));
       else
         assert(org_pubobj->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
-      fprintf(stderr, "-public_to_private-> %p private\n", privobj);
+      dprintf(("-public_to_private-> %p private\n", privobj));
       return privobj;
     }
   return NULL;
@@ -308,19 +308,19 @@
   struct tx_descriptor *d = thread_descriptor;
   if (P->h_tid & GCFLAG_STUB)
     {
-      fprintf(stderr, "S");
+      dprintf(("S"));
     }
   int is_old = (P->h_tid & GCFLAG_OLD) != 0;
   int in_nurs = (d->nursery_base <= (char*)P && ((char*)P) < d->nursery_end);
   if (in_nurs)
     {
       assert(!is_old);
-      fprintf(stderr, "Y ");
+      dprintf(("Y "));
     }
   else
     {
       assert(is_old);
-      fprintf(stderr, "O ");
+      dprintf(("O "));
     }
 }
 
@@ -330,14 +330,14 @@
   struct tx_descriptor *d = thread_descriptor;
   revision_t v;
 
-  fprintf(stderr, "_stm_nonrecord_barrier: %p ", P);
+  dprintf(("_stm_nonrecord_barrier: %p ", P));
   _check_flags(P);
 
  restart_all:
   if (P->h_revision == stm_private_rev_num)
     {
       /* private */
-      fprintf(stderr, "private\n");
+      dprintf(("private\n"));
       return P;
     }
 
@@ -345,22 +345,22 @@
     {
       /* private too, with a backup copy */
       assert(IS_POINTER(P->h_revision));
-      fprintf(stderr, "private_from_protected\n");
+      dprintf(("private_from_protected\n"));
       return P;
     }
 
   if (P->h_tid & GCFLAG_PUBLIC)
     {
-      fprintf(stderr, "public ");
+      dprintf(("public "));
 
       while (v = P->h_revision, IS_POINTER(v))
         {
           if (P->h_tid & GCFLAG_NURSERY_MOVED)
-            fprintf(stderr, "nursery_moved ");
+            dprintf(("nursery_moved "));
 
           if (v & 2)
             {
-              fprintf(stderr, "stub ");
+              dprintf(("stub "));
               gcptr L = _find_public_to_private(P);
               if (L != NULL)
                 return L;
@@ -369,7 +369,7 @@
 
           P = (gcptr)v;
           assert(P->h_tid & GCFLAG_PUBLIC);
-          fprintf(stderr, "-> %p public ", P);
+          dprintf(("-> %p public ", P));
           _check_flags(P);
         }
 
@@ -379,14 +379,14 @@
 
       if (UNLIKELY(v > d->start_time))
         {
-          fprintf(stderr, "too recent!\n");
+          dprintf(("too recent!\n"));
           return NULL;   // object too recent
         }
-      fprintf(stderr, "\n");
+      dprintf(("\n"));
     }
   else
     {
-      fprintf(stderr, "protected\n");
+      dprintf(("protected\n"));
     }
   return P;
 
@@ -394,23 +394,23 @@
   if (STUB_THREAD(P) == d->public_descriptor)
     {
       P = (gcptr)(v - 2);
-      fprintf(stderr, "-> %p ", P);
+      dprintf(("-> %p ", P));
       _check_flags(P);
     }
   else
     {
       P = (gcptr)(v - 2);
       /* cannot _check_flags(P): foreign! */
-      fprintf(stderr, "-foreign-> %p ", P);
+      dprintf(("-foreign-> %p ", P));
       if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
         {
           P = (gcptr)P->h_revision;     /* the backup copy */
           /* cannot _check_flags(P): foreign! */
-          fprintf(stderr, "-backup-> %p ", P);
+          dprintf(("-backup-> %p ", P));
         }
       if (!(P->h_tid & GCFLAG_PUBLIC))
         {
-          fprintf(stderr, "protected by someone else!\n");
+          dprintf(("protected by someone else!\n"));
           return (gcptr)-1;
         }
     }
@@ -467,7 +467,7 @@
   P->h_revision = (revision_t)B;
 
   gcptrlist_insert(&d->private_from_protected, P);
-  fprintf(stderr, "private_from_protected: insert %p (backup %p)\n", P, B);
+  dprintf(("private_from_protected: insert %p (backup %p)\n", P, B));
 
   return P;   /* always returns its arg: the object is converted in-place */
 }
@@ -480,7 +480,7 @@
 #ifdef _GC_DEBUG
   wlog_t *entry;
   G2L_FIND(d->public_to_private, R, entry, goto not_found);
-  assert(!"R is already in public_to_private");
+  stm_fatalerror("R is already in public_to_private\n");
  not_found:
 #endif
 
@@ -513,8 +513,8 @@
   assert(stm_private_rev_num < 0);
   assert(stm_private_rev_num & 1);
   g2l_insert(&d->public_to_private, R, L);
-  fprintf(stderr, "write_barrier: adding %p -> %p to public_to_private\n",
-          R, L);
+  dprintf(("write_barrier: adding %p -> %p to public_to_private\n",
+           R, L));
 
   /* must remove R from the read_barrier_cache, because returning R is no
      longer a valid result */
@@ -603,7 +603,7 @@
 
   spinlock_release(d->public_descriptor->collection_lock);
 
-  fprintf(stderr, "write_barrier: %p -> %p -> %p\n", P, R, W);
+  dprintf(("write_barrier: %p -> %p -> %p\n", P, R, W));
 
   return W;
 }
@@ -661,8 +661,8 @@
             }
           else
             {
-              fprintf(stderr, "validation failed: "
-                      "%p has a more recent revision\n", R);
+              dprintf(("validation failed: "
+                       "%p has a more recent revision\n", R));
               return 0;
             }
         }
@@ -696,8 +696,8 @@
 
                      For now we always abort.
                   */
-                  fprintf(stderr, "validation failed: "
-                          "%p is locked by another thread\n", R);
+                  dprintf(("validation failed: "
+                           "%p is locked by another thread\n", R));
                   return 0;
                 }
             }
@@ -709,7 +709,7 @@
 static void ValidateNow(struct tx_descriptor *d)
 {
   d->start_time = GetGlobalCurTime(d);   // copy from the global time
-  fprintf(stderr, "et.c: ValidateNow: %ld\n", (long)d->start_time);
+  dprintf(("et.c: ValidateNow: %ld\n", (long)d->start_time));
 
   /* subtle: we have to normalize stolen objects, because doing so
      might add a few extra objects in the list_of_read_objects */
@@ -792,11 +792,8 @@
         {
           if (_stm_decode_abort_info(d, elapsed_time,
                                      num, d->longest_abort_info) != size)
-            {
-              fprintf(stderr,
-                      "during stm abort: object mutated unexpectedly\n");
-              abort();
-            }
+            stm_fatalerror("during stm abort: object mutated unexpectedly\n");
+
           d->longest_abort_info_time = elapsed_time;
         }
     }
@@ -836,19 +833,15 @@
   /* release the lock */
   spinlock_release(d->public_descriptor->collection_lock);
 
-  fprintf(stderr,
-          "\n"
+  dprintf(("\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
           "!!!!!!!!!!!!!!!!!!!!!  [%lx] abort %d\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
           "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-          "\n", (long)d->public_descriptor_index, num);
+          "\n", (long)d->public_descriptor_index, num));
   if (num != ABRT_MANUAL && d->max_aborts >= 0 && !d->max_aborts--)
-    {
-      fprintf(stderr, "unexpected abort!\n");
-      abort();
-    }
+    stm_fatalerror("unexpected abort!\n");
 
   // notifies the CPU that we're potentially in a spin loop
   SpinLoop(SPLP_ABORT);
@@ -862,7 +855,7 @@
 {
   if (d->active >= 0)
     {
-      fprintf(stderr, "abort %d after collect!\n", reason);
+      dprintf(("abort %d after collect!\n", reason));
       assert(d->active == 1);   /* not 2, which means inevitable */
       d->active = -reason;
     }
@@ -998,7 +991,7 @@
       L->h_revision = expected;
 
 #ifdef DUMP_EXTRA
-      fprintf(stderr, "%p->h_revision = %p (CancelLocks)\n", R, (gcptr)v);
+      dprintf(("%p->h_revision = %p (CancelLocks)\n", R, (gcptr)v));
 #endif
       assert(R->h_revision == my_lock);
       ACCESS_ONCE(R->h_revision) = v;
@@ -1028,8 +1021,8 @@
       assert(L->h_revision != localrev);   /* modified by AcquireLocks() */
 
 #ifdef DUMP_EXTRA
-      fprintf(stderr, "%p->h_revision = %p (UpdateChainHeads)\n",
-              L, (gcptr)new_revision);
+      dprintf(("%p->h_revision = %p (UpdateChainHeads)\n",
+               L, (gcptr)new_revision));
 #endif
       L->h_revision = new_revision;
 
@@ -1068,9 +1061,8 @@
       assert(R->h_revision != localrev);
 
 #ifdef DUMP_EXTRA
-      fprintf(stderr, "%p->h_revision = %p (stub to %p)\n",
-              R, (gcptr)v, (gcptr)item->val->h_revision);
-      /*mark*/
+      dprintf(("%p->h_revision = %p (stub to %p)\n",
+               R, (gcptr)v, (gcptr)item->val->h_revision));
 #endif
       ACCESS_ONCE(R->h_revision) = v;
 
@@ -1139,13 +1131,13 @@
       else
         {
           stmgcpage_free(B);
-          fprintf(stderr, "commit: free backup at %p\n", B);
+          dprintf(("commit: free backup at %p\n", B));
         }
     };
   gcptrlist_clear(&d->private_from_protected);
   d->num_private_from_protected_known_old = 0;
   d->num_read_objects_known_old = 0;
-  fprintf(stderr, "private_from_protected: clear (commit)\n");
+  dprintf(("private_from_protected: clear (commit)\n"));
 }
 
 void AbortPrivateFromProtected(struct tx_descriptor *d)
@@ -1201,13 +1193,13 @@
                  size - offsetof(struct stm_object_s, h_revision));
           assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
           stmgcpage_free(B);
-          fprintf(stderr, "abort: free backup at %p\n", B);
+          dprintf(("abort: free backup at %p\n", B));
         }
     };
   gcptrlist_clear(&d->private_from_protected);
   d->num_private_from_protected_known_old = 0;
   d->num_read_objects_known_old = 0;
-  fprintf(stderr, "private_from_protected: clear (abort)\n");
+  dprintf(("private_from_protected: clear (abort)\n"));
 }
 
 void CommitTransaction(void)
@@ -1227,8 +1219,7 @@
       cur_time = d->start_time;
       if (!bool_cas(&global_cur_time, cur_time + 1, cur_time + 2))
         {
-          assert(!"global_cur_time modified even though we are inev.");
-          abort();
+          stm_fatalerror("global_cur_time modified even though we are inev\n");
         }
       inev_mutex_release();
     }
@@ -1265,11 +1256,11 @@
   d->setjmp_buf = NULL;
   gcptrlist_clear(&d->list_of_read_objects);
 
-  fprintf(stderr, "\n"
+  dprintf(("\n"
           "*************************************\n"
           "**************************************  committed %ld\n"
           "*************************************\n",
-          (long)cur_time);
+           (long)cur_time));
 
   revision_t localrev = stm_private_rev_num;
   //UpdateProtectedChainHeads(d, cur_time, localrev);
@@ -1278,7 +1269,7 @@
   revision_t newrev = -(cur_time + 1);
   assert(newrev & 1);
   ACCESS_ONCE(stm_private_rev_num) = newrev;
-  fprintf(stderr, "%p: stm_local_revision = %ld\n", d, (long)newrev);
+  dprintf(("%p: stm_local_revision = %ld\n", d, (long)newrev));
   assert(d->private_revision_ref == &stm_private_rev_num);
   assert(d->read_barrier_cache_ref == &stm_read_barrier_cache);
 
@@ -1326,8 +1317,8 @@
                 (XXX statically we should know when we're outside
                 a transaction) */
 
-  fprintf(stderr, "[%lx] inevitable: %s\n",
-          (long)d->public_descriptor_index, why);
+  dprintf(("[%lx] inevitable: %s\n",
+           (long)d->public_descriptor_index, why));
 
   cur_time = acquire_inev_mutex_and_mark_global_cur_time(d);
   if (d->start_time != cur_time)
@@ -1458,7 +1449,7 @@
 void _stm_test_forget_previous_state(void)
 {
   /* debug: reset all global states, between tests */
-  fprintf(stderr, "=======================================================\n");
+  dprintf(("=======================================================\n"));
   assert(thread_descriptor == NULL);
   memset(stm_descriptor_array, 0, sizeof(stm_descriptor_array));
   descriptor_array_free_list = 0;
@@ -1484,9 +1475,8 @@
 {
   if (GCFLAG_PREBUILT != PREBUILT_FLAGS)
     {
-      fprintf(stderr, "fix PREBUILT_FLAGS in stmgc.h by giving "
-                      "it the same value as GCFLAG_PREBUILT!\n");
-      abort();
+      stm_fatalerror("fix PREBUILT_FLAGS in stmgc.h by giving "
+                     "it the same value as GCFLAG_PREBUILT!\n");
     }
 
   if (thread_descriptor == NULL)
@@ -1515,9 +1505,8 @@
           /* no item in the free list */
           descriptor_array_free_list = i + 1;
           if (descriptor_array_free_list == MAX_THREADS) {
-              fprintf(stderr, "error: too many threads at the same time "
-                              "in this process");
-              abort();
+              stm_fatalerror("error: too many threads at the same time "
+                             "in this process");
           }
           pd = stm_malloc(sizeof(struct tx_public_descriptor));
           memset(pd, 0, sizeof(struct tx_public_descriptor));
@@ -1540,8 +1529,8 @@
       stm_tx_head = d;
       thread_descriptor = d;
 
-      fprintf(stderr, "[%lx] pthread %lx starting\n",
-              (long)d->public_descriptor_index, (long)pthread_self());
+      dprintf(("[%lx] pthread %lx starting\n",
+               (long)d->public_descriptor_index, (long)pthread_self()));
 
       stmgcpage_init_tls();
       stmgcpage_release_global_lock();
@@ -1611,7 +1600,7 @@
                      d->num_spinloops[i]);
 
     p += sprintf(p, "]\n");
-    fprintf(stderr, "%s", line);
+    dprintf(("%s", line));
 
     stm_free(d, sizeof(struct tx_descriptor));
 }
diff --git a/c4/fprintcolor.c b/c4/fprintcolor.c
--- a/c4/fprintcolor.c
+++ b/c4/fprintcolor.c
@@ -1,10 +1,31 @@
 #include "stmimpl.h"
 
+
+void stm_fatalerror(const char *format, ...)
+{
+    va_list ap;
+
+#ifdef _GC_DEBUG
+    dprintf(("STM Subsystem: Fatal Error\n"));
+#else
+    fprintf(stderr, "STM Subsystem: Fatal Error\n");
+#endif
+
+    va_start(ap, format);
+    vfprintf(stderr, format, ap);
+    va_end(ap);
+
+    abort();
+}
+
+
+#ifdef _GC_DEBUG
+
 static __thread revision_t tcolor = 0;
 static revision_t tnextid = 0;
 
 
-int threadcolor_fprintf(FILE *stream, const char *format, ...)
+int threadcolor_printf(const char *format, ...)
 {
     char buffer[2048];
     va_list ap;
@@ -26,7 +47,9 @@
     va_end(ap);
 
     strcpy(buffer + size + result, "\033[0m");
-    fputs(buffer, stream);
+    fputs(buffer, stderr);
 
     return result;
 }
+
+#endif
diff --git a/c4/fprintcolor.h b/c4/fprintcolor.h
--- a/c4/fprintcolor.h
+++ b/c4/fprintcolor.h
@@ -1,8 +1,20 @@
 #include <stdarg.h>
+#include "stmimpl.h"
 
 
-#define fprintf threadcolor_fprintf
+void stm_fatalerror(const char *format, ...)
+     __attribute__((format (printf, 1, 2), noreturn));
 
 
-int threadcolor_fprintf(FILE *stream, const char *format, ...)
-     __attribute__((format (printf, 2, 3)));
+#ifdef _GC_DEBUG
+
+#define dprintf(args)   threadcolor_printf args
+
+int threadcolor_printf(const char *format, ...)
+     __attribute__((format (printf, 1, 2)));
+
+#else
+
+#define dprintf(args)   do { } while(0)
+
+#endif
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -101,8 +101,7 @@
     /* Allocate and return a new page for the given size_class. */
     page_header_t *page = (page_header_t *)stm_malloc(GC_PAGE_SIZE);
     if (!page) {
-        fprintf(stderr, "allocate_new_page: out of memory!\n");
-        abort();
+        stm_fatalerror("allocate_new_page: out of memory!\n");
     }
     struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
     gcp->count_pages++;
@@ -155,8 +154,7 @@
         return result;
     }
     else {
-        fprintf(stderr, "XXX stmgcpage_malloc: too big!\n");
-        abort();
+        stm_fatalerror("XXX stmgcpage_malloc: too big!\n");
     }
 }
 
@@ -178,8 +176,7 @@
         //stm_dbgmem_not_used(obj, size_class * WORD, 0);
     }
     else {
-        fprintf(stderr, "XXX stmgcpage_free: too big!\n");
-        abort();
+        stm_fatalerror("XXX stmgcpage_free: too big!\n");
     }
 }
 
@@ -196,7 +193,7 @@
 
  restart:
     if (obj->h_tid & GCFLAG_VISITED) {
-        fprintf(stderr, "[already visited: %p]\n", obj);
+        dprintf(("[already visited: %p]\n", obj));
         assert(obj == *pobj);
         assert((obj->h_revision & 3) ||   /* either odd, or stub */
                (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
@@ -301,7 +298,8 @@
     while (root != end) {
         gcptr o = *root;
         visit(root);
-        fprintf(stderr, "visit stack root: %p -> %p\n", o, *root);
+        dprintf(("visit stack root: %p -> %p\n", o, *root));
+        (void)o;   /* silence "warning: unused variable 'o'" */
         root++;
     }
 }
@@ -396,9 +394,8 @@
         revision_t v = obj->h_revision;
         if (IS_POINTER(v)) {
             /* has a more recent revision.  Oups. */
-            fprintf(stderr,
-                    "ABRT_COLLECT_MAJOR %p: %p was read but modified already\n",
-                    d, obj);
+            dprintf(("ABRT_COLLECT_MAJOR %p: "
+                     "%p was read but modified already\n", d, obj));
             AbortTransactionAfterCollect(d, ABRT_COLLECT_MAJOR);
             return;
         }
@@ -429,7 +426,7 @@
      */
     wlog_t *item;
 
-    fprintf(stderr, "fix public_to_private on thread %p\n", d);
+    dprintf(("fix public_to_private on thread %p\n", d));
 
     G2L_LOOP_FORWARD(d->public_to_private, item) {
 
@@ -440,7 +437,7 @@
                (item->val->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
 
         item->addr->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
-        fprintf(stderr, "\tpublic_to_private: %p\n", item->addr);
+        dprintf(("\tpublic_to_private: %p\n", item->addr));
 
     } G2L_LOOP_END;
 }
@@ -488,7 +485,7 @@
                 else {
 #ifdef DUMP_EXTRA
                     if (p->h_tid != DEBUG_WORD(0xDD)) {
-                        fprintf(stderr, "| freeing %p\n", p);
+                        dprintf(("| freeing %p\n", p));
                     }
 #endif
                     /* skip the assignment if compiled without asserts */
@@ -509,7 +506,7 @@
             for (j = 0; j < objs_per_page; j++) {
                 assert(!(p->h_tid & GCFLAG_VISITED));
                 if (p->h_tid != DEBUG_WORD(0xDD)) {
-                    fprintf(stderr, "| freeing %p (with page %p)\n", p, lpage);
+                    dprintf(("| freeing %p (with page %p)\n", p, lpage));
                 }
                 p = (gcptr)(((char *)p) + obj_size);
             }
@@ -643,7 +640,7 @@
 static void major_collect(void)
 {
     stmgcpage_acquire_global_lock();
-    fprintf(stderr, ",-----\n| running major collection...\n");
+    dprintf((",-----\n| running major collection...\n"));
 
     force_minor_collections();
 
@@ -662,10 +659,10 @@
     free_closed_thread_descriptors();
     update_next_threshold();
 
-    fprintf(stderr, "| %lu bytes alive, %lu not used, countdown %lu\n`-----\n",
-            (unsigned long)mc_total_in_use,
-            (unsigned long)(mc_total_reserved - mc_total_in_use),
-            (unsigned long)countdown_next_major_coll);
+    dprintf(("| %lu bytes alive, %lu not used, countdown %lu\n`-----\n",
+             (unsigned long)mc_total_in_use,
+             (unsigned long)(mc_total_reserved - mc_total_in_use),
+             (unsigned long)countdown_next_major_coll));
     stmgcpage_release_global_lock();
 }
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -18,8 +18,8 @@
     d->nursery_end = d->nursery_base + GC_NURSERY;
     d->nursery_current = d->nursery_base;
 
-    fprintf(stderr, "minor: nursery is at [%p to %p]\n", d->nursery_base,
-            d->nursery_end);
+    dprintf(("minor: nursery is at [%p to %p]\n", d->nursery_base,
+             d->nursery_end));
 }
 
 void stmgc_done_nursery(void)
@@ -155,8 +155,8 @@
             return mangle_hash((revision_t)p);
         }
 
-        fprintf(stderr, "stm_id(%p) has orig fst: %p\n", 
-                p, (gcptr)p->h_original);
+        dprintf(("stm_id(%p) has orig fst: %p\n", 
+                 p, (gcptr)p->h_original));
         return mangle_hash(p->h_original);
     } 
     else if (!(p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
@@ -170,7 +170,7 @@
            old at the same time, and we see the OLD flag 
            before h_original has been set.
         */
-        fprintf(stderr, "stm_id(%p) is old, orig=0 fst: %p\n", p, p);
+        dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p));
         return mangle_hash((revision_t)p);
     }
     
@@ -185,13 +185,13 @@
     */
     if (p->h_original) { /* maybe now? */
         result = p->h_original;
-        fprintf(stderr, "stm_id(%p) has orig: %p\n", 
-                p, (gcptr)p->h_original);
+        dprintf(("stm_id(%p) has orig: %p\n", 
+                 p, (gcptr)p->h_original));
     }
     else if (p->h_tid & GCFLAG_OLD) {
         /* it must be this exact object */
         result = (revision_t)p;
-        fprintf(stderr, "stm_id(%p) is old, orig=0: %p\n", p, p);
+        dprintf(("stm_id(%p) is old, orig=0: %p\n", p, p));
     }
     else {
         /* must create shadow original object or use
@@ -203,8 +203,8 @@
             // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
             
             result = (revision_t)B;
-            fprintf(stderr, "stm_id(%p) young, pfp, use backup %p\n", 
-                    p, (gcptr)p->h_original);
+            dprintf(("stm_id(%p) young, pfp, use backup %p\n", 
+                     p, (gcptr)p->h_original));
         }
         else {
             gcptr O = stmgc_duplicate_old(p);
@@ -213,7 +213,7 @@
             O->h_tid |= GCFLAG_PUBLIC;
             
             result = (revision_t)O;
-            fprintf(stderr, "stm_id(%p) young, make shadow %p\n", p, O); 
+            dprintf(("stm_id(%p) young, make shadow %p\n", p, O));
         }
     }
     
@@ -242,7 +242,7 @@
 
     gcptr fresh_old_copy = stmgc_duplicate_old(obj);
 
-    fprintf(stderr, "minor: %p is copied to %p\n", obj, fresh_old_copy);
+    dprintf(("minor: %p is copied to %p\n", obj, fresh_old_copy));
     return fresh_old_copy;
 }
 
@@ -255,7 +255,7 @@
     memcpy(id, obj, size);
     id->h_tid &= ~GCFLAG_HAS_ID;
     id->h_tid |= GCFLAG_OLD;
-    fprintf(stderr, "copy_to_old_id_copy(%p -> %p)\n", obj, id);
+    dprintf(("copy_to_old_id_copy(%p -> %p)\n", obj, id));
 }
 
 static void visit_if_young(gcptr *root)
@@ -340,7 +340,7 @@
            in the past, but someone made even more changes.
            Nothing to do now.
         */
-        fprintf(stderr, "trace_stub: %p not a stub, ignored\n", S);
+        dprintf(("trace_stub: %p not a stub, ignored\n", S));
         return;
     }
 
@@ -349,7 +349,7 @@
         /* Bah, it's indeed a stub but for another thread.  Nothing
            to do now.
         */
-        fprintf(stderr, "trace_stub: %p stub wrong thread, ignored\n", S);
+        dprintf(("trace_stub: %p stub wrong thread, ignored\n", S));
         return;
     }
 
@@ -357,7 +357,7 @@
        feet because we hold our own collection_lock.
     */
     gcptr L = (gcptr)(w - 2);
-    fprintf(stderr, "trace_stub: %p stub -> %p\n", S, L);
+    dprintf(("trace_stub: %p stub -> %p\n", S, L));
     visit_if_young(&L);
     assert(S->h_tid & GCFLAG_STUB);
     S->h_revision = ((revision_t)L) | 2;
@@ -404,14 +404,14 @@
                We are in a case where we know the transaction will not
                be able to commit successfully.
             */
-            fprintf(stderr, "public_to_young: %p was modified! abort!\n", P);
+            dprintf(("public_to_young: %p was modified! abort!\n", P));
             item->val = NULL;
             AbortTransactionAfterCollect(d, ABRT_COLLECT_MINOR);
             continue;
         }
 
-        fprintf(stderr, "public_to_young: %p -> %p in public_to_private\n",
-                item->addr, item->val);
+        dprintf(("public_to_young: %p -> %p in public_to_private\n",
+                 item->addr, item->val));
         assert(_stm_is_private(item->val));
         visit_if_young(&item->val);
         continue;
@@ -422,11 +422,11 @@
                It must come from an older transaction that aborted.
                Nothing to do now.
             */
-            fprintf(stderr, "public_to_young: %p ignored\n", P);
+            dprintf(("public_to_young: %p ignored\n", P));
             continue;
         }
 
-        fprintf(stderr, "public_to_young: %p -> ", P);
+        dprintf(("public_to_young: %p -> ", P));
         trace_stub(d, (gcptr)v);
     }
 
@@ -495,8 +495,8 @@
 
 static void minor_collect(struct tx_descriptor *d)
 {
-    fprintf(stderr, "minor collection [%p to %p]\n",
-            d->nursery_base, d->nursery_end);
+    dprintf(("minor collection [%p to %p]\n",
+             d->nursery_base, d->nursery_end));
 
     /* acquire the "collection lock" first */
     setup_minor_collect(d);
@@ -527,8 +527,8 @@
     d->nursery_base = stm_malloc(GC_NURSERY);
     memset(d->nursery_base, 0, GC_NURSERY);
     d->nursery_end = d->nursery_base + GC_NURSERY;
-    fprintf(stderr, "minor: nursery moved to [%p to %p]\n", d->nursery_base,
-            d->nursery_end);
+    dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
+             d->nursery_end));
 #else
     memset(d->nursery_base, 0, GC_NURSERY);
 #endif
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -63,7 +63,7 @@
 
  done:
     *pobj = stub;
-    fprintf(stderr, "  stolen: fixing *%p: %p -> %p\n", pobj, obj, stub);
+    dprintf(("  stolen: fixing *%p: %p -> %p\n", pobj, obj, stub));
 }
 
 void stm_steal_stub(gcptr P)
@@ -120,7 +120,7 @@
         if (B->h_tid & GCFLAG_PUBLIC_TO_PRIVATE) {
             /* already stolen */
             assert(B->h_tid & GCFLAG_PUBLIC);
-            fprintf(stderr, "already stolen: %p -> %p <-> %p\n", P, L, B);
+            dprintf(("already stolen: %p -> %p <-> %p\n", P, L, B));
             L = B;
             goto already_stolen;
         }
@@ -130,13 +130,13 @@
            don't want to walk over the feet of the foreign thread
         */
         gcptrlist_insert2(&foreign_pd->stolen_objects, B, L);
-        fprintf(stderr, "stolen: %p -> %p <-> %p\n", P, L, B);
+        dprintf(("stolen: %p -> %p <-> %p\n", P, L, B));
         L = B;
     }
     else {
         if (L->h_tid & GCFLAG_PUBLIC) {
             /* already stolen */
-            fprintf(stderr, "already stolen: %p -> %p\n", P, L);
+            dprintf(("already stolen: %p -> %p\n", P, L));
 
             /* note that we should follow h_revision at least one more
                step: it is necessary if L is public but young (and then
@@ -145,12 +145,12 @@
             v = ACCESS_ONCE(L->h_revision);
             if (IS_POINTER(v)) {
                 L = (gcptr)v;
-                fprintf(stderr, "\t---> %p\n", L);
+                dprintf(("\t---> %p\n", L));
             }
             goto already_stolen;
         }
 
-        fprintf(stderr, "stolen: %p -> %p\n", P, L);
+        dprintf(("stolen: %p -> %p\n", P, L));
 
         
         if (!(L->h_tid & GCFLAG_OLD)) { 
@@ -182,7 +182,7 @@
                L. */
             gcptrlist_insert2(&foreign_pd->stolen_objects, L, NULL);
             L = O;
-            fprintf(stderr, "\t---> %p\n", L);
+            dprintf(("\t---> %p\n", L));
         }
 
         assert(L->h_tid & GCFLAG_OLD);
@@ -261,7 +261,7 @@
 
         /* this is definitely needed: all keys in public_to_private
            must appear in list_of_read_objects */
-        fprintf(stderr, "n.readobj: %p -> %p\n", B, L);
+        dprintf(("n.readobj: %p -> %p\n", B, L));
         assert(!(B->h_tid & GCFLAG_STUB));
         gcptrlist_insert(&d->list_of_read_objects, B);
 
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -31,8 +31,7 @@
     struct tx_descriptor *d = thread_descriptor;
     d->shadowstack = malloc(sizeof(gcptr) * LENGTH_SHADOW_STACK);
     if (!d->shadowstack) {
-        fprintf(stderr, "out of memory: shadowstack\n");
-        abort();
+        stm_fatalerror("out of memory: shadowstack\n");
     }
     stm_shadowstack = d->shadowstack;
     d->shadowstack_end_ref = &stm_shadowstack;
@@ -78,7 +77,7 @@
 gcptr stm_read_barrier(gcptr obj)
 {
     //if (FXCACHE_AT(obj) == obj)
-    //    fprintf(stderr, "read_barrier: in cache: %p\n", obj);
+    //    dprintf(("read_barrier: in cache: %p\n", obj));
 
     /* XXX inline in the caller, optimize to get the smallest code */
     if (UNLIKELY((obj->h_revision != stm_private_rev_num) &&
@@ -230,12 +229,12 @@
     int err = pthread_rwlock_rdlock(&rwlock_shared);
     assert(err == 0);
     //assert(stmgc_nursery_hiding(thread_descriptor, 0));
-    fprintf(stderr, "stm_start_sharedlock\n");
+    dprintf(("stm_start_sharedlock\n"));
 }
 
 void stm_stop_sharedlock(void)
 {
-    fprintf(stderr, "stm_stop_sharedlock\n");
+    dprintf(("stm_stop_sharedlock\n"));
     //assert(stmgc_nursery_hiding(thread_descriptor, 1));
     int err = pthread_rwlock_unlock(&rwlock_shared);
     assert(err == 0);
@@ -245,12 +244,12 @@
 {
     int err = pthread_rwlock_wrlock(&rwlock_shared);
     assert(err == 0);
-    fprintf(stderr, "start_exclusivelock\n");
+    dprintf(("start_exclusivelock\n"));
 }
 
 static void stop_exclusivelock(void)
 {
-    fprintf(stderr, "stop_exclusivelock\n");
+    dprintf(("stop_exclusivelock\n"));
     int err = pthread_rwlock_unlock(&rwlock_shared);
     assert(err == 0);
 }
@@ -317,10 +316,10 @@
 
 void stm_clear_between_tests(void)
 {
-    fprintf(stderr, "\n"
+    dprintf(("\n"
             "===============================================================\n"
             "========================[  START  ]============================\n"
             "===============================================================\n"
-            "\n");
+            "\n"));
     gcptrlist_clear(&stm_prebuilt_gcroots);
 }

From noreply at buildbot.pypy.org  Mon Jun 24 17:05:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 17:05:56 +0200 (CEST)
Subject: [pypy-commit] stmgc default: merge heads
Message-ID: <20130624150556.46BA91C2FAE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r265:b2b7620a4429
Date: 2013-06-24 17:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/b2b7620a4429/

Log:	merge heads

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -29,6 +29,7 @@
     struct stm_object_s hdr;
     long value;
     revision_t id;
+    revision_t hash;
     struct node *next;
 };
 typedef struct node * nodeptr;
@@ -84,7 +85,14 @@
     gcptr x = calloc(1, size);
     x->h_tid = PREBUILT_FLAGS | tid;
     x->h_revision = PREBUILT_REVISION;
-    x->h_original = 0;
+    return x;
+}
+
+gcptr allocate_pseudoprebuilt_with_hash(size_t size, int tid,
+                                        revision_t hash)
+{
+    gcptr x = allocate_pseudoprebuilt(size, tid);
+    x->h_original = hash;
     return x;
 }
 
@@ -240,8 +248,17 @@
     
     td.num_roots = PREBUILT + NUMROOTS;
     for (i = 0; i < PREBUILT; i++) {
-        td.roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
-                                              GCTID_STRUCT_NODE);
+        if (i % 3 == 0) {
+            td.roots[i] = allocate_pseudoprebuilt_with_hash(
+                                          sizeof(struct node), 
+                                          GCTID_STRUCT_NODE,
+                                          i);
+            ((nodeptr)td.roots[i])->hash = i;
+        }
+        else {
+            td.roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
+                                                  GCTID_STRUCT_NODE);
+        }
     }
     for (i = PREBUILT; i < PREBUILT + NUMROOTS; i++) {
         td.roots[i] = (gcptr)allocate_node();
@@ -251,17 +268,19 @@
 
 gcptr do_step(gcptr p)
 {
-    nodeptr w_r, w_sr;
-    gcptr _r, _sr;
+    nodeptr w_r, w_sr, w_t;
+    gcptr _r, _sr, _t;
     int num, k;
 
+    _t = NULL;
     num = get_rand(td.num_roots);
     _r = td.roots[num];
-
+    
     num = get_rand(SHARED_ROOTS);
     _sr = shared_roots[num];
 
-    k = get_rand(17);
+    k = get_rand(19);
+    check(p);
 
     switch (k) {
     case 0: // remove a root
@@ -323,28 +342,42 @@
         pop_roots();
         p = NULL;
         break;
-    case 15: /* test stm_id on non-shared roots */
-        w_r = (nodeptr)read_barrier(_r);
-        if (w_r->id) {
-            assert(w_r->id == stm_id((gcptr)w_r));
-            assert(w_r->id == stm_id((gcptr)_r));
+    case 15: /* test stm_id on (non-)shared roots */
+        _t = _r;
+    case 16:
+        if (!_t)
+            _t = _sr;
+        w_t = (nodeptr)read_barrier(_t);
+        if (w_t->id) {
+            assert(w_t->id == stm_id((gcptr)w_t));
+            assert(w_t->id == stm_id((gcptr)_t));
         } 
         else {
-            w_r = (nodeptr)write_barrier(_r);
-            w_r->id = stm_id((gcptr)w_r);
-            assert(w_r->id == stm_id((gcptr)_r));
+            w_t = (nodeptr)write_barrier(_t);
+            w_t->id = stm_id((gcptr)w_t);
+            assert(w_t->id == stm_id((gcptr)_t));
         }
-    case 16: /* test stm_id on shared roots */
-        w_sr = (nodeptr)read_barrier(_sr);
-        if (w_sr->id) {
-            assert(w_sr->id == stm_id((gcptr)w_sr));
-            assert(w_sr->id == stm_id((gcptr)_sr));
-        } 
+        break;
+    case 17: /* test stm_hash on (non-)shared roots */
+        _t = _r;
+    case 18:
+        if (!_t)
+            _t = _sr;
+        w_t = (nodeptr)read_barrier(_t);
+        if (w_t->hash) {
+            assert(w_t->hash == stm_hash((gcptr)w_t));
+            assert(w_t->hash == stm_hash((gcptr)_t));
+        }
         else {
-            w_sr = (nodeptr)write_barrier(_sr);
-            w_sr->id = stm_id((gcptr)w_sr);
-            assert(w_sr->id == stm_id((gcptr)_sr));
+            w_t = (nodeptr)write_barrier(_t);
+            w_t->hash = stm_hash((gcptr)w_t);
+            assert(w_t->hash == stm_hash((gcptr)_t));
         }
+        if (w_t->hash < PREBUILT || w_t->hash < SHARED_ROOTS) {
+            // should be with predefined hash
+            assert (stm_id((gcptr)w_t) != stm_hash((gcptr)w_t));
+        }
+        break;
     }
     return p;
 }
@@ -449,8 +482,17 @@
     default_seed = time(NULL) / 3600 / 24;
     
     for (i = 0; i < SHARED_ROOTS; i++) {
-        shared_roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
-                                                  GCTID_STRUCT_NODE);
+        if (i % 3 == 0) {
+            shared_roots[i] = allocate_pseudoprebuilt_with_hash(
+                                          sizeof(struct node), 
+                                          GCTID_STRUCT_NODE,
+                                          i);
+            ((nodeptr)shared_roots[i])->hash = i;
+        }
+        else {
+            shared_roots[i] = allocate_pseudoprebuilt(sizeof(struct node), 
+                                                      GCTID_STRUCT_NODE);
+        }
     }    
     
     status = sem_init(&done, 0, 0);

From noreply at buildbot.pypy.org  Mon Jun 24 17:17:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 17:17:32 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Give different seeds to each thread.
Message-ID: <20130624151732.A1E351C034D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r266:4acf3f474301
Date: 2013-06-24 17:17 +0200
http://bitbucket.org/pypy/stmgc/changeset/4acf3f474301/

Log:	Give different seeds to each thread.

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -242,7 +242,7 @@
 void setup_thread()
 {
     int i;
-    td.thread_seed = default_seed;
+    td.thread_seed = default_seed++;
     td.steps_left = STEPS_PER_THREAD;
     td.interruptible = 0;
     
@@ -479,7 +479,8 @@
     
     // seed changes daily
     // a bit pointless for now..
-    default_seed = time(NULL) / 3600 / 24;
+    default_seed = time(NULL);
+    default_seed -= (default_seed % (3600 * 24));
     
     for (i = 0; i < SHARED_ROOTS; i++) {
         if (i % 3 == 0) {

From noreply at buildbot.pypy.org  Mon Jun 24 17:25:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 17:25:52 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Adapt mangle_hash() to its docstring
 and always return different results
Message-ID: <20130624152552.A0C491C2FAE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r267:a93d0b0977e9
Date: 2013-06-24 17:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/a93d0b0977e9/

Log:	Adapt mangle_hash() to its docstring and always return different
	results for different arguments. (So it may now give negative
	results too.)

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -373,7 +373,8 @@
             w_t->hash = stm_hash((gcptr)w_t);
             assert(w_t->hash == stm_hash((gcptr)_t));
         }
-        if (w_t->hash < PREBUILT || w_t->hash < SHARED_ROOTS) {
+        if (w_t->hash >= 0 && (w_t->hash < PREBUILT ||
+                               w_t->hash < SHARED_ROOTS)) {
             // should be with predefined hash
             assert (stm_id((gcptr)w_t) != stm_hash((gcptr)w_t));
         }
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -118,7 +118,7 @@
        This formula is reversible: two different values of 'i' will
        always give two different results.
     */
-    return n ^ (n >> 4);
+    return n ^ (((urevision_t)n) >> 4);
 }
 
 
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -6,6 +6,7 @@
 
 
 typedef intptr_t revision_t;
+typedef uintptr_t urevision_t;
 
 typedef struct stm_object_s {
     revision_t h_tid;

From noreply at buildbot.pypy.org  Mon Jun 24 17:34:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 17:34:40 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tweak mangle_hash() to be only called
 from stm_hash(), not stm_id()
Message-ID: <20130624153440.736631C2FCC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r268:1d78e1a31bc9
Date: 2013-06-24 17:33 +0200
http://bitbucket.org/pypy/stmgc/changeset/1d78e1a31bc9/

Log:	Tweak mangle_hash() to be only called from stm_hash(), not stm_id()

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -138,7 +138,7 @@
             return orig->h_original;
         }
     }
-    return stm_id(p);
+    return mangle_hash(stm_id(p));
 }
 
 
@@ -152,12 +152,12 @@
             /* h_original may contain a specific hash value,
                but in case of the prebuilt original version, 
                its memory location is the id */
-            return mangle_hash((revision_t)p);
+            return (revision_t)p;
         }
 
         dprintf(("stm_id(%p) has orig fst: %p\n", 
                  p, (gcptr)p->h_original));
-        return mangle_hash(p->h_original);
+        return p->h_original;
     } 
     else if (!(p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
                && (p->h_tid & GCFLAG_OLD)) {
@@ -171,7 +171,7 @@
            before h_original has been set.
         */
         dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p));
-        return mangle_hash((revision_t)p);
+        return (revision_t)p;
     }
     
 
@@ -218,11 +218,14 @@
     }
     
     spinlock_release(d->public_descriptor->collection_lock);
-    return mangle_hash(result);
+    return result;
 }
 
 revision_t stm_pointer_equal(gcptr p1, gcptr p2)
 {
+    /* fast path for two equal pointers */
+    if (p1 == p2)
+        return 1;
     /* types must be the same */
     if ((p1->h_tid & STM_USER_TID_MASK) != (p2->h_tid & STM_USER_TID_MASK))
         return 0;

From noreply at buildbot.pypy.org  Mon Jun 24 17:53:46 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 17:53:46 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add an XXX
Message-ID: <20130624155346.4C19A1C2FAC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r269:f1bb618388e3
Date: 2013-06-24 17:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/f1bb618388e3/

Log:	Add an XXX

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -207,6 +207,8 @@
                      p, (gcptr)p->h_original));
         }
         else {
+            /* XXX use stmgcpage_malloc() directly, we don't need to copy
+             * the contents yet */
             gcptr O = stmgc_duplicate_old(p);
             p->h_original = (revision_t)O;
             p->h_tid |= GCFLAG_HAS_ID;

From noreply at buildbot.pypy.org  Mon Jun 24 18:04:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Jun 2013 18:04:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Shooting in the dark,
 but one possible reading of the man page shows
Message-ID: <20130624160430.2C6AD1C2FCE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r64964:83f89665175e
Date: 2013-06-24 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/83f89665175e/

Log:	Shooting in the dark, but one possible reading of the man page shows
	that when readdir() returns a non-null pointer, the value of errno
	might be randomly modified.

diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -1143,9 +1143,9 @@
                 dirp = os_opendir(path)
                 if not dirp:
                     raise OSError(rposix.get_errno(), "os_opendir failed")
-                rposix.set_errno(0)
                 result = []
                 while True:
+                    rposix.set_errno(0)
                     direntp = os_readdir(dirp)
                     if not direntp:
                         error = rposix.get_errno()

From noreply at buildbot.pypy.org  Mon Jun 24 22:23:05 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Jun 2013 22:23:05 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: write a failing test
	(thanks armin)
Message-ID: <20130624202305.1FC131C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r64965:614b7559c1e7
Date: 2013-06-24 22:22 +0200
http://bitbucket.org/pypy/pypy/changeset/614b7559c1e7/

Log:	write a failing test (thanks armin)

diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -1474,7 +1474,41 @@
             "int_add": 2, "jump": 1
         })
 
+    def test_frame_nonstandard_no_virtualizable(self):
 
+        driver1 = JitDriver(greens=[], reds=['i', 's', 'frame'])
+        driver2 = JitDriver(greens=[], reds=['frame'],
+                            virtualizables=['frame'])
+
+        class Frame(object):
+            _virtualizable2_ = ['x']
+
+        def g(frame):
+            driver2.jit_merge_point(frame=frame)
+            frame.x += 1
+            return frame
+
+        def f():
+            i = 0
+            s = 0
+            frame = Frame()
+            frame.x = 0
+            g(frame)
+            while i < 10:
+                driver1.jit_merge_point(frame=frame, s=s, i=i)
+                frame = g(frame)
+                s += frame.x
+                i += 1
+            return s
+
+        def main():
+            res = 0
+            for i in range(10):
+                res += f()
+            return res
+
+        res = self.meta_interp(main, [])
+        assert res == main()
 
 class TestLLtype(ExplicitVirtualizableTests,
                  ImplicitVirtualizableTests,

From noreply at buildbot.pypy.org  Mon Jun 24 22:33:56 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 24 Jun 2013 22:33:56 +0200 (CEST)
Subject: [pypy-commit] pypy mtrand: implement numpy.random.mtrand in pure
	python
Message-ID: <20130624203356.E96E11C034D@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: mtrand
Changeset: r64966:ed24d59708bb
Date: 2013-06-24 22:25 +0300
http://bitbucket.org/pypy/pypy/changeset/ed24d59708bb/

Log:	implement numpy.random.mtrand in pure python


From noreply at buildbot.pypy.org  Mon Jun 24 22:33:58 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 24 Jun 2013 22:33:58 +0200 (CEST)
Subject: [pypy-commit] pypy mtrand: test,
	implement functions as calls to random.xxx
Message-ID: <20130624203358.448221C034D@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: mtrand
Changeset: r64967:cef553310960
Date: 2013-06-24 23:32 +0300
http://bitbucket.org/pypy/pypy/changeset/cef553310960/

Log:	test, implement functions as calls to random.xxx

diff --git a/lib_pypy/numpypy/random/__init__.py b/lib_pypy/numpypy/random/__init__.py
new file mode 100644
diff --git a/lib_pypy/numpypy/random/mtrand.py b/lib_pypy/numpypy/random/mtrand.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/random/mtrand.py
@@ -0,0 +1,17 @@
+import random
+from numpypy import zeros
+def random_sample(length=0):
+    if length == 0:
+        return random.random()
+    ret = zeros((length,))
+    for x in xrange(length):
+        ret[x] = random.random()
+    return ret
+
+def randn(length=0):
+    if length == 0:
+        return random.gauss(0., 1.)
+    ret = zeros((length,))
+    for x in xrange(length):
+        ret[x] = random.gauss(0., 1.)
+    return ret
diff --git a/pypy/module/test_lib_pypy/numpypy/random/test_random.py b/pypy/module/test_lib_pypy/numpypy/random/test_random.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/numpypy/random/test_random.py
@@ -0,0 +1,21 @@
+import py
+try:
+    import numpypy
+except:
+    pass
+
+def test_mtrand():
+    from numpy.random import mtrand
+    a = mtrand.random_sample()
+    assert isinstance(a, float)
+    a = mtrand.random_sample(10)
+    assert a.shape == (10,)
+
+def test_randn():
+    from numpy.random.mtrand import randn
+    from numpy import array
+    a = array([randn() for i in xrange(1000)])
+    b = randn(1000)
+    assert -0.1 < a.mean() < 0.1
+    assert -0.1 < b.mean() < 0.1
+    assert b.shape == (1000,)

From noreply at buildbot.pypy.org  Mon Jun 24 23:56:06 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 24 Jun 2013 23:56:06 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: add ndarray.round ufunc
Message-ID: <20130624215606.A90951C034D@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64968:596c8db865fb
Date: 2013-06-24 23:41 +0300
http://bitbucket.org/pypy/pypy/changeset/596c8db865fb/

Log:	add ndarray.round ufunc


From noreply at buildbot.pypy.org  Mon Jun 24 23:56:07 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 24 Jun 2013 23:56:07 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: add passing numpy -A test
Message-ID: <20130624215607.EF17A1C034D@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64969:2968002e23f1
Date: 2013-06-24 23:44 +0300
http://bitbucket.org/pypy/pypy/changeset/2968002e23f1/

Log:	add passing numpy -A test

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -286,7 +286,7 @@
 
         skip('sign of nan is non-determinant')
         assert (signbit([float('nan'), float('-nan'), -float('nan')]) ==
-            [False, True, True]).all()    
+            [False, True, True]).all()
 
     def test_reciprocal(self):
         from numpypy import array, reciprocal, complex64, complex128
@@ -334,6 +334,13 @@
         assert all([math.copysign(1, f(abs(float("nan")))) == 1 for f in floor, ceil, trunc])
         assert all([math.copysign(1, f(-abs(float("nan")))) == -1 for f in floor, ceil, trunc])
 
+    def test_around(self):
+        from numpypy import array
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -1.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == a.round()).all()
+
+
     def test_copysign(self):
         from numpypy import array, copysign
 
@@ -364,7 +371,7 @@
             assert b[i] == res
 
     def test_exp2(self):
-        import math 
+        import math
         from numpypy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
@@ -759,8 +766,8 @@
              complex(inf, inf), complex(inf, ninf), complex(0, inf),
              complex(ninf, ninf), complex(nan, 0), complex(0, nan),
              complex(nan, nan)]
-        assert (isfinite(a) == [True, True, False, False, False, 
-                        False, False, False, False, False]).all() 
+        assert (isfinite(a) == [True, True, False, False, False,
+                        False, False, False, False, False]).all()
 
     def test_logical_ops(self):
         from numpypy import logical_and, logical_or, logical_xor, logical_not
@@ -864,7 +871,7 @@
         #numpy returns (a.real*b.real + a.imag*b.imag) / abs(b)**2
         expect = [3., -23., 1.]
         for i in range(len(a)):
-            assert b[i] == expect[i] 
+            assert b[i] == expect[i]
         b = floor_divide(a[0], 0.)
         assert math.isnan(b.real)
         assert b.imag == 0.

From noreply at buildbot.pypy.org  Mon Jun 24 23:56:09 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 24 Jun 2013 23:56:09 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: start to implement round(),
 need to add it to types.py
Message-ID: <20130624215609.3DE811C034D@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64970:03aaf579cabc
Date: 2013-06-25 00:54 +0300
http://bitbucket.org/pypy/pypy/changeset/03aaf579cabc/

Log:	start to implement round(), need to add it to types.py

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -558,9 +558,17 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "resize not implemented yet"))
 
-    def descr_round(self, space, w_decimals=0, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "round not implemented yet"))
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0, w_out=None):
+        if space.is_none(w_out):
+            w_out = None
+        elif not isinstance(w_out, W_NDimArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                "return arrays must be of ArrayType"))
+        out = interp_dtype.dtype_agreement(space, [self], self.get_shape(),
+                                           w_out)
+        loop.round(space, self, self.get_shape(), decimals, out)
+        return out
 
     def descr_searchsorted(self, space, w_v, w_side='left'):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -975,6 +983,7 @@
     byteswap = interp2app(W_NDimArray.descr_byteswap),
     choose   = interp2app(W_NDimArray.descr_choose),
     clip     = interp2app(W_NDimArray.descr_clip),
+    round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
 
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -12,6 +12,7 @@
 from pypy.module.micronumpy.iter import PureShapeIterator
 from pypy.module.micronumpy import constants
 from pypy.module.micronumpy.support import int_w
+from rpython.rlib.rfloat import round_double
 
 call2_driver = jit.JitDriver(name='numpy_call2',
                              greens = ['shapelen', 'func', 'calc_dtype',
@@ -173,7 +174,7 @@
         iter = x_iter
     shapelen = len(shape)
     while not iter.done():
-        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, 
+        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
                                         arr_dtype=arr_dtype)
         w_cond = arr_iter.getitem()
         if arr_dtype.itemtype.bool(w_cond):
@@ -188,7 +189,7 @@
     return out
 
 axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce',
-                                    greens=['shapelen', 
+                                    greens=['shapelen',
                                             'func', 'dtype',
                                             'identity'],
                                     reds='auto')
@@ -228,7 +229,7 @@
     arg_driver = jit.JitDriver(name='numpy_' + op_name,
                                greens = ['shapelen', 'dtype'],
                                reds = 'auto')
-    
+
     def argmin_argmax(arr):
         result = 0
         idx = 1
@@ -265,7 +266,7 @@
      result.shape == [3, 5, 2, 4]
      broadcast shape should be [3, 5, 2, 7, 4]
      result should skip dims 3 which is len(result_shape) - 1
-        (note that if right is 1d, result should 
+        (note that if right is 1d, result should
                   skip len(result_shape))
      left should skip 2, 4 which is a.ndims-1 + range(right.ndims)
           except where it==(right.ndims-2)
@@ -283,9 +284,9 @@
     righti = right.create_dot_iter(broadcast_shape, right_skip)
     while not outi.done():
         dot_driver.jit_merge_point(dtype=dtype)
-        lval = lefti.getitem().convert_to(dtype) 
-        rval = righti.getitem().convert_to(dtype) 
-        outval = outi.getitem().convert_to(dtype) 
+        lval = lefti.getitem().convert_to(dtype)
+        rval = righti.getitem().convert_to(dtype)
+        outval = outi.getitem().convert_to(dtype)
         v = dtype.itemtype.mul(lval, rval)
         value = dtype.itemtype.add(v, outval).convert_to(dtype)
         outi.setitem(value)
@@ -355,7 +356,7 @@
         setitem_filter_driver.jit_merge_point(shapelen=shapelen,
                                               index_dtype=index_dtype,
                                               arr_dtype=arr_dtype,
-                                             ) 
+                                             )
         if index_iter.getitem_bool():
             arr_iter.setitem(value_iter.getitem())
             value_iter.next()
@@ -572,6 +573,23 @@
         out_iter.next()
         min_iter.next()
 
+round_driver = jit.JitDriver(greens = ['shapelen', 'dtype'],
+                                    reds = 'auto')
+
+def round(space, arr, shape, decimals, out):
+    arr_iter = arr.create_iter(shape)
+    dtype = out.get_dtype()
+    shapelen = len(shape)
+    out_iter = out.create_iter(shape)
+    while not arr_iter.done():
+        round_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+        #w_v = dtype.itemtype.round(arr_iter.getitem().convert_to(dtype),
+        #             decimals)
+        w_v = arr_iter.getitem().convert_to(dtype)
+        out_iter.setitem(w_v)
+        arr_iter.next()
+        out_iter.next()
+
 diagonal_simple_driver = jit.JitDriver(greens = ['axis1', 'axis2'],
                                        reds = 'auto')
 
@@ -613,4 +631,4 @@
         out_iter.setitem(arr.getitem_index(space, indexes))
         iter.next()
         out_iter.next()
-       
+
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -334,7 +334,7 @@
         assert all([math.copysign(1, f(abs(float("nan")))) == 1 for f in floor, ceil, trunc])
         assert all([math.copysign(1, f(-abs(float("nan")))) == -1 for f in floor, ceil, trunc])
 
-    def test_around(self):
+    def test_round(self):
         from numpypy import array
         ninf, inf = float("-inf"), float("inf")
         a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])

From noreply at buildbot.pypy.org  Tue Jun 25 07:13:22 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Jun 2013 07:13:22 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: emit FORCE_VIRTUALIZABLE
	in more cases
Message-ID: <20130625051322.48CFE1C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r64971:44af7c4839a8
Date: 2013-06-25 07:12 +0200
http://bitbucket.org/pypy/pypy/changeset/44af7c4839a8/

Log:	emit FORCE_VIRTUALIZABLE in more cases

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -708,30 +708,27 @@
     def _nonstandard_virtualizable(self, pc, box, fielddescr):
         # returns True if 'box' is actually not the "standard" virtualizable
         # that is stored in metainterp.virtualizable_boxes[-1]
-        if (self.metainterp.jitdriver_sd.virtualizable_info is None and
-            self.metainterp.jitdriver_sd.greenfield_info is None):
-            return True      # can occur in case of multiple JITs
-        standard_box = self.metainterp.virtualizable_boxes[-1]
-        if standard_box is box:
-            return False
         if self.metainterp.heapcache.is_nonstandard_virtualizable(box):
             return True
-        vinfo = self.metainterp.jitdriver_sd.virtualizable_info
-        if vinfo is not fielddescr.get_vinfo():
-            self.metainterp.heapcache.nonstandard_virtualizables_now_known(box)
-            return True
-        eqbox = self.metainterp.execute_and_record(rop.PTR_EQ, None,
-                                                   box, standard_box)
-        eqbox = self.implement_guard_value(eqbox, pc)
-        isstandard = eqbox.getint()
-        if isstandard:
-            self.metainterp.replace_box(box, standard_box)
-        else:
-            if not self.metainterp.heapcache.is_unescaped(box):
-                self.metainterp.execute_and_record(rop.FORCE_VIRTUALIZABLE,
-                                                   fielddescr, box)
-            self.metainterp.heapcache.nonstandard_virtualizables_now_known(box)
-        return not isstandard
+        if (self.metainterp.jitdriver_sd.virtualizable_info is not None or
+            self.metainterp.jitdriver_sd.greenfield_info is not None):
+            standard_box = self.metainterp.virtualizable_boxes[-1]
+            if standard_box is box:
+                return False
+            vinfo = self.metainterp.jitdriver_sd.virtualizable_info
+            if vinfo is fielddescr.get_vinfo():
+                eqbox = self.metainterp.execute_and_record(rop.PTR_EQ, None,
+                                                           box, standard_box)
+                eqbox = self.implement_guard_value(eqbox, pc)
+                isstandard = eqbox.getint()
+                if isstandard:
+                    self.metainterp.replace_box(box, standard_box)
+                    return False
+        if not self.metainterp.heapcache.is_unescaped(box):
+            self.metainterp.execute_and_record(rop.FORCE_VIRTUALIZABLE,
+                                               fielddescr, box)
+        self.metainterp.heapcache.nonstandard_virtualizables_now_known(box)
+        return True
 
     def _get_virtualizable_field_index(self, fielddescr):
         # Get the index of a fielddescr.  Must only be called for

From noreply at buildbot.pypy.org  Tue Jun 25 07:24:11 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Jun 2013 07:24:11 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: Write one more test for
 another case in _nonstandard_virtualizable if chain
Message-ID: <20130625052411.597831C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r64972:102265dc8e2a
Date: 2013-06-25 07:23 +0200
http://bitbucket.org/pypy/pypy/changeset/102265dc8e2a/

Log:	Write one more test for another case in _nonstandard_virtualizable
	if chain

diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -1510,6 +1510,40 @@
         res = self.meta_interp(main, [])
         assert res == main()
 
+    def test_two_virtualizables_mixed(self):
+        driver1 = JitDriver(greens=[], reds=['i', 's', 'frame',
+                                             'subframe'])
+        driver2 = JitDriver(greens=[], reds=['subframe'],
+                            virtualizables=['subframe'])
+
+        class Frame(object):
+            _virtualizable2_ = ['x']
+
+        class SubFrame(object):
+            _virtualizable2_ = ['x']
+
+        def g(subframe):
+            driver2.jit_merge_point(subframe=subframe)
+            subframe.x += 1
+
+        def f():
+            i = 0
+            frame = Frame()
+            frame.x = 0
+            subframe = SubFrame()
+            subframe.x = 0
+            s = 0
+            while i < 10:
+                driver1.jit_merge_point(frame=frame, subframe=subframe, i=i,
+                                        s=s)
+                g(subframe)
+                s += subframe.x
+                i += 1
+            return s
+
+        res = self.meta_interp(f, [])
+        assert res == f()
+
 class TestLLtype(ExplicitVirtualizableTests,
                  ImplicitVirtualizableTests,
                  LLJitMixin):

From noreply at buildbot.pypy.org  Tue Jun 25 07:43:13 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Jun 2013 07:43:13 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: Write some virtualizable
	docs
Message-ID: <20130625054313.1BA6A1C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r64973:6a62114110de
Date: 2013-06-25 07:42 +0200
http://bitbucket.org/pypy/pypy/changeset/6a62114110de/

Log:	Write some virtualizable docs

diff --git a/pypy/doc/jit/index.rst b/pypy/doc/jit/index.rst
--- a/pypy/doc/jit/index.rst
+++ b/pypy/doc/jit/index.rst
@@ -23,7 +23,10 @@
 
 - Hooks_ debugging facilities available to a python programmer
 
+- Virtualizable_ how virtualizables work and what they are (in other words how
+  to make frames more efficient).
 
 .. _Overview: overview.html
 .. _Notes: pyjitpl5.html
 .. _Hooks: ../jit-hooks.html
+.. _Virtualizable: virtualizable.html
diff --git a/pypy/doc/jit/virtualizable.rst b/pypy/doc/jit/virtualizable.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/jit/virtualizable.rst
@@ -0,0 +1,58 @@
+
+Virtualizables
+==============
+
+**Note:** this document does not have a proper introduction as to how
+to understand the basics. We should write some. If you happen to be here
+and you're missing context, feel free to pester us on IRC.
+
+Problem description
+-------------------
+
+The JIT is very good at making sure some objects are never allocated if they
+don't escape from the trace. Such objects are called ``virtuals``. However,
+if we're dealing with frames, virtuals are often not good enough. Frames
+can escape and they can also be allocated already at the moment we enter the
+JIT. In such cases we need some extra object that can still be optimized away,
+despite existing on heap.
+
+Solution
+--------
+
+We introduce virtualizables. They're objects that exist on the heap, but their
+fields are not always in sync with whatever happens in the assembler. One
+example is that virtualizable fields can store virtual objects without
+forcing them. It's very useful for frames. The definition looks like this::
+
+    class Frame(object):
+       _virtualizable2_ = ['locals[*]', 'stackdepth']
+
+And we use them in ``JitDriver`` like this::
+
+    jitdriver = JitDriver(greens=[], reds=['frame'], virtualizables=['frame'])
+
+This declaration means that ``stackdepth`` is a virtualizable **field**, while
+``locals`` is a virtualizable **array** (a list stored on a virtualizable).
+There are various rules about using virtualizables, especially using
+virtualizable arrays that can be very confusing. Those will usually end
+up with a compile-time error (as opposed to strange behavior). The rules are:
+
+* Each array access must be with a known positive index that cannot raise
+  an ``IndexError``. Using ``no = jit.hint(no, promote=True)`` might be useful
+  to get a constant-number access.
+
+* If you allocate a new virtualizable in the JIT, it has to be done like this
+  (for example if we're in ``Frame.__init__``)::
+
+    self = hint(self, access_directly=True, fresh_virtualizable=True)
+
+  that way you can populate the fields directly.
+
+* If you use virtualizable outside of the JIT - it's very expensive and
+  sometimes aborts tracing. Consider it carefully as to how do it only for
+  debugging purposes and not every time (e.g. ``sys._getframe`` call).
+
+* If you have something equivalent of a Python generator, where the
+  virtualizable survives for longer, you want to force it before returning.
+  It's better to do it that way than by an external call some time later.
+  It's done using ``jit.hint(frame, force_virtualizable=True)``

From noreply at buildbot.pypy.org  Tue Jun 25 08:09:09 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Jun 2013 08:09:09 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: Add a hint (unsupported
 so far) to force the virtualizable early
Message-ID: <20130625060909.E1D6D1C034D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r64974:7cc0a5947ad1
Date: 2013-06-25 08:08 +0200
http://bitbucket.org/pypy/pypy/changeset/7cc0a5947ad1/

Log:	Add a hint (unsupported so far) to force the virtualizable early

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -66,6 +66,9 @@
                             Useful in say Frame.__init__ when we do want
                             to store things directly on it. Has to come with
                             access_directly=True
+    * force_virtualizable - a performance hint to force the virtualizable early
+                            (useful e.g. for python generators that are going
+                            to be read later anyway)
     """
     return x
 

From noreply at buildbot.pypy.org  Tue Jun 25 11:23:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 11:23:59 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Give a 3rd compilation mode, "release",
 which disables even the asserts.
Message-ID: <20130625092359.C22981C034D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r270:4b9a102163cd
Date: 2013-06-25 11:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/4b9a102163cd/

Log:	Give a 3rd compilation mode, "release", which disables even the
	asserts. (The gain in speed is inconsistent.)

diff --git a/c4/Makefile b/c4/Makefile
--- a/c4/Makefile
+++ b/c4/Makefile
@@ -2,14 +2,16 @@
 # Makefile for the demos.
 #
 
+DEBUG_EXE = debug-demo1 debug-demo2 debug-demo_random
 BUILD_EXE = build-demo1 build-demo2 build-demo_random
-DEBUG_EXE = debug-demo1 debug-demo2 debug-demo_random
+RELEASE_EXE = release-demo1 release-demo2 release-demo_random
 
-build: $(BUILD_EXE)
-debug: $(DEBUG_EXE)
+debug: $(DEBUG_EXE)       # with prints and asserts
+build: $(BUILD_EXE)       # without prints, but with asserts
+release: $(RELEASE_EXE)   # without prints nor asserts
 
 clean:
-	rm -f $(BUILD_EXE) $(DEBUG_EXE)
+	rm -f $(BUILD_EXE) $(DEBUG_EXE) $(RELEASE_EXE)
 
 
 H_FILES = atomic_ops.h stmgc.h stmimpl.h \
@@ -29,5 +31,8 @@
 debug-%: %.c ${H_FILES} ${C_FILES}
 	gcc -lrt -pthread ${DEBUG} $< -o debug-$* -Wall ${C_FILES}
 
+release-%: %.c ${H_FILES} ${C_FILES} stmgc.c
+	gcc -lrt -pthread -DNDEBUG -O2 -g $< -o release-$* -Wall stmgc.c
+
 test-%:
 	./$* 2>/dev/null | grep "check ok"
diff --git a/c4/demo1.c b/c4/demo1.c
--- a/c4/demo1.c
+++ b/c4/demo1.c
@@ -11,7 +11,11 @@
 #include "fprintcolor.h"
 
 
-#define UPPER_BOUND 100
+#ifdef _GC_DEBUG
+# define UPPER_BOUND 100
+#else
+# define UPPER_BOUND 5000
+#endif
 #define NUMTHREADS  4
 
 
@@ -134,7 +138,8 @@
     stm_finalize();
 
     status = sem_post(&done);
-    assert(status == 0);
+    if (status != 0)
+        stm_fatalerror("status != 0\n");
     return NULL;
 }
 
@@ -158,7 +163,8 @@
 {
   pthread_t th;
   int status = pthread_create(&th, NULL, func, arg);
-  assert(status == 0);
+  if (status != 0)
+      stm_fatalerror("status != 0\n");
   pthread_detach(th);
   printf("started new thread\n");
 }
@@ -168,7 +174,8 @@
   int i, status;
 
   status = sem_init(&done, 0, 0);
-  assert(status == 0);
+  if (status != 0)
+      stm_fatalerror("status != 0\n");
 
   for (i = 0; i < NUMTHREADS; i++)
     newthread(demo1, NULL);
@@ -176,7 +183,8 @@
   for (i=0; i < NUMTHREADS; i++)
     {
       status = sem_wait(&done);
-      assert(status == 0);
+      if (status != 0)
+          stm_fatalerror("status != 0\n");
       printf("thread finished\n");
     }
 
diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -154,7 +154,8 @@
 void final_check(void)
 {
     long sum;
-    
+
+    printf("final check\n");
     stm_initialize();
     
     sum = check_sorted();
@@ -171,7 +172,8 @@
 {
     pthread_t th;
     int status = pthread_create(&th, NULL, func, arg);
-    assert(status == 0);
+    if (status != 0)
+        stm_fatalerror("newthread: pthread_create failure\n");
     pthread_detach(th);
     printf("started new thread\n");
 }
diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -467,7 +467,8 @@
 {
     pthread_t th;
     int status = pthread_create(&th, NULL, func, arg);
-    assert(status == 0);
+    if (status != 0)
+        stm_fatalerror("newthread: pthread_create failure\n");
     pthread_detach(th);
     printf("started new thread\n");
 }
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -305,6 +305,7 @@
 
 static void _check_flags(gcptr P)
 {
+#ifndef NDEBUG
   struct tx_descriptor *d = thread_descriptor;
   if (P->h_tid & GCFLAG_STUB)
     {
@@ -322,6 +323,7 @@
       assert(is_old);
       dprintf(("O "));
     }
+#endif
 }
 
 gcptr _stm_nonrecord_barrier(gcptr P)
@@ -962,7 +964,6 @@
 
 static void CancelLocks(struct tx_descriptor *d)
 {
-  revision_t my_lock = d->my_lock;
   wlog_t *item;
 
   if (!g2l_any_entry(&d->public_to_private))
@@ -984,7 +985,7 @@
 
       if (v == expected)
         {
-          assert(R->h_revision != my_lock);
+          assert(R->h_revision != d->my_lock);
           break;    /* done */
         }
 
@@ -993,7 +994,7 @@
 #ifdef DUMP_EXTRA
       dprintf(("%p->h_revision = %p (CancelLocks)\n", R, (gcptr)v));
 #endif
-      assert(R->h_revision == my_lock);
+      assert(R->h_revision == d->my_lock);
       ACCESS_ONCE(R->h_revision) = v;
 
     } G2L_LOOP_END;
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -39,13 +39,17 @@
 void stmgcpage_acquire_global_lock(void)
 {
     int err = pthread_mutex_lock(&mutex_gc_lock);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stmgcpage_acquire_global_lock: "
+                       "pthread_mutex_lock() failure\n");
 }
 
 void stmgcpage_release_global_lock(void)
 {
     int err = pthread_mutex_unlock(&mutex_gc_lock);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stmgcpage_release_global_lock: "
+                       "pthread_mutex_unlock() failure\n");
 }
 
 
@@ -158,7 +162,9 @@
     }
 }
 
+#ifndef NDEBUG
 static unsigned char random_char = 0x55;
+#endif
 
 void stmgcpage_free(gcptr obj)
 {
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -57,7 +57,8 @@
 void stm_initialize(void)
 {
     int r = DescriptorInit();
-    assert(r == 1);
+    if (r != 1)
+        stm_fatalerror("stm_initialize: DescriptorInit failure\n");
     stmgc_init_nursery();
     init_shadowstack();
     //stmgcpage_init_tls();
@@ -227,7 +228,9 @@
 void stm_start_sharedlock(void)
 {
     int err = pthread_rwlock_rdlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stm_start_sharedlock: "
+                       "pthread_rwlock_rdlock failure\n");
     //assert(stmgc_nursery_hiding(thread_descriptor, 0));
     dprintf(("stm_start_sharedlock\n"));
 }
@@ -237,13 +240,17 @@
     dprintf(("stm_stop_sharedlock\n"));
     //assert(stmgc_nursery_hiding(thread_descriptor, 1));
     int err = pthread_rwlock_unlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stm_stop_sharedlock: "
+                       "pthread_rwlock_unlock failure\n");
 }
 
 static void start_exclusivelock(void)
 {
     int err = pthread_rwlock_wrlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("start_exclusivelock: "
+                       "pthread_rwlock_wrlock failure\n");
     dprintf(("start_exclusivelock\n"));
 }
 
@@ -251,7 +258,9 @@
 {
     dprintf(("stop_exclusivelock\n"));
     int err = pthread_rwlock_unlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stop_exclusivelock: "
+                       "pthread_rwlock_unlock failure\n");
 }
 
 void stm_start_single_thread(void)
@@ -289,9 +298,8 @@
 
     /* Warning, may block waiting for rwlock_in_transaction while another
        thread runs a major GC */
-    struct tx_descriptor *d = thread_descriptor;
-    assert(d->active);
-    assert(in_single_thread != d);
+    assert(thread_descriptor->active);
+    assert(in_single_thread != thread_descriptor);
 
     stm_stop_sharedlock();
     /* another thread should be waiting in start_exclusivelock(),

From noreply at buildbot.pypy.org  Tue Jun 25 11:25:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 11:25:45 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Use stm_pointer_equal().
Message-ID: <20130625092545.E38FB1C034D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r271:b5764c2a5742
Date: 2013-06-25 11:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/b5764c2a5742/

Log:	Use stm_pointer_equal().

diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -97,23 +97,9 @@
             r_current = (struct node*)stm_read_barrier((gcptr)r_next);
             r_next = (struct node*)stm_read_barrier((gcptr)tmp);
         }
-        // results from consecutive read_barriers can differ. needs Ptr_Eq()
-        int i = 0;
-        while (!(stm_read_barrier((gcptr)r_prev->next) ==
-                 stm_read_barrier((gcptr)r_current) &&
-                 stm_read_barrier((gcptr)r_current->next) ==
-                 stm_read_barrier((gcptr)r_next))) {
-            asm volatile ("pause":::"memory");  /* smp_spinloop() */
-            i++;
-            assert(i < 1000000);
-        }
-        // for now:
-        assert(((nodeptr)stm_read_barrier((gcptr)r_prev->next))->value 
-               == r_current->value
-               && 
-               ((nodeptr)stm_read_barrier((gcptr)r_current->next))->value
-               == r_next->value);
-        
+        assert(stm_pointer_equal((gcptr)r_prev->next, (gcptr)r_current));
+        assert(stm_pointer_equal((gcptr)r_current->next, (gcptr)r_next));
+
         r_prev = r_current;
         r_current = r_next;
         r_next = r_next->next;

From noreply at buildbot.pypy.org  Tue Jun 25 11:59:16 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 25 Jun 2013 11:59:16 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: hg merge default
Message-ID: <20130625095916.C20601C0175@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64975:d7d523d14d09
Date: 2013-06-25 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/d7d523d14d09/

Log:	hg merge default

diff too long, truncating to 2000 out of 7016 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,6 +134,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -168,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,6 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
-jrel_op('JUMP_IF_NOT_DEBUG', 204)     # Distance to target address
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -278,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,48 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -707,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -934,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -8,7 +8,8 @@
 interpreter is written mostly in RPython (with pieces in Python), while
 the RPython compiler is written in Python. The hard to understand part
 is that Python is a meta-programming language for RPython, that is,
-RPython is considered from live objects **after** the imports are done.
+"being valid RPython" is a question that only makes sense on the
+live objects **after** the imports are done.
 This might require more explanation. You start writing RPython from
 ``entry_point``, a good starting point is
 ``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
@@ -37,7 +38,7 @@
 In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
 however, ``generator`` is not.
 
-A good introductory level articles are available:
+The following introductory level articles are available:
 
 * Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
 
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -76,4 +76,4 @@
 The rest of directories serve specific niche goal and are unlikely a good
 entry point.
 
-.. _`introduction to RPython`: getting-started-dev.rst
+.. _`introduction to RPython`: getting-started-dev.html
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -50,3 +50,10 @@
 .. branch: win32-fixes3
 Skip and fix some non-translated (own) tests for win32 builds
 
+.. branch: ctypes-byref
+Add the '_obj' attribute on ctypes pointer() and byref() objects
+
+.. branch: argsort-segfault
+Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
+
+.. branch: dtype-isnative
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -904,6 +904,9 @@
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
+    def newlist_unicode(self, list_u):
+        return self.newlist([self.wrap(u) for u in list_u])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -5,6 +5,22 @@
 
 @unwrap_spec(type=str)
 def newdict(space, type):
+    """ newdict(type)
+
+    Create a normal dict with a special implementation strategy.
+
+    type is a string and can be:
+
+    * "module" - equivalent to some_module.__dict__
+
+    * "instance" - equivalent to an instance dict with a not-changing-much
+                   set of keys
+
+    * "kwargs" - keyword args dict equivalent of what you get from **kwargs
+                 in a function, optimized for passing around
+
+    * "strdict" - string-key only dict. This one should be chosen automatically
+    """
     if type == 'module':
         return space.newdict(module=True)
     elif type == 'instance':
@@ -18,6 +34,10 @@
                               type)
 
 def dictstrategy(space, w_obj):
+    """ dictstrategy(dict)
+
+    show the underlaying strategy used by a dict object
+    """
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError,
                              space.wrap("expecting dict object"))
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -13,7 +13,6 @@
     _mixin_ = True
 
     def __init__(self, space, name, size, align, enumerators, enumvalues):
-        name = "enum " + name
         self._super.__init__(self, space, size, name, len(name), align)
         self.enumerators2values = {}   # str -> int
         self.enumvalues2erators = {}   # int -> str
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -289,16 +289,6 @@
                "with verify() (see pypy/module/_cffi_backend/ctypefunc.py "
                "for details)"))
 
-        if USE_C_LIBFFI_MSVC and is_result_type:
-            # MSVC returns small structures in registers.  Pretend int32 or
-            # int64 return type.  This is needed as a workaround for what
-            # is really a bug of libffi_msvc seen as an independent library
-            # (ctypes has a similar workaround).
-            if ctype.size <= 4:
-                return clibffi.ffi_type_sint32
-            if ctype.size <= 8:
-                return clibffi.ffi_type_sint64
-
         # walk the fields, expanding arrays into repetitions; first,
         # only count how many flattened fields there are
         nflat = 0
@@ -318,6 +308,16 @@
                                "a struct with a zero-length array"))
             nflat += flat
 
+        if USE_C_LIBFFI_MSVC and is_result_type:
+            # MSVC returns small structures in registers.  Pretend int32 or
+            # int64 return type.  This is needed as a workaround for what
+            # is really a bug of libffi_msvc seen as an independent library
+            # (ctypes has a similar workaround).
+            if ctype.size <= 4:
+                return clibffi.ffi_type_sint32
+            if ctype.size <= 8:
+                return clibffi.ffi_type_sint64
+
         # allocate an array of (nflat + 1) ffi_types
         elements = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * (nflat + 1))
         elements = rffi.cast(FFI_TYPE_PP, elements)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         else:
             extra = " *"
         self.is_file = (ctitem.name == "struct _IO_FILE" or
-                        ctitem.name == "struct $FILE")
+                        ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -25,7 +25,6 @@
     custom_field_pos = False
 
     def __init__(self, space, name):
-        name = '%s %s' % (self.kind, name)
         W_CType.__init__(self, space, -1, name, len(name))
 
     def check_complete(self, w_errorcls=None):
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 
 from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr,
     ctypearray, ctypestruct, ctypevoid, ctypeenum)
@@ -115,13 +116,15 @@
 # ____________________________________________________________
 
 SF_MSVC_BITFIELDS = 1
+SF_GCC_ARM_BITFIELDS = 2
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+elif rffi_platform.getdefined('__arm__', ''):
+    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
     DEFAULT_SFLAGS = 0
 
-
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -180,7 +183,7 @@
         # field is an anonymous bitfield
         falign = ftype.alignof()
         do_align = True
-        if fbitsize >= 0:
+        if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
                 # GCC: anonymous bitfields (of any size) don't cause alignment
                 do_align = (fname != '')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -641,6 +641,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -648,7 +650,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -657,7 +659,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -689,7 +691,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -708,7 +710,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -728,7 +730,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -744,7 +746,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -758,7 +760,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -792,7 +794,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -803,7 +805,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -832,7 +834,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -937,7 +939,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -953,7 +955,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -965,7 +967,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -985,7 +987,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1023,7 +1025,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1048,7 +1050,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1221,7 +1223,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1238,7 +1240,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1275,26 +1277,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1308,27 +1311,27 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1432,7 +1435,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1455,7 +1458,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1485,14 +1488,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1561,7 +1564,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1601,7 +1604,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1619,7 +1622,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1697,7 +1700,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1714,7 +1717,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1725,7 +1728,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1735,7 +1738,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1745,7 +1748,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1754,7 +1757,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1766,7 +1769,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1775,7 +1778,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1785,7 +1788,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1795,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1818,7 +1821,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1845,7 +1848,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1957,7 +1960,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1977,7 +1980,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2164,7 +2167,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2183,7 +2186,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2192,7 +2195,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2308,7 +2311,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2320,8 +2323,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2353,12 +2356,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2366,7 +2369,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2424,7 +2427,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2440,14 +2443,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2508,7 +2511,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2540,7 +2543,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2563,7 +2566,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2762,20 +2765,20 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2783,11 +2786,14 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
@@ -2795,10 +2801,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2807,6 +2818,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -13,19 +13,9 @@
 from pypy.module._rawffi.interp_rawffi import TYPEMAP
 from pypy.module._rawffi.interp_rawffi import size_alignment
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
 
-def push_elem(ll_array, pos, value):
-    TP = lltype.typeOf(value)
-    ll_array = rffi.cast(rffi.CArrayPtr(TP), ll_array)
-    ll_array[pos] = value
-push_elem._annspecialcase_ = 'specialize:argtype(2)'
-
-def get_elem(ll_array, pos, ll_t):
-    ll_array = rffi.cast(rffi.CArrayPtr(ll_t), ll_array)
-    return ll_array[pos]
-get_elem._annspecialcase_ = 'specialize:arg(2)'
-
 
 class W_Array(W_DataShape):
     def __init__(self, basicffitype, size):
@@ -57,7 +47,7 @@
                                                 " array length"))
             for num in range(iterlength):
                 w_item = items_w[num]
-                unwrap_value(space, push_elem, result.ll_buffer, num,
+                unwrap_value(space, write_ptr, result.ll_buffer, num,
                              self.itemcode, w_item)
         return space.wrap(result)
 
@@ -118,7 +108,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, push_elem, self.ll_buffer, num,
+        unwrap_value(space, write_ptr, self.ll_buffer, num,
                      self.shape.itemcode, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
@@ -136,7 +126,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, get_elem, self.ll_buffer, num,
+        return wrap_value(space, read_ptr, self.ll_buffer, num,
                           self.shape.itemcode)
 
     def descr_getitem(self, space, w_index):
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._rawffi.array import push_elem
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from pypy.module._rawffi.structure import W_Structure
 from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp,
      unwrap_value, unpack_argshapes, got_libffi_error)
@@ -40,7 +40,7 @@
                 args_w[i] = space.wrap(rffi.cast(rffi.ULONG, ll_args[i]))
         w_res = space.call(w_callable, space.newtuple(args_w))
         if callback_ptr.result is not None: # don't return void
-            unwrap_value(space, push_elem, ll_res, 0,
+            unwrap_value(space, write_ptr, ll_res, 0,
                          callback_ptr.result, w_res)
     except OperationError, e:
         tbprint(space, space.wrap(e.get_traceback()),
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,7 @@
 
 from rpython.rlib.clibffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
 import rpython.rlib.rposix as rposix
 
@@ -43,6 +44,8 @@
 }
 TYPEMAP_PTR_LETTERS = "POszZ"
 TYPEMAP_NUMBER_LETTERS = "bBhHiIlLqQ?"
+TYPEMAP_FLOAT_LETTERS = "fd" # XXX long doubles are not propperly supported in
+                             # rpython, so we ignore then here
 
 if _MS_WINDOWS:
     TYPEMAP['X'] = ffi_type_pointer
@@ -239,6 +242,52 @@
 )
 
 unroll_letters_for_numbers = unrolling_iterable(TYPEMAP_NUMBER_LETTERS)
+unroll_letters_for_floats = unrolling_iterable(TYPEMAP_FLOAT_LETTERS)
+
+_ARM = rffi_platform.getdefined('__arm__', '')
+
+def read_ptr(ptr, ofs, TP):
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as t_array:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, t_array),
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.sizeof(TP))
+                ptr_val = t_array[0]
+                return ptr_val
+    else:
+        return rffi.cast(T, ptr)[ofs]
+read_ptr._annspecialcase_ = 'specialize:arg(2)'
+
+def write_ptr(ptr, ofs, value):
+    TP = lltype.typeOf(value)
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as s_array:
+                s_array[0] = value
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.cast(rffi.VOIDP, s_array),
+                    rffi.sizeof(TP))
+                return
+    else:
+        rffi.cast(T, ptr)[ofs] = value
+write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -12,10 +12,13 @@
 from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance
 from pypy.module._rawffi.interp_rawffi import wrap_value, unwrap_value
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
-from pypy.module._rawffi.interp_rawffi import size_alignment, LL_TYPEMAP
+from pypy.module._rawffi.interp_rawffi import LL_TYPEMAP
 from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers
+from pypy.module._rawffi.interp_rawffi import size_alignment
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib import clibffi
-from rpython.rlib.rarithmetic import intmask, r_uint, signedtype, widen
+from rpython.rlib.rarithmetic import intmask, signedtype, widen
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong
 
 def unpack_fields(space, w_fields):
     fields_w = space.unpackiterable(w_fields)
@@ -260,10 +263,17 @@
 
 def LOW_BIT(x):
     return x & 0xFFFF
+
 def NUM_BITS(x):
     return x >> 16
-def BIT_MASK(x):
-    return (1 << x) - 1
+
+def BIT_MASK(x, ll_t):
+    if ll_t is lltype.SignedLongLong:
+        return (r_longlong(1) << x) - 1
+    elif ll_t is lltype.UnsignedLongLong:
+        return (r_ulonglong(1) << x) - 1
+    return (1  << x) -1
+BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
@@ -279,20 +289,20 @@
             lowbit = LOW_BIT(bitsize)
             if numbits:

From noreply at buildbot.pypy.org  Tue Jun 25 12:04:47 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Jun 2013 12:04:47 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: oops,
 this is harmless but can explode some assertion somewhere
Message-ID: <20130625100447.7BD8D1C11BA@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r64976:7d0fce8ba2f7
Date: 2013-06-25 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/7d0fce8ba2f7/

Log:	oops, this is harmless but can explode some assertion somewhere

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1329,6 +1329,7 @@
         fail_locs = [self.loc(v) for v in op.getfailargs()]
         self.assembler.store_force_descr(op, fail_locs,
                                          self.fm.get_frame_depth())
+        self.possibly_free_vars(guard_op.getfailargs())
 
     def consider_keepalive(self, op):
         pass

From noreply at buildbot.pypy.org  Tue Jun 25 12:11:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 12:11:08 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130625101108.5E67B1C0175@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r272:801f010da858
Date: 2013-06-25 12:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/801f010da858/

Log:	in-progress

diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -97,7 +97,7 @@
 DuObject *DuContainer_GetRef(DuObject *container);
 void DuContainer_SetRef(DuObject *container, DuObject *newobj);
 
-DuObject *DuSymbol_FromString(char *name);
+DuObject *DuSymbol_FromString(const char *name);
 char *DuSymbol_AsString(DuObject *ob);
 
 DuObject *DuCons_New(DuObject *car, DuObject *cdr);
@@ -123,10 +123,11 @@
                              DuObject *arglist, DuObject *progn);
 DuObject *_DuFrame_EvalCall(DuObject *frame, DuObject *symbol,
                             DuObject *rest, int execute_now);
+DuObject *_Du_GetGlobals(void);
 
 void Du_Initialize(void);
 void Du_Finalize(void);
-extern DuObject *Du_Globals;
+#define Du_Globals        (_Du_GetGlobals())
 
 void Du_TransactionAdd(DuObject *code, DuObject *frame);
 void Du_TransactionRun(void);
diff --git a/duhton/frame.c b/duhton/frame.c
--- a/duhton/frame.c
+++ b/duhton/frame.c
@@ -15,6 +15,17 @@
     struct dictentry *entries;
 } DuFrameObject;
 
+DuFrameObject Du_GlobalsFrame = {
+    DuOBJECT_HEAD_INIT(DUTYPE_FRAME),
+    0,
+    NULL,
+};
+
+DuObject *_Du_GetGlobals()
+{
+    return (DuObject *)&Du_GlobalsFrame;
+}
+
 DuObject *DuFrame_New()
 {
     DuFrameObject *ob = (DuFrameObject *)DuObject_New(&DuFrame_Type);
diff --git a/duhton/glob.c b/duhton/glob.c
--- a/duhton/glob.c
+++ b/duhton/glob.c
@@ -2,12 +2,6 @@
 #include <sys/select.h>
 
 
-static void _du_getargs0(const char *name, DuObject *cons, DuObject *locals)
-{
-    if (cons != Du_None)
-        Du_FatalError("%s: expected no argument", name);
-}
-
 static void _du_getargs1(const char *name, DuObject *cons, DuObject *locals,
                          DuObject **a)
 {
@@ -93,13 +87,14 @@
         DuObject *next = _DuCons_NEXT(cons);
 
         _du_save3(symbol, next, locals);
-        result = Du_Eval(expr, locals);
+        DuObject *obj = Du_Eval(expr, locals);
         _du_restore3(symbol, next, locals);
 
-        _du_save2(next, locals);
+        _du_save3(next, locals, obj);
         DuFrame_SetSymbol(locals, symbol, obj);
-        _du_restore2(next, locals);
+        _du_restore3(next, locals, obj);
 
+        result = obj;
         cons = next;
     }
     return result;
@@ -169,7 +164,7 @@
         DuObject *next = _DuCons_NEXT(cons);
 
         _du_save2(next, locals);
-        DuObject *obj = Du_Eval(next, locals);
+        DuObject *obj = Du_Eval(expr, locals);
         result += sign * DuInt_AsInt(obj);
         _du_restore2(next, locals);
 
@@ -232,7 +227,7 @@
 DuObject *du_type(DuObject *cons, DuObject *locals)
 {
     DuObject *obj;
-    _du_getarg1("type", cons, locals, &obj);
+    _du_getargs1("type", cons, locals, &obj);
 
     return DuSymbol_FromString(Du_TYPE(obj)->dt_name);
 }
@@ -268,13 +263,12 @@
 
 DuObject *du_container(DuObject *cons, DuObject *locals)
 {
-    DuObject *container;
     DuObject *obj;
 
     if (cons == Du_None)
         obj = Du_None;
     else
-        _du_getarg1("container", cons, locals, &obj);
+        _du_getargs1("container", cons, locals, &obj);
 
     return DuContainer_New(obj);
 }
@@ -315,45 +309,58 @@
 
 DuObject *du_set(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
         Du_FatalError("set: expected at least two arguments");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_save2(next, locals);
+    DuObject *obj = Du_Eval(expr, locals);
+    _du_restore2(next, locals);
+
+    _du_read1(next);
+    DuObject *expr2 = _DuCons_CAR(next);
+    DuObject *next2 = _DuCons_NEXT(next);
 
     if (DuList_Check(obj)) {
-        if (_DuCons_NEXT(_DuCons_NEXT(cons)) == Du_None ||
-            _DuCons_NEXT(_DuCons_NEXT(_DuCons_NEXT(cons))) != Du_None)
+        _du_read1(next2);
+        if (next2 == Du_None || _DuCons_NEXT(next2) != Du_None)
             Du_FatalError("set with a list: expected three arguments");
-        DuObject *index = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
-        DuObject *newobj = Du_Eval(
-                    _DuCons_CAR(_DuCons_NEXT(_DuCons_NEXT(cons))), locals);
+
+        _du_save3(obj, next2, locals);
+        DuObject *index = Du_Eval(expr2, locals);
+        _du_restore3(obj, next2, locals);
+
+        _du_save2(obj, index);
+        DuObject *newobj = Du_Eval(_DuCons_CAR(next2), locals);
+        _du_restore2(obj, index);
+
         DuList_SetItem(obj, DuInt_AsInt(index), newobj);
-        Du_DECREF(index);
-        Du_DECREF(newobj);
     }
     else if (DuContainer_Check(obj)) {
-        if (_DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
+        if (next2 != Du_None)
             Du_FatalError("set with a container: expected two arguments");
-        DuObject *newobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+
+        _du_save1(obj);
+        DuObject *newobj = Du_Eval(expr2, locals);
+        _du_restore1(obj);
+
         DuContainer_SetRef(obj, newobj);
-        Du_DECREF(newobj);
     }
     else
-        Du_FatalError("set: bad argument type '%s'", obj->ob_type->dt_name);
+        Du_FatalError("set: bad argument type '%s'", Du_TYPE(obj)->dt_name);
 
-    Du_DECREF(obj);
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
 DuObject *du_append(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None ||
-        _DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
-        Du_FatalError("append: expected two arguments");
-    DuObject *lst = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *newobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    DuObject *lst, *newobj;
+    _du_getargs2("append", cons, locals, &lst, &newobj);
+
     DuList_Append(lst, newobj);
-    Du_DECREF(lst);
     return newobj;
 }
 
@@ -361,50 +368,69 @@
 {
     if (cons == Du_None)
         Du_FatalError("pop: expected at least one argument");
-    DuObject *lst = Du_Eval(_DuCons_CAR(cons), locals);
+
+    _du_read1(cons);
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_save2(next, locals);
+    DuObject *lst = Du_Eval(expr, locals);
+    _du_restore2(next, locals);
+
     int index;
-    if (_DuCons_NEXT(cons) == Du_None) {
+    if (next == Du_None) {
         index = DuList_Size(lst) - 1;
         if (index < 0)
             Du_FatalError("pop: empty list");
     }
-    else if (_DuCons_NEXT(_DuCons_NEXT(cons)) == Du_None) {
-        DuObject *indexobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    else {
+        _du_read1(next);
+        DuObject *expr2 = _DuCons_CAR(next);
+        DuObject *next2 = _DuCons_NEXT(next);
+
+        if (next2 != Du_None)
+            Du_FatalError("pop: expected at most two arguments");
+
+        _du_save1(lst);
+        DuObject *indexobj = Du_Eval(expr2, locals);
+        _du_restore1(lst);
+
         index = DuInt_AsInt(indexobj);
-        Du_DECREF(indexobj);
     }
-    else
-        Du_FatalError("pop: expected at most two arguments");
 
-    DuObject *res = DuList_Pop(lst, index);
-    Du_DECREF(lst);
-    return res;
+    return DuList_Pop(lst, index);
 }
 
 DuObject *du_len(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("len: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("len", cons, locals, &obj);
+
     int length = DuObject_Length(obj);
-    Du_DECREF(obj);
     return DuInt_FromInt(length);
 }
 
 DuObject *du_if(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
         Du_FatalError("if: expected at least two arguments");
-    DuObject *cond = Du_Eval(_DuCons_CAR(cons), locals);
-    int cond_int = DuObject_IsTrue(cond);
-    Du_DECREF(cond);
-    if (cond_int != 0) {
+
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_save2(next, locals);
+    DuObject *cond = Du_Eval(expr, locals);
+    _du_restore2(next, locals);
+
+    _du_read1(next);
+    if (DuObject_IsTrue(cond) != 0) {
         /* true path */
-        return Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+        return Du_Eval(_DuCons_CAR(next), locals);
     }
     else {
         /* false path */
-        return Du_Progn(_DuCons_NEXT(_DuCons_NEXT(cons)), locals);
+        return Du_Progn(_DuCons_NEXT(next), locals);
     }
 }
 
@@ -412,63 +438,73 @@
 {
     if (cons == Du_None)
         Du_FatalError("while: expected at least one argument");
+
+    _du_read1(cons);
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
     while (1) {
-        DuObject *cond = Du_Eval(_DuCons_CAR(cons), locals);
-        int cond_int = DuObject_IsTrue(cond);
-        Du_DECREF(cond);
-        if (cond_int == 0)
+        _du_save3(expr, next, locals);
+        DuObject *cond = Du_Eval(expr, locals);
+        _du_restore3(expr, next, locals);
+
+        if (!DuObject_IsTrue(cond))
             break;
-        DuObject *res = Du_Progn(_DuCons_NEXT(cons), locals);
-        Du_DECREF(res);
+
+        _du_save3(expr, next, locals);
+        Du_Progn(next, locals);
+        _du_restore3(expr, next, locals);
     }
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
 DuObject *du_defun(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
         Du_FatalError("defun: expected at least two arguments");
+
     DuObject *name = _DuCons_CAR(cons);
-    DuObject *arglist = _DuCons_CAR(_DuCons_NEXT(cons));
-    DuObject *progn = _DuCons_NEXT(_DuCons_NEXT(cons));
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_read1(next);
+    DuObject *arglist = _DuCons_CAR(next);
+    DuObject *progn = _DuCons_NEXT(next);
+
     DuFrame_SetUserFunction(locals, name, arglist, progn);
-    Du_INCREF(Du_None);
+
     return Du_None;
 }
 
 DuObject *du_car(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("car: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *res = DuCons_Car(obj);
-    Du_DECREF(obj);
-    return res;
+    DuObject *obj;
+    _du_getargs1("car", cons, locals, &obj);
+
+    return DuCons_Car(obj);
 }
 
 DuObject *du_cdr(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("car: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *res = DuCons_Cdr(obj);
-    Du_DECREF(obj);
-    return res;
+    DuObject *obj;
+    _du_getargs1("cdr", cons, locals, &obj);
+
+    return DuCons_Cdr(obj);
 }
 
 DuObject *du_not(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("not: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("not", cons, locals, &obj);
+
     int res = !DuObject_IsTrue(obj);
-    Du_DECREF(obj);
     return DuInt_FromInt(res);
 }
 
 DuObject *du_transaction(DuObject *cons, DuObject *locals)
 {
+    Du_FatalError("transaction: not implemented");
+#if 0
     if (cons == Du_None)
         Du_FatalError("transaction: expected at least one argument");
     DuObject *sym = _DuCons_CAR(cons);
@@ -476,61 +512,57 @@
     _DuFrame_EvalCall(locals, sym, rest, 0);
     Du_INCREF(Du_None);
     return Du_None;
+#endif
 }
 
 DuObject *du_sleepms(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("sleepms: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("sleepms", cons, locals, &obj);
+
     int ms = DuInt_AsInt(obj);
-    Du_DECREF(obj);
 
     struct timeval t;
     t.tv_sec = ms / 1000;
     t.tv_usec = (ms % 1000) * 1000;
     select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &t);
 
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
 DuObject *du_defined(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
         Du_FatalError("defined?: expected one argument");
     DuObject *ob = _DuCons_CAR(cons);
+
+    _du_save1(ob);
     DuObject *res = DuFrame_GetSymbol(locals, ob);
+    _du_restore1(ob);
+
     if (res == NULL)
-        DuFrame_GetSymbol(Du_Globals, ob);
-    if (res != NULL)
-        Du_DECREF(res);
+        res = DuFrame_GetSymbol(Du_Globals, ob);
+
     return DuInt_FromInt(res != NULL);
 }
 
 DuObject *du_assert(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("defined?: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("assert", cons, locals, &obj);
+
     if (!DuInt_AsInt(obj)) {
         printf("assert failed: ");
+        _du_read1(cons);
         Du_Print(_DuCons_CAR(cons), 1);
         Du_FatalError("assert failed");
     }
-    Du_DECREF(obj);
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
-DuObject *Du_Globals;
-
 void Du_Initialize(void)
 {
-    _Du_AME_InitThreadDescriptor();
-    _Du_InitializeObjects();
-
-    Du_Globals = DuFrame_New();
     DuFrame_SetBuiltinMacro(Du_Globals, "progn", Du_Progn);
     DuFrame_SetBuiltinMacro(Du_Globals, "setq", du_setq);
     DuFrame_SetBuiltinMacro(Du_Globals, "print", du_print);
@@ -567,9 +599,4 @@
 
 void Du_Finalize(void)
 {
-    Du_DECREF(Du_Globals);
-    Du_Globals = NULL;
-
-    _Du_FinalizeObjects();
-    _Du_AME_FiniThreadDescriptor();
 }
diff --git a/duhton/listobject.c b/duhton/listobject.c
--- a/duhton/listobject.c
+++ b/duhton/listobject.c
@@ -21,12 +21,14 @@
 {
     int i;
     _du_read1(ob);
-    if (ob->ob_count == 0) {
+
+    DuTupleObject *p = ob->ob_tuple;
+    _du_read1(p);
+
+    if (p->ob_count == 0) {
         printf("[]");
     }
     else {
-        DuTupleObject *p = ob->ob_tuple;
-        _du_read1(p);
         printf("[ ");
         for (i=0; i<p->ob_count; i++) {
             Du_Print(p->ob_items[i], 0);
@@ -63,8 +65,8 @@
     DuTupleObject *newitems = DuTuple_New(newcount);
 
     for (i=0; i<newcount-1; i++)
-        newitems[i] = olditems[i];
-    newitems[newcount-1] = x;
+        newitems->ob_items[i] = olditems->ob_items[i];
+    newitems->ob_items[newcount-1] = x;
 
     ob->ob_tuple = newitems;
 }
@@ -124,6 +126,7 @@
     if (index < 0 || index >= p->ob_count)
         Du_FatalError("list_pop: index out of range");
     DuObject *result = p->ob_items[index];
+    int i;
     p->ob_count--;
     for (i=index; i<p->ob_count; i++)
         p->ob_items[i] = p->ob_items[i+1];
@@ -149,7 +152,7 @@
 static DuTupleObject du_empty_tuple = {
     DuOBJECT_HEAD_INIT(DUTYPE_TUPLE),
     0,
-}
+};
 
 DuObject *DuList_New()
 {
diff --git a/duhton/symbol.c b/duhton/symbol.c
--- a/duhton/symbol.c
+++ b/duhton/symbol.c
@@ -43,7 +43,7 @@
     (eval_fn)symbol_eval,
 };
 
-DuObject *DuSymbol_FromString(char *name)
+DuObject *DuSymbol_FromString(const char *name)
 {
     DuSymbolObject *p, *head = &_Du_AllSymbols;
     for (p=head; p != NULL; p=p->next) {
diff --git a/duhton/transaction.c b/duhton/transaction.c
--- a/duhton/transaction.c
+++ b/duhton/transaction.c
@@ -1,7 +1,7 @@
 #include "duhton.h"
 
 
-void Du_TransactionAdd(DuObject *frame)
+void Du_TransactionAdd(DuObject *code, DuObject *frame)
 {
     /* XXX */
 }

From noreply at buildbot.pypy.org  Tue Jun 25 14:02:01 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Jun 2013 14:02:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: do not use backup as shadow-original,
 always use additional shadow
Message-ID: <20130625120201.3759A1C3408@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r273:d01dac528994
Date: 2013-06-25 14:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/d01dac528994/

Log:	do not use backup as shadow-original, always use additional shadow

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -55,7 +55,6 @@
     unsigned int thread_seed;
     gcptr roots[MAXROOTS];
     gcptr roots_outside_perform[MAXROOTS];
-    gcptr current_root;
     int num_roots;
     int num_roots_outside_perform;
     int steps_left;
@@ -68,6 +67,12 @@
 int classify(gcptr p);
 void check(gcptr p);
 
+static int is_private(gcptr P)
+{
+  return (P->h_revision == stm_private_rev_num) ||
+    (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+}
+
 int get_rand(int max)
 {
     return (int)(rand_r(&td.thread_seed) % (unsigned int)max);
@@ -101,7 +106,8 @@
     int i;
     for (i = 0; i < td.num_roots; i++) {
         check(td.roots[i]);
-        stm_push_root(td.roots[i]);
+        if (td.roots[i])
+            stm_push_root(td.roots[i]);
     }
 }
 
@@ -109,7 +115,8 @@
 {
     int i;
     for (i = td.num_roots - 1; i >= 0; i--) {
-        td.roots[i] = stm_pop_root();
+        if (td.roots[i])
+            td.roots[i] = stm_pop_root();
         check(td.roots[i]);
     }
 }
@@ -173,6 +180,7 @@
         check(p);
         w = stm_write_barrier(p);
         check(w);
+        assert(is_private(w));
     }
     return w;
 }
@@ -351,7 +359,7 @@
         if (w_t->id) {
             assert(w_t->id == stm_id((gcptr)w_t));
             assert(w_t->id == stm_id((gcptr)_t));
-        } 
+        }
         else {
             w_t = (nodeptr)write_barrier(_t);
             w_t->id = stm_id((gcptr)w_t);
@@ -425,9 +433,7 @@
 int run_me()
 {
     gcptr p = NULL;
-    while (td.steps_left) {
-        td.steps_left--;
-
+    while (td.steps_left-->0) {
         if (td.steps_left % 8 == 0)
             fprintf(stdout, "#");
 
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1120,14 +1120,6 @@
                 break;
             }
         }      
-      else if (P->h_original == (revision_t)B) {
-        /* The backup is the "id object" */
-        assert(!(P->h_tid & GCFLAG_HAS_ID));
-
-        B->h_tid &= ~GCFLAG_BACKUP_COPY;
-        B->h_tid |= GCFLAG_PUBLIC;
-        B->h_revision = (revision_t)P;
-      }
       else
         {
           stmgcpage_free(B);
@@ -1159,7 +1151,7 @@
         {
           assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
           P->h_tid |= GCFLAG_PUBLIC;
-          P->h_tid &= ~GCFLAG_HAS_ID; // just in case
+          assert(!(P->h_tid & GCFLAG_HAS_ID));
           if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
           /* P becomes a public outdated object.  It may create an
              exception documented in doc-objects.txt: a public but young
@@ -1168,24 +1160,16 @@
              stealing will follow its h_revision (to B).
           */
         }
-      else if (P->h_original == (revision_t)B) {
-        /* The backup is the "id object".  P becomes outdated. */
-        assert(!(P->h_tid & GCFLAG_HAS_ID));
-        P->h_tid |= GCFLAG_PUBLIC;
-        B->h_tid |= GCFLAG_PUBLIC;
-        B->h_tid &= ~GCFLAG_BACKUP_COPY;
-        if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
-      }
       else
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
-
+          assert(!(P->h_original) 
+                 || (B->h_original == (revision_t)P->h_original));
+          assert(!(P->h_original && !B->h_original));
           if (P->h_original && !B->h_original) // might occur because of
             B->h_original = P->h_original; //replace_ptr_to_protected_with_stub
 
-          assert(!(P->h_original) 
-                 || (B->h_original == (revision_t)P->h_original));
           size_t size = stmcb_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -159,18 +159,10 @@
                  p, (gcptr)p->h_original));
         return p->h_original;
     } 
-    else if (!(p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
-               && (p->h_tid & GCFLAG_OLD)) {
-        /* we can be sure that p->h_original doesn't
-           get set during the if and the else-if 
-           
-           XXX: check for priv_from_protected may not be 
-           necessary. only if this func may be called on 
-           another thread's young objects that are made 
-           old at the same time, and we see the OLD flag 
-           before h_original has been set.
-        */
-        dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p));
+    else if (p->h_tid & GCFLAG_OLD) {
+        /* old objects must have an h_original xOR be
+           the original itself. */
+        /* dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p)); */
         return (revision_t)p;
     }
     
@@ -188,35 +180,24 @@
         dprintf(("stm_id(%p) has orig: %p\n", 
                  p, (gcptr)p->h_original));
     }
-    else if (p->h_tid & GCFLAG_OLD) {
-        /* it must be this exact object */
-        result = (revision_t)p;
-        dprintf(("stm_id(%p) is old, orig=0: %p\n", p, p));
-    }
     else {
-        /* must create shadow original object or use
+        /* must create shadow original object XXX: or use
            backup, if exists */
+        
+        /* XXX use stmgcpage_malloc() directly, we don't need to copy
+         * the contents yet */
+        gcptr O = stmgc_duplicate_old(p);
+        p->h_original = (revision_t)O;
+        p->h_tid |= GCFLAG_HAS_ID;
+        O->h_tid |= GCFLAG_PUBLIC;
+        
         if (p->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
             gcptr B = (gcptr)p->h_revision;
-            /* don't set HAS_ID, otherwise nursery will copy over backup */
-            p->h_original = (revision_t)B;
-            // B->h_tid |= GCFLAG_PUBLIC; done by CommitPrivateFromProtected
-            
-            result = (revision_t)B;
-            dprintf(("stm_id(%p) young, pfp, use backup %p\n", 
-                     p, (gcptr)p->h_original));
+            B->h_original = (revision_t)O;
         }
-        else {
-            /* XXX use stmgcpage_malloc() directly, we don't need to copy
-             * the contents yet */
-            gcptr O = stmgc_duplicate_old(p);
-            p->h_original = (revision_t)O;
-            p->h_tid |= GCFLAG_HAS_ID;
-            O->h_tid |= GCFLAG_PUBLIC;
-            
-            result = (revision_t)O;
-            dprintf(("stm_id(%p) young, make shadow %p\n", p, O));
-        }
+        
+        result = (revision_t)O;
+        dprintf(("stm_id(%p) young, make shadow %p\n", p, O));
     }
     
     spinlock_release(d->public_descriptor->collection_lock);
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -96,6 +96,7 @@
         if (L->h_original) {
             /* L has an original, may be GCFLAG_HAS_ID */
             B->h_original = L->h_original;
+            L->h_tid &= ~GCFLAG_HAS_ID;
         }
         else if (L->h_tid & GCFLAG_OLD) {
             /* If old, it must be the original */
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -498,10 +498,11 @@
     test_stub_for_refs_from_stolen(old=True)
 
 
-def id_with_stealing(a=0, b=0):
+def id_with_stealing(a, b, c):
     p = palloc_refs(2)
     qlist = []
     rlist = []
+    plist = []
     qid = []
     pid = []
     rid = []
@@ -509,13 +510,14 @@
         r1 = nalloc(HDR)
         q1 = nalloc(HDR)
         p1 = lib.stm_write_barrier(p)   # private copy
+        plist.append(p1)
         qlist.append(q1)
         rlist.append(r1)
-        if a:
+        if c and a:
             # id on young priv
             qid.append(lib.stm_id(q1))
             assert q1.h_tid & GCFLAG_HAS_ID
-        if b:
+        if c and b:
             # id on pub_to_priv
             assert follow_original(p1) == p
             pid.append(lib.stm_id(p1))
@@ -526,21 +528,23 @@
         lib.stm_begin_inevitable_transaction()
         # p old public -> stub -> p1 young prot
         # q1 young prot, r1 young prot
-        if not a:
+        if c and not a:
             # id on young prot
             qid.append(lib.stm_id(q1))
             assert q1.h_tid & GCFLAG_HAS_ID
-        if not b:
+        if c and not b:
             # id on young prot
             assert follow_original(p1) == p
             pid.append(lib.stm_id(p1))
             assert not (p1.h_tid & GCFLAG_HAS_ID)
             
-        r1w = lib.stm_write_barrier(r1) # priv_from_prot
-        assert r1w.h_tid & GCFLAG_PRIVATE_FROM_PROTECTED
-        rid.append(lib.stm_id(r1w))
-        assert not (r1w.h_tid & GCFLAG_HAS_ID) # use backup
-        assert follow_original(r1w) == follow_revision(r1w)
+        r1w = lib.stm_write_barrier(r1)
+        assert classify(r1w) == "private_from_protected"
+        assert not (r1w.h_tid & GCFLAG_OLD)
+        if c:
+                rid.append(lib.stm_id(r1w))
+        # assert not (r1w.h_tid & GCFLAG_HAS_ID) # use backup <- XXX
+        # assert follow_original(r1w) == follow_revision(r1w)
         
         r.set(2)
         r.wait(3)     # wait until the other thread really started
@@ -548,26 +552,45 @@
         r.wait(2)
         r.set(3)
 
-        p2 = lib.stm_read_barrier(p)    # steals
+        assert classify(plist[-1]) == "protected"
+        p2 = lib.stm_read_barrier(plist[-1])    # steals
+        if not c:
+            pid.append(lib.stm_id(p2))
         assert pid[-1] == lib.stm_id(p2)
 
         r2 = lib.getptr(p2, 1)
         r3 = lib.stm_read_barrier(r2)
+        if not c:
+            rid.append(lib.stm_id(r3))
         assert lib.stm_id(r3) == rid[-1]
         
         q2 = lib.getptr(p2, 0)
+        q3 = lib.stm_read_barrier(q2)
+        if not c:
+            qid.append(lib.stm_id(q3))
+
+        assert lib.stm_id(q3) == qid[-1]
         assert lib.stm_id(q2) == qid[-1]
 
-        q3 = lib.stm_read_barrier(q2)
-        assert lib.stm_id(q3) == qid[-1]
-
     run_parallel(f1, f2)
 
 def test_id_with_stealing():
-    id_with_stealing(1, 1)
-    id_with_stealing(1, 0)
-    id_with_stealing(0, 1)
-    id_with_stealing(0, 0)
+    id_with_stealing(1, 1, 0)
+def test_id_with_stealing1():
+    id_with_stealing(1, 0, 0)
+def test_id_with_stealing2():
+    id_with_stealing(0, 1, 0)
+def test_id_with_stealing3():
+    id_with_stealing(0, 0, 0)
+def test_id_with_stealing4():
+    id_with_stealing(1, 1, 1)
+def test_id_with_stealing5():
+    id_with_stealing(1, 0, 1)
+def test_id_with_stealing6():
+    id_with_stealing(0, 1, 1)
+def test_id_with_stealing7():
+    id_with_stealing(0, 0, 1)
+
 
 
 def test_prehash_simple():

From noreply at buildbot.pypy.org  Tue Jun 25 14:02:02 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Jun 2013 14:02:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: merge
Message-ID: <20130625120202.5C74F1C3408@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r274:5634ab8d11f6
Date: 2013-06-25 14:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/5634ab8d11f6/

Log:	merge

diff --git a/c4/Makefile b/c4/Makefile
--- a/c4/Makefile
+++ b/c4/Makefile
@@ -2,14 +2,16 @@
 # Makefile for the demos.
 #
 
+DEBUG_EXE = debug-demo1 debug-demo2 debug-demo_random
 BUILD_EXE = build-demo1 build-demo2 build-demo_random
-DEBUG_EXE = debug-demo1 debug-demo2 debug-demo_random
+RELEASE_EXE = release-demo1 release-demo2 release-demo_random
 
-build: $(BUILD_EXE)
-debug: $(DEBUG_EXE)
+debug: $(DEBUG_EXE)       # with prints and asserts
+build: $(BUILD_EXE)       # without prints, but with asserts
+release: $(RELEASE_EXE)   # without prints nor asserts
 
 clean:
-	rm -f $(BUILD_EXE) $(DEBUG_EXE)
+	rm -f $(BUILD_EXE) $(DEBUG_EXE) $(RELEASE_EXE)
 
 
 H_FILES = atomic_ops.h stmgc.h stmimpl.h \
@@ -29,5 +31,8 @@
 debug-%: %.c ${H_FILES} ${C_FILES}
 	gcc -lrt -pthread ${DEBUG} $< -o debug-$* -Wall ${C_FILES}
 
+release-%: %.c ${H_FILES} ${C_FILES} stmgc.c
+	gcc -lrt -pthread -DNDEBUG -O2 -g $< -o release-$* -Wall stmgc.c
+
 test-%:
 	./$* 2>/dev/null | grep "check ok"
diff --git a/c4/demo1.c b/c4/demo1.c
--- a/c4/demo1.c
+++ b/c4/demo1.c
@@ -11,7 +11,11 @@
 #include "fprintcolor.h"
 
 
-#define UPPER_BOUND 100
+#ifdef _GC_DEBUG
+# define UPPER_BOUND 100
+#else
+# define UPPER_BOUND 5000
+#endif
 #define NUMTHREADS  4
 
 
@@ -134,7 +138,8 @@
     stm_finalize();
 
     status = sem_post(&done);
-    assert(status == 0);
+    if (status != 0)
+        stm_fatalerror("status != 0\n");
     return NULL;
 }
 
@@ -158,7 +163,8 @@
 {
   pthread_t th;
   int status = pthread_create(&th, NULL, func, arg);
-  assert(status == 0);
+  if (status != 0)
+      stm_fatalerror("status != 0\n");
   pthread_detach(th);
   printf("started new thread\n");
 }
@@ -168,7 +174,8 @@
   int i, status;
 
   status = sem_init(&done, 0, 0);
-  assert(status == 0);
+  if (status != 0)
+      stm_fatalerror("status != 0\n");
 
   for (i = 0; i < NUMTHREADS; i++)
     newthread(demo1, NULL);
@@ -176,7 +183,8 @@
   for (i=0; i < NUMTHREADS; i++)
     {
       status = sem_wait(&done);
-      assert(status == 0);
+      if (status != 0)
+          stm_fatalerror("status != 0\n");
       printf("thread finished\n");
     }
 
diff --git a/c4/demo2.c b/c4/demo2.c
--- a/c4/demo2.c
+++ b/c4/demo2.c
@@ -97,23 +97,9 @@
             r_current = (struct node*)stm_read_barrier((gcptr)r_next);
             r_next = (struct node*)stm_read_barrier((gcptr)tmp);
         }
-        // results from consecutive read_barriers can differ. needs Ptr_Eq()
-        int i = 0;
-        while (!(stm_read_barrier((gcptr)r_prev->next) ==
-                 stm_read_barrier((gcptr)r_current) &&
-                 stm_read_barrier((gcptr)r_current->next) ==
-                 stm_read_barrier((gcptr)r_next))) {
-            asm volatile ("pause":::"memory");  /* smp_spinloop() */
-            i++;
-            assert(i < 1000000);
-        }
-        // for now:
-        assert(((nodeptr)stm_read_barrier((gcptr)r_prev->next))->value 
-               == r_current->value
-               && 
-               ((nodeptr)stm_read_barrier((gcptr)r_current->next))->value
-               == r_next->value);
-        
+        assert(stm_pointer_equal((gcptr)r_prev->next, (gcptr)r_current));
+        assert(stm_pointer_equal((gcptr)r_current->next, (gcptr)r_next));
+
         r_prev = r_current;
         r_current = r_next;
         r_next = r_next->next;
@@ -154,7 +140,8 @@
 void final_check(void)
 {
     long sum;
-    
+
+    printf("final check\n");
     stm_initialize();
     
     sum = check_sorted();
@@ -171,7 +158,8 @@
 {
     pthread_t th;
     int status = pthread_create(&th, NULL, func, arg);
-    assert(status == 0);
+    if (status != 0)
+        stm_fatalerror("newthread: pthread_create failure\n");
     pthread_detach(th);
     printf("started new thread\n");
 }
diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -473,7 +473,8 @@
 {
     pthread_t th;
     int status = pthread_create(&th, NULL, func, arg);
-    assert(status == 0);
+    if (status != 0)
+        stm_fatalerror("newthread: pthread_create failure\n");
     pthread_detach(th);
     printf("started new thread\n");
 }
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -305,6 +305,7 @@
 
 static void _check_flags(gcptr P)
 {
+#ifndef NDEBUG
   struct tx_descriptor *d = thread_descriptor;
   if (P->h_tid & GCFLAG_STUB)
     {
@@ -322,6 +323,7 @@
       assert(is_old);
       dprintf(("O "));
     }
+#endif
 }
 
 gcptr _stm_nonrecord_barrier(gcptr P)
@@ -962,7 +964,6 @@
 
 static void CancelLocks(struct tx_descriptor *d)
 {
-  revision_t my_lock = d->my_lock;
   wlog_t *item;
 
   if (!g2l_any_entry(&d->public_to_private))
@@ -984,7 +985,7 @@
 
       if (v == expected)
         {
-          assert(R->h_revision != my_lock);
+          assert(R->h_revision != d->my_lock);
           break;    /* done */
         }
 
@@ -993,7 +994,7 @@
 #ifdef DUMP_EXTRA
       dprintf(("%p->h_revision = %p (CancelLocks)\n", R, (gcptr)v));
 #endif
-      assert(R->h_revision == my_lock);
+      assert(R->h_revision == d->my_lock);
       ACCESS_ONCE(R->h_revision) = v;
 
     } G2L_LOOP_END;
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -39,13 +39,17 @@
 void stmgcpage_acquire_global_lock(void)
 {
     int err = pthread_mutex_lock(&mutex_gc_lock);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stmgcpage_acquire_global_lock: "
+                       "pthread_mutex_lock() failure\n");
 }
 
 void stmgcpage_release_global_lock(void)
 {
     int err = pthread_mutex_unlock(&mutex_gc_lock);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stmgcpage_release_global_lock: "
+                       "pthread_mutex_unlock() failure\n");
 }
 
 
@@ -158,7 +162,9 @@
     }
 }
 
+#ifndef NDEBUG
 static unsigned char random_char = 0x55;
+#endif
 
 void stmgcpage_free(gcptr obj)
 {
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -57,7 +57,8 @@
 void stm_initialize(void)
 {
     int r = DescriptorInit();
-    assert(r == 1);
+    if (r != 1)
+        stm_fatalerror("stm_initialize: DescriptorInit failure\n");
     stmgc_init_nursery();
     init_shadowstack();
     //stmgcpage_init_tls();
@@ -227,7 +228,9 @@
 void stm_start_sharedlock(void)
 {
     int err = pthread_rwlock_rdlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stm_start_sharedlock: "
+                       "pthread_rwlock_rdlock failure\n");
     //assert(stmgc_nursery_hiding(thread_descriptor, 0));
     dprintf(("stm_start_sharedlock\n"));
 }
@@ -237,13 +240,17 @@
     dprintf(("stm_stop_sharedlock\n"));
     //assert(stmgc_nursery_hiding(thread_descriptor, 1));
     int err = pthread_rwlock_unlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stm_stop_sharedlock: "
+                       "pthread_rwlock_unlock failure\n");
 }
 
 static void start_exclusivelock(void)
 {
     int err = pthread_rwlock_wrlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("start_exclusivelock: "
+                       "pthread_rwlock_wrlock failure\n");
     dprintf(("start_exclusivelock\n"));
 }
 
@@ -251,7 +258,9 @@
 {
     dprintf(("stop_exclusivelock\n"));
     int err = pthread_rwlock_unlock(&rwlock_shared);
-    assert(err == 0);
+    if (err != 0)
+        stm_fatalerror("stop_exclusivelock: "
+                       "pthread_rwlock_unlock failure\n");
 }
 
 void stm_start_single_thread(void)
@@ -289,9 +298,8 @@
 
     /* Warning, may block waiting for rwlock_in_transaction while another
        thread runs a major GC */
-    struct tx_descriptor *d = thread_descriptor;
-    assert(d->active);
-    assert(in_single_thread != d);
+    assert(thread_descriptor->active);
+    assert(in_single_thread != thread_descriptor);
 
     stm_stop_sharedlock();
     /* another thread should be waiting in start_exclusivelock(),
diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -97,7 +97,7 @@
 DuObject *DuContainer_GetRef(DuObject *container);
 void DuContainer_SetRef(DuObject *container, DuObject *newobj);
 
-DuObject *DuSymbol_FromString(char *name);
+DuObject *DuSymbol_FromString(const char *name);
 char *DuSymbol_AsString(DuObject *ob);
 
 DuObject *DuCons_New(DuObject *car, DuObject *cdr);
@@ -123,10 +123,11 @@
                              DuObject *arglist, DuObject *progn);
 DuObject *_DuFrame_EvalCall(DuObject *frame, DuObject *symbol,
                             DuObject *rest, int execute_now);
+DuObject *_Du_GetGlobals(void);
 
 void Du_Initialize(void);
 void Du_Finalize(void);
-extern DuObject *Du_Globals;
+#define Du_Globals        (_Du_GetGlobals())
 
 void Du_TransactionAdd(DuObject *code, DuObject *frame);
 void Du_TransactionRun(void);
diff --git a/duhton/frame.c b/duhton/frame.c
--- a/duhton/frame.c
+++ b/duhton/frame.c
@@ -15,6 +15,17 @@
     struct dictentry *entries;
 } DuFrameObject;
 
+DuFrameObject Du_GlobalsFrame = {
+    DuOBJECT_HEAD_INIT(DUTYPE_FRAME),
+    0,
+    NULL,
+};
+
+DuObject *_Du_GetGlobals()
+{
+    return (DuObject *)&Du_GlobalsFrame;
+}
+
 DuObject *DuFrame_New()
 {
     DuFrameObject *ob = (DuFrameObject *)DuObject_New(&DuFrame_Type);
diff --git a/duhton/glob.c b/duhton/glob.c
--- a/duhton/glob.c
+++ b/duhton/glob.c
@@ -2,12 +2,6 @@
 #include <sys/select.h>
 
 
-static void _du_getargs0(const char *name, DuObject *cons, DuObject *locals)
-{
-    if (cons != Du_None)
-        Du_FatalError("%s: expected no argument", name);
-}
-
 static void _du_getargs1(const char *name, DuObject *cons, DuObject *locals,
                          DuObject **a)
 {
@@ -93,13 +87,14 @@
         DuObject *next = _DuCons_NEXT(cons);
 
         _du_save3(symbol, next, locals);
-        result = Du_Eval(expr, locals);
+        DuObject *obj = Du_Eval(expr, locals);
         _du_restore3(symbol, next, locals);
 
-        _du_save2(next, locals);
+        _du_save3(next, locals, obj);
         DuFrame_SetSymbol(locals, symbol, obj);
-        _du_restore2(next, locals);
+        _du_restore3(next, locals, obj);
 
+        result = obj;
         cons = next;
     }
     return result;
@@ -169,7 +164,7 @@
         DuObject *next = _DuCons_NEXT(cons);
 
         _du_save2(next, locals);
-        DuObject *obj = Du_Eval(next, locals);
+        DuObject *obj = Du_Eval(expr, locals);
         result += sign * DuInt_AsInt(obj);
         _du_restore2(next, locals);
 
@@ -232,7 +227,7 @@
 DuObject *du_type(DuObject *cons, DuObject *locals)
 {
     DuObject *obj;
-    _du_getarg1("type", cons, locals, &obj);
+    _du_getargs1("type", cons, locals, &obj);
 
     return DuSymbol_FromString(Du_TYPE(obj)->dt_name);
 }
@@ -268,13 +263,12 @@
 
 DuObject *du_container(DuObject *cons, DuObject *locals)
 {
-    DuObject *container;
     DuObject *obj;
 
     if (cons == Du_None)
         obj = Du_None;
     else
-        _du_getarg1("container", cons, locals, &obj);
+        _du_getargs1("container", cons, locals, &obj);
 
     return DuContainer_New(obj);
 }
@@ -315,45 +309,58 @@
 
 DuObject *du_set(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
         Du_FatalError("set: expected at least two arguments");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_save2(next, locals);
+    DuObject *obj = Du_Eval(expr, locals);
+    _du_restore2(next, locals);
+
+    _du_read1(next);
+    DuObject *expr2 = _DuCons_CAR(next);
+    DuObject *next2 = _DuCons_NEXT(next);
 
     if (DuList_Check(obj)) {
-        if (_DuCons_NEXT(_DuCons_NEXT(cons)) == Du_None ||
-            _DuCons_NEXT(_DuCons_NEXT(_DuCons_NEXT(cons))) != Du_None)
+        _du_read1(next2);
+        if (next2 == Du_None || _DuCons_NEXT(next2) != Du_None)
             Du_FatalError("set with a list: expected three arguments");
-        DuObject *index = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
-        DuObject *newobj = Du_Eval(
-                    _DuCons_CAR(_DuCons_NEXT(_DuCons_NEXT(cons))), locals);
+
+        _du_save3(obj, next2, locals);
+        DuObject *index = Du_Eval(expr2, locals);
+        _du_restore3(obj, next2, locals);
+
+        _du_save2(obj, index);
+        DuObject *newobj = Du_Eval(_DuCons_CAR(next2), locals);
+        _du_restore2(obj, index);
+
         DuList_SetItem(obj, DuInt_AsInt(index), newobj);
-        Du_DECREF(index);
-        Du_DECREF(newobj);
     }
     else if (DuContainer_Check(obj)) {
-        if (_DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
+        if (next2 != Du_None)
             Du_FatalError("set with a container: expected two arguments");
-        DuObject *newobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+
+        _du_save1(obj);
+        DuObject *newobj = Du_Eval(expr2, locals);
+        _du_restore1(obj);
+
         DuContainer_SetRef(obj, newobj);
-        Du_DECREF(newobj);
     }
     else
-        Du_FatalError("set: bad argument type '%s'", obj->ob_type->dt_name);
+        Du_FatalError("set: bad argument type '%s'", Du_TYPE(obj)->dt_name);
 
-    Du_DECREF(obj);
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
 DuObject *du_append(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) == Du_None ||
-        _DuCons_NEXT(_DuCons_NEXT(cons)) != Du_None)
-        Du_FatalError("append: expected two arguments");
-    DuObject *lst = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *newobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    DuObject *lst, *newobj;
+    _du_getargs2("append", cons, locals, &lst, &newobj);
+
     DuList_Append(lst, newobj);
-    Du_DECREF(lst);
     return newobj;
 }
 
@@ -361,50 +368,69 @@
 {
     if (cons == Du_None)
         Du_FatalError("pop: expected at least one argument");
-    DuObject *lst = Du_Eval(_DuCons_CAR(cons), locals);
+
+    _du_read1(cons);
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_save2(next, locals);
+    DuObject *lst = Du_Eval(expr, locals);
+    _du_restore2(next, locals);
+
     int index;
-    if (_DuCons_NEXT(cons) == Du_None) {
+    if (next == Du_None) {
         index = DuList_Size(lst) - 1;
         if (index < 0)
             Du_FatalError("pop: empty list");
     }
-    else if (_DuCons_NEXT(_DuCons_NEXT(cons)) == Du_None) {
-        DuObject *indexobj = Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+    else {
+        _du_read1(next);
+        DuObject *expr2 = _DuCons_CAR(next);
+        DuObject *next2 = _DuCons_NEXT(next);
+
+        if (next2 != Du_None)
+            Du_FatalError("pop: expected at most two arguments");
+
+        _du_save1(lst);
+        DuObject *indexobj = Du_Eval(expr2, locals);
+        _du_restore1(lst);
+
         index = DuInt_AsInt(indexobj);
-        Du_DECREF(indexobj);
     }
-    else
-        Du_FatalError("pop: expected at most two arguments");
 
-    DuObject *res = DuList_Pop(lst, index);
-    Du_DECREF(lst);
-    return res;
+    return DuList_Pop(lst, index);
 }
 
 DuObject *du_len(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("len: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("len", cons, locals, &obj);
+
     int length = DuObject_Length(obj);
-    Du_DECREF(obj);
     return DuInt_FromInt(length);
 }
 
 DuObject *du_if(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
         Du_FatalError("if: expected at least two arguments");
-    DuObject *cond = Du_Eval(_DuCons_CAR(cons), locals);
-    int cond_int = DuObject_IsTrue(cond);
-    Du_DECREF(cond);
-    if (cond_int != 0) {
+
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_save2(next, locals);
+    DuObject *cond = Du_Eval(expr, locals);
+    _du_restore2(next, locals);
+
+    _du_read1(next);
+    if (DuObject_IsTrue(cond) != 0) {
         /* true path */
-        return Du_Eval(_DuCons_CAR(_DuCons_NEXT(cons)), locals);
+        return Du_Eval(_DuCons_CAR(next), locals);
     }
     else {
         /* false path */
-        return Du_Progn(_DuCons_NEXT(_DuCons_NEXT(cons)), locals);
+        return Du_Progn(_DuCons_NEXT(next), locals);
     }
 }
 
@@ -412,63 +438,73 @@
 {
     if (cons == Du_None)
         Du_FatalError("while: expected at least one argument");
+
+    _du_read1(cons);
+    DuObject *expr = _DuCons_CAR(cons);
+    DuObject *next = _DuCons_NEXT(cons);
+
     while (1) {
-        DuObject *cond = Du_Eval(_DuCons_CAR(cons), locals);
-        int cond_int = DuObject_IsTrue(cond);
-        Du_DECREF(cond);
-        if (cond_int == 0)
+        _du_save3(expr, next, locals);
+        DuObject *cond = Du_Eval(expr, locals);
+        _du_restore3(expr, next, locals);
+
+        if (!DuObject_IsTrue(cond))
             break;
-        DuObject *res = Du_Progn(_DuCons_NEXT(cons), locals);
-        Du_DECREF(res);
+
+        _du_save3(expr, next, locals);
+        Du_Progn(next, locals);
+        _du_restore3(expr, next, locals);
     }
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
 DuObject *du_defun(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) == Du_None)
         Du_FatalError("defun: expected at least two arguments");
+
     DuObject *name = _DuCons_CAR(cons);
-    DuObject *arglist = _DuCons_CAR(_DuCons_NEXT(cons));
-    DuObject *progn = _DuCons_NEXT(_DuCons_NEXT(cons));
+    DuObject *next = _DuCons_NEXT(cons);
+
+    _du_read1(next);
+    DuObject *arglist = _DuCons_CAR(next);
+    DuObject *progn = _DuCons_NEXT(next);
+
     DuFrame_SetUserFunction(locals, name, arglist, progn);
-    Du_INCREF(Du_None);
+
     return Du_None;
 }
 
 DuObject *du_car(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("car: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *res = DuCons_Car(obj);
-    Du_DECREF(obj);
-    return res;
+    DuObject *obj;
+    _du_getargs1("car", cons, locals, &obj);
+
+    return DuCons_Car(obj);
 }
 
 DuObject *du_cdr(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("car: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
-    DuObject *res = DuCons_Cdr(obj);
-    Du_DECREF(obj);
-    return res;
+    DuObject *obj;
+    _du_getargs1("cdr", cons, locals, &obj);
+
+    return DuCons_Cdr(obj);
 }
 
 DuObject *du_not(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("not: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("not", cons, locals, &obj);
+
     int res = !DuObject_IsTrue(obj);
-    Du_DECREF(obj);
     return DuInt_FromInt(res);
 }
 
 DuObject *du_transaction(DuObject *cons, DuObject *locals)
 {
+    Du_FatalError("transaction: not implemented");
+#if 0
     if (cons == Du_None)
         Du_FatalError("transaction: expected at least one argument");
     DuObject *sym = _DuCons_CAR(cons);
@@ -476,61 +512,57 @@
     _DuFrame_EvalCall(locals, sym, rest, 0);
     Du_INCREF(Du_None);
     return Du_None;
+#endif
 }
 
 DuObject *du_sleepms(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("sleepms: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("sleepms", cons, locals, &obj);
+
     int ms = DuInt_AsInt(obj);
-    Du_DECREF(obj);
 
     struct timeval t;
     t.tv_sec = ms / 1000;
     t.tv_usec = (ms % 1000) * 1000;
     select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &t);
 
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
 DuObject *du_defined(DuObject *cons, DuObject *locals)
 {
+    _du_read1(cons);
     if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
         Du_FatalError("defined?: expected one argument");
     DuObject *ob = _DuCons_CAR(cons);
+
+    _du_save1(ob);
     DuObject *res = DuFrame_GetSymbol(locals, ob);
+    _du_restore1(ob);
+
     if (res == NULL)
-        DuFrame_GetSymbol(Du_Globals, ob);
-    if (res != NULL)
-        Du_DECREF(res);
+        res = DuFrame_GetSymbol(Du_Globals, ob);
+
     return DuInt_FromInt(res != NULL);
 }
 
 DuObject *du_assert(DuObject *cons, DuObject *locals)
 {
-    if (cons == Du_None || _DuCons_NEXT(cons) != Du_None)
-        Du_FatalError("defined?: expected one argument");
-    DuObject *obj = Du_Eval(_DuCons_CAR(cons), locals);
+    DuObject *obj;
+    _du_getargs1("assert", cons, locals, &obj);
+
     if (!DuInt_AsInt(obj)) {
         printf("assert failed: ");
+        _du_read1(cons);
         Du_Print(_DuCons_CAR(cons), 1);
         Du_FatalError("assert failed");
     }
-    Du_DECREF(obj);
-    Du_INCREF(Du_None);
     return Du_None;
 }
 
-DuObject *Du_Globals;
-
 void Du_Initialize(void)
 {
-    _Du_AME_InitThreadDescriptor();
-    _Du_InitializeObjects();
-
-    Du_Globals = DuFrame_New();
     DuFrame_SetBuiltinMacro(Du_Globals, "progn", Du_Progn);
     DuFrame_SetBuiltinMacro(Du_Globals, "setq", du_setq);
     DuFrame_SetBuiltinMacro(Du_Globals, "print", du_print);
@@ -567,9 +599,4 @@
 
 void Du_Finalize(void)
 {
-    Du_DECREF(Du_Globals);
-    Du_Globals = NULL;
-
-    _Du_FinalizeObjects();
-    _Du_AME_FiniThreadDescriptor();
 }
diff --git a/duhton/listobject.c b/duhton/listobject.c
--- a/duhton/listobject.c
+++ b/duhton/listobject.c
@@ -21,12 +21,14 @@
 {
     int i;
     _du_read1(ob);
-    if (ob->ob_count == 0) {
+
+    DuTupleObject *p = ob->ob_tuple;
+    _du_read1(p);
+
+    if (p->ob_count == 0) {
         printf("[]");
     }
     else {
-        DuTupleObject *p = ob->ob_tuple;
-        _du_read1(p);
         printf("[ ");
         for (i=0; i<p->ob_count; i++) {
             Du_Print(p->ob_items[i], 0);
@@ -63,8 +65,8 @@
     DuTupleObject *newitems = DuTuple_New(newcount);
 
     for (i=0; i<newcount-1; i++)
-        newitems[i] = olditems[i];
-    newitems[newcount-1] = x;
+        newitems->ob_items[i] = olditems->ob_items[i];
+    newitems->ob_items[newcount-1] = x;
 
     ob->ob_tuple = newitems;
 }
@@ -124,6 +126,7 @@
     if (index < 0 || index >= p->ob_count)
         Du_FatalError("list_pop: index out of range");
     DuObject *result = p->ob_items[index];
+    int i;
     p->ob_count--;
     for (i=index; i<p->ob_count; i++)
         p->ob_items[i] = p->ob_items[i+1];
@@ -149,7 +152,7 @@
 static DuTupleObject du_empty_tuple = {
     DuOBJECT_HEAD_INIT(DUTYPE_TUPLE),
     0,
-}
+};
 
 DuObject *DuList_New()
 {
diff --git a/duhton/symbol.c b/duhton/symbol.c
--- a/duhton/symbol.c
+++ b/duhton/symbol.c
@@ -43,7 +43,7 @@
     (eval_fn)symbol_eval,
 };
 
-DuObject *DuSymbol_FromString(char *name)
+DuObject *DuSymbol_FromString(const char *name)
 {
     DuSymbolObject *p, *head = &_Du_AllSymbols;
     for (p=head; p != NULL; p=p->next) {
diff --git a/duhton/transaction.c b/duhton/transaction.c
--- a/duhton/transaction.c
+++ b/duhton/transaction.c
@@ -1,7 +1,7 @@
 #include "duhton.h"
 
 
-void Du_TransactionAdd(DuObject *frame)
+void Du_TransactionAdd(DuObject *code, DuObject *frame)
 {
     /* XXX */
 }

From noreply at buildbot.pypy.org  Tue Jun 25 15:32:23 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Jun 2013 15:32:23 +0200 (CEST)
Subject: [pypy-commit] stmgc default: stm_id cleanups and fixes
Message-ID: <20130625133223.DBC541C0175@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r275:d19d08d186b1
Date: 2013-06-25 15:31 +0200
http://bitbucket.org/pypy/stmgc/changeset/d19d08d186b1/

Log:	stm_id cleanups and fixes

diff --git a/c4/atomic_ops.h b/c4/atomic_ops.h
--- a/c4/atomic_ops.h
+++ b/c4/atomic_ops.h
@@ -2,7 +2,7 @@
 #define _SRCSTM_ATOMIC_OPS_
 
 #include <assert.h>
-
+#define IMPLIES(a, b)   (!(a) || (b))
 
 /* Ask the compiler to really reload the revision_t argument from memory.
    That's all that this macro does; it does not imply any type of barrier.
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1031,8 +1031,9 @@
       stub->h_tid = (L->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
                                                    | GCFLAG_STUB
                                                    | GCFLAG_OLD;
+      stub->h_revision = ((revision_t)L) | 2;
+
       assert(!(L->h_tid & GCFLAG_HAS_ID));
-      stub->h_revision = ((revision_t)L) | 2;
       if (L->h_original) {
         stub->h_original = L->h_original;
       }
@@ -1040,8 +1041,14 @@
         stub->h_original = (revision_t)L;
       }
       else {
-        L->h_original = (revision_t)stub;
+        /* There shouldn't be a public, young object without
+           a h_original. They only come from stealing which
+           always sets h_original */
         assert(0);
+        /* L->h_original = (revision_t)stub; */
+        /* if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) { */
+        /*   ((gcptr)L->h_revision)->h_original = (revision_t)stub; */
+        /* } */
       }
       
       item->val = stub;
@@ -1165,12 +1172,6 @@
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
-          assert(!(P->h_original) 
-                 || (B->h_original == (revision_t)P->h_original));
-          assert(!(P->h_original && !B->h_original));
-          if (P->h_original && !B->h_original) // might occur because of
-            B->h_original = P->h_original; //replace_ptr_to_protected_with_stub
-
           size_t size = stmcb_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -162,7 +162,7 @@
     else if (p->h_tid & GCFLAG_OLD) {
         /* old objects must have an h_original xOR be
            the original itself. */
-        /* dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p)); */
+        dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p));
         return (revision_t)p;
     }
     
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -53,7 +53,13 @@
         stub->h_original = (revision_t)obj;
     }
     else {
-        obj->h_original = (revision_t)stub;
+        /* There shouldn't be a public, young object without
+           a h_original. But there can be protected ones. */
+        assert(!(obj->h_tid & GCFLAG_PUBLIC));
+        obj->h_original = (revision_t)stub;        
+        if (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+            ((gcptr)obj->h_revision)->h_original = (revision_t)stub;
+        }
     }
 
     g2l_insert(&sd->all_stubs, obj, stub);
@@ -93,23 +99,23 @@
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
         
-        if (L->h_original) {
-            /* L has an original, may be GCFLAG_HAS_ID */
-            B->h_original = L->h_original;
-            L->h_tid &= ~GCFLAG_HAS_ID;
-        }
-        else if (L->h_tid & GCFLAG_OLD) {
-            /* If old, it must be the original */
-            assert(!(L->h_tid & GCFLAG_HAS_ID));
-            /* original must be L */
+        /* On young objects here, h_original is always set
+         and never GCFLAG_HAS_ID. This is because a stealing
+         thread can only reach a priv_from_prot object through
+         public old stubs/objects that serve as originals if
+         needed.
+         If h_original is set, then it is already set in the
+         backup, too.
+        */
+        assert(!(L->h_tid & GCFLAG_HAS_ID));
+        assert(IMPLIES(!(L->h_tid & GCFLAG_OLD), L->h_original));
+        assert(IMPLIES(L->h_tid & GCFLAG_OLD,
+                       (B->h_original == (revision_t)L) 
+                       || (B->h_original == L->h_original)));
+        if (!L->h_original && L->h_tid & GCFLAG_OLD) {
+            /* If old, L must be the original */
             B->h_original = (revision_t)L;
         }
-        else {
-            /* we can make the backup the "original"
-             since L hasn't decided yet */
-            L->h_original = (revision_t)B;
-            assert(0);
-        }
 
         /* B is now a backup copy, i.e. a protected object, and we own
            the foreign thread's collection_lock, so we can read/write the
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -539,6 +539,7 @@
             assert not (p1.h_tid & GCFLAG_HAS_ID)
             
         r1w = lib.stm_write_barrier(r1)
+        lib.setptr(p1, 1, r1w)
         assert classify(r1w) == "private_from_protected"
         assert not (r1w.h_tid & GCFLAG_OLD)
         if c:

From noreply at buildbot.pypy.org  Tue Jun 25 15:55:58 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 25 Jun 2013 15:55:58 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: removed the rsqueakvm-nojit
 benchmark because it takes too much time on the server.
Message-ID: <20130625135558.9789C1C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r474:ae6f205a03e4
Date: 2013-06-25 15:55 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/ae6f205a03e4/

Log:	removed the rsqueakvm-nojit benchmark because it takes too much time
	on the server.

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -221,7 +221,7 @@
     "lang-smalltalk",
     executables=[
         Executable("rsqueakvm", "./targetimageloadingsmalltalk-c"),
-        Executable("rsqueakvm-nojit", "./targetimageloadingsmalltalk-nojit-c")
+        # Executable("rsqueakvm-nojit", "./targetimageloadingsmalltalk-nojit-c")
     ],
     arguments=["images/%s.image" % SqueakImage, '-m', 'runSPyBenchmarks']
 )

From noreply at buildbot.pypy.org  Tue Jun 25 16:43:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 16:43:39 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Progress
Message-ID: <20130625144339.0EE261C3462@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r277:5252837946d1
Date: 2013-06-25 16:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/5252837946d1/

Log:	Progress

diff --git a/duhton/frame.c b/duhton/frame.c
--- a/duhton/frame.c
+++ b/duhton/frame.c
@@ -34,9 +34,14 @@
     DuFrameNodeObject *ob_nodes;
 } DuFrameObject;
 
+static DuFrameNodeObject du_empty_framenode = {
+    DuOBJECT_HEAD_INIT(DUTYPE_FRAMENODE),
+    0,
+};
+
 DuFrameObject Du_GlobalsFrame = {
     DuOBJECT_HEAD_INIT(DUTYPE_FRAME),
-    NULL,
+    &du_empty_framenode,
 };
 
 DuObject *_Du_GetGlobals()
@@ -47,7 +52,7 @@
 DuObject *DuFrame_New()
 {
     DuFrameObject *ob = (DuFrameObject *)DuObject_New(&DuFrame_Type);
-    ob->ob_nodes = NULL;
+    ob->ob_nodes = &du_empty_framenode;
     return (DuObject *)ob;
 }
 
@@ -132,6 +137,7 @@
             newentries[i] = entries[i];
 
         DuSymbol_Ensure("find_entry", symbol);
+        newentries[left].symbol_id = search_id;
         newentries[left].symbol = symbol;
         newentries[left].value = NULL;
         newentries[left].builtin_macro = NULL;
@@ -164,30 +170,40 @@
 _parse_arguments(DuObject *symbol, DuObject *arguments,
                  DuObject *formallist, DuObject *caller, DuObject *callee)
 {
-    abort();
-#if 0
     while (DuCons_Check(formallist)) {
         if (!DuCons_Check(arguments))
             Du_FatalError("call to '%s': not enough arguments",
                           DuSymbol_AsString(symbol));
+
+        _du_read1(arguments);
+        DuObject *arg = _DuCons_CAR(arguments);
+        DuObject *argumentsnext = _DuCons_NEXT(arguments);
+
+        _du_save3(symbol, argumentsnext, caller);
+        _du_save2(formallist, callee);
+        DuObject *obj = Du_Eval(arg, caller);
+        _du_restore2(formallist, callee);
+
+        _du_read1(formallist);
         DuObject *sym = _DuCons_CAR(formallist);
-        DuObject *obj = Du_Eval(_DuCons_CAR(arguments), caller);
+        DuObject *formallistnext = _DuCons_NEXT(formallist);
+
+        _du_save2(formallistnext, callee);
         DuFrame_SetSymbol(callee, sym, obj);
-        Du_DECREF(obj);
-        formallist = _DuCons_NEXT(formallist);
-        arguments = _DuCons_NEXT(arguments);
+        _du_restore2(formallistnext, callee);
+        _du_restore3(symbol, argumentsnext, caller);
+
+        formallist = formallistnext;
+        arguments = argumentsnext;
     }
     if (arguments != Du_None)
         Du_FatalError("call to '%s': too many arguments",
                       DuSymbol_AsString(symbol));
-#endif
 }
 
 DuObject *_DuFrame_EvalCall(DuObject *frame, DuObject *symbol,
                             DuObject *rest, int execute_now)
 {
-    stm_fatalerror("_DuFrame_EvalCall\n");
-#if 0
     struct dictentry *e;
     DuFrame_Ensure("_DuFrame_EvalCall", frame);
 
@@ -205,18 +221,24 @@
         }
     }
     if (e->func_progn) {
+        DuObject *func = e->func_progn;
+        _du_save1(func);
+        _du_save3(frame, symbol, rest);
         DuObject *callee_frame = DuFrame_New();
-        DuObject *res;
+        _du_restore3(frame, symbol, rest);
+
+        _du_save1(callee_frame);
         _parse_arguments(symbol, rest, e->func_arglist, frame, callee_frame);
+        _du_restore1(callee_frame);
+        _du_restore1(func);
+
         if (execute_now) {
-            res = Du_Progn(e->func_progn, callee_frame);
+            return Du_Progn(func, callee_frame);
         }
         else {
-            Du_TransactionAdd(e->func_progn, callee_frame);
-            res = NULL;
+            Du_TransactionAdd(func, callee_frame);
+            return NULL;
         }
-        Du_DECREF(callee_frame);
-        return res;
     }
     if (e->builtin_macro) {
         if (!execute_now)
@@ -227,63 +249,50 @@
  not_defined:
     Du_FatalError("symbol not defined as a function: '%s'",
                   DuSymbol_AsString(symbol));
-#endif
 }
 
 DuObject *DuFrame_GetSymbol(DuObject *frame, DuObject *symbol)
 {
-    stm_fatalerror("DuFrame_GetSymbol\n");
-#if 0
     struct dictentry *e;
     DuFrame_Ensure("DuFrame_GetSymbol", frame);
 
     e = find_entry((DuFrameObject *)frame, symbol, 0);
-    if (e && e->value) {
-        Du_INCREF(e->value);
-        return e->value;
-    }
-    else
-        return NULL;
-#endif
+    return e ? e->value : NULL;
 }
 
 void DuFrame_SetSymbol(DuObject *frame, DuObject *symbol, DuObject *value)
 {
-    stm_fatalerror("DuFrame_SetSymbol\n");
-#if 0
     struct dictentry *e;
     DuFrame_Ensure("DuFrame_SetSymbol", frame);
 
+    _du_save1(value);
     e = find_entry((DuFrameObject *)frame, symbol, 1);
-    if (e->value) Du_DECREF(e->value);
-    e->value = value; Du_INCREF(value);
-#endif
+    _du_restore1(value);
+
+    e->value = value;
 }
 
 void DuFrame_SetSymbolStr(DuObject *frame, char *name, DuObject *value)
 {
-    stm_fatalerror("DuFrame_SetSymbolStr\n");
-#if 0
+    _du_save2(frame, value);
     DuObject *sym = DuSymbol_FromString(name);
+    _du_restore2(frame, value);
+
     DuFrame_SetSymbol(frame, sym, value);
-    Du_DECREF(sym);
-#endif
 }
 
 void DuFrame_SetUserFunction(DuObject *frame, DuObject *symbol,
                              DuObject *arglist, DuObject *progn)
 {
-    stm_fatalerror("DuFrame_SetUserFunction\n");
-#if 0
     struct dictentry *e;
     DuFrame_Ensure("DuFrame_SetUserFunction", frame);
 
+    _du_save2(arglist, progn);
     e = find_entry((DuFrameObject *)frame, symbol, 1);
-    if (e->func_arglist) Du_DECREF(e->func_arglist);
-    if (e->func_progn)   Du_DECREF(e->func_progn);
-    e->func_arglist = arglist; Du_INCREF(arglist);
-    e->func_progn   = progn;   Du_INCREF(progn);
-#endif
+    _du_restore2(arglist, progn);
+
+    e->func_arglist = arglist;
+    e->func_progn = progn;
 }
 
 void DuFrame_Ensure(char *where, DuObject *ob)
diff --git a/duhton/object.c b/duhton/object.c
--- a/duhton/object.c
+++ b/duhton/object.c
@@ -71,7 +71,7 @@
     vfprintf(stderr, msg, ap);
     fprintf(stderr, "\n");
     va_end(ap);
-    exit(1);
+    abort();
 }
 
 void Du_Print(DuObject *ob, int newline)

From noreply at buildbot.pypy.org  Tue Jun 25 16:43:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 16:43:40 +0200 (CEST)
Subject: [pypy-commit] stmgc default: merge heads
Message-ID: <20130625144340.0B3E01C346B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r278:9935b830ffc8
Date: 2013-06-25 16:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/9935b830ffc8/

Log:	merge heads

diff --git a/c4/atomic_ops.h b/c4/atomic_ops.h
--- a/c4/atomic_ops.h
+++ b/c4/atomic_ops.h
@@ -2,7 +2,7 @@
 #define _SRCSTM_ATOMIC_OPS_
 
 #include <assert.h>
-
+#define IMPLIES(a, b)   (!(a) || (b))
 
 /* Ask the compiler to really reload the revision_t argument from memory.
    That's all that this macro does; it does not imply any type of barrier.
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1031,8 +1031,9 @@
       stub->h_tid = (L->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
                                                    | GCFLAG_STUB
                                                    | GCFLAG_OLD;
+      stub->h_revision = ((revision_t)L) | 2;
+
       assert(!(L->h_tid & GCFLAG_HAS_ID));
-      stub->h_revision = ((revision_t)L) | 2;
       if (L->h_original) {
         stub->h_original = L->h_original;
       }
@@ -1040,8 +1041,14 @@
         stub->h_original = (revision_t)L;
       }
       else {
-        L->h_original = (revision_t)stub;
+        /* There shouldn't be a public, young object without
+           a h_original. They only come from stealing which
+           always sets h_original */
         assert(0);
+        /* L->h_original = (revision_t)stub; */
+        /* if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) { */
+        /*   ((gcptr)L->h_revision)->h_original = (revision_t)stub; */
+        /* } */
       }
       
       item->val = stub;
@@ -1165,12 +1172,6 @@
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
-          assert(!(P->h_original) 
-                 || (B->h_original == (revision_t)P->h_original));
-          assert(!(P->h_original && !B->h_original));
-          if (P->h_original && !B->h_original) // might occur because of
-            B->h_original = P->h_original; //replace_ptr_to_protected_with_stub
-
           size_t size = stmcb_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -162,7 +162,7 @@
     else if (p->h_tid & GCFLAG_OLD) {
         /* old objects must have an h_original xOR be
            the original itself. */
-        /* dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p)); */
+        dprintf(("stm_id(%p) is old, orig=0 fst: %p\n", p, p));
         return (revision_t)p;
     }
     
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -53,7 +53,13 @@
         stub->h_original = (revision_t)obj;
     }
     else {
-        obj->h_original = (revision_t)stub;
+        /* There shouldn't be a public, young object without
+           a h_original. But there can be protected ones. */
+        assert(!(obj->h_tid & GCFLAG_PUBLIC));
+        obj->h_original = (revision_t)stub;        
+        if (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
+            ((gcptr)obj->h_revision)->h_original = (revision_t)stub;
+        }
     }
 
     g2l_insert(&sd->all_stubs, obj, stub);
@@ -93,23 +99,23 @@
     if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) {
         gcptr B = (gcptr)L->h_revision;     /* the backup copy */
         
-        if (L->h_original) {
-            /* L has an original, may be GCFLAG_HAS_ID */
-            B->h_original = L->h_original;
-            L->h_tid &= ~GCFLAG_HAS_ID;
-        }
-        else if (L->h_tid & GCFLAG_OLD) {
-            /* If old, it must be the original */
-            assert(!(L->h_tid & GCFLAG_HAS_ID));
-            /* original must be L */
+        /* On young objects here, h_original is always set
+         and never GCFLAG_HAS_ID. This is because a stealing
+         thread can only reach a priv_from_prot object through
+         public old stubs/objects that serve as originals if
+         needed.
+         If h_original is set, then it is already set in the
+         backup, too.
+        */
+        assert(!(L->h_tid & GCFLAG_HAS_ID));
+        assert(IMPLIES(!(L->h_tid & GCFLAG_OLD), L->h_original));
+        assert(IMPLIES(L->h_tid & GCFLAG_OLD,
+                       (B->h_original == (revision_t)L) 
+                       || (B->h_original == L->h_original)));
+        if (!L->h_original && L->h_tid & GCFLAG_OLD) {
+            /* If old, L must be the original */
             B->h_original = (revision_t)L;
         }
-        else {
-            /* we can make the backup the "original"
-             since L hasn't decided yet */
-            L->h_original = (revision_t)B;
-            assert(0);
-        }
 
         /* B is now a backup copy, i.e. a protected object, and we own
            the foreign thread's collection_lock, so we can read/write the
diff --git a/c4/test/test_et.py b/c4/test/test_et.py
--- a/c4/test/test_et.py
+++ b/c4/test/test_et.py
@@ -539,6 +539,7 @@
             assert not (p1.h_tid & GCFLAG_HAS_ID)
             
         r1w = lib.stm_write_barrier(r1)
+        lib.setptr(p1, 1, r1w)
         assert classify(r1w) == "private_from_protected"
         assert not (r1w.h_tid & GCFLAG_OLD)
         if c:

From noreply at buildbot.pypy.org  Tue Jun 25 16:43:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 16:43:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: in-progress
Message-ID: <20130625144338.04AE81C345E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r276:25825a02e0f9
Date: 2013-06-25 16:21 +0200
http://bitbucket.org/pypy/stmgc/changeset/25825a02e0f9/

Log:	in-progress

diff --git a/duhton/Makefile b/duhton/Makefile
--- a/duhton/Makefile
+++ b/duhton/Makefile
@@ -2,10 +2,10 @@
 all: duhton_debug duhton
 
 duhton: *.c *.h ../c4/*.c ../c4/*.h
-	gcc -pthread -g -O2 -o duhton *.c ../c4/stmgc.c -Wall
+	gcc -lrt -pthread -g -O2 -o duhton *.c ../c4/stmgc.c -Wall
 
 duhton_debug: *.c *.h ../c4/*.c ../c4/*.h
-	gcc -pthread -g -DDu_DEBUG -o duhton_debug *.c ../c4/stmgc.c -Wall
+	gcc -lrt -pthread -g -DDu_DEBUG -o duhton_debug *.c ../c4/stmgc.c -Wall
 
 clean:
 	rm -f duhton duhton_debug
diff --git a/duhton/consobject.c b/duhton/consobject.c
--- a/duhton/consobject.c
+++ b/duhton/consobject.c
@@ -5,6 +5,13 @@
     DuObject *car, *cdr;
 } DuConsObject;
 
+
+void cons_trace(DuConsObject *ob, void visit(gcptr *))
+{
+    visit(&ob->car);
+    visit(&ob->cdr);
+}
+
 void cons_print(DuConsObject *ob)
 {
     DuObject *p;
@@ -39,6 +46,7 @@
     "cons",
     DUTYPE_CONS,
     sizeof(DuConsObject),
+    (trace_fn)cons_trace,
     (print_fn)cons_print,
     (eval_fn)cons_eval,
 };
diff --git a/duhton/containerobject.c b/duhton/containerobject.c
--- a/duhton/containerobject.c
+++ b/duhton/containerobject.c
@@ -5,6 +5,12 @@
     DuObject *ob_reference;
 } DuContainerObject;
 
+
+void container_trace(DuContainerObject *ob, void visit(gcptr *))
+{
+    visit(&ob->ob_reference);
+}
+
 void container_print(DuContainerObject *ob)
 {
     printf("<container ");
@@ -32,6 +38,7 @@
     "container",
     DUTYPE_CONTAINER,
     sizeof(DuContainerObject),
+    (trace_fn)container_trace,
     (print_fn)container_print,
 };
 
diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -2,6 +2,7 @@
 #define _DUHTON_H_
 
 #include "../c4/stmgc.h"
+#include "../c4/fprintcolor.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
@@ -21,6 +22,7 @@
 #endif
 
 
+typedef void(*trace_fn)(DuObject *, void visit(gcptr *));
 typedef void(*print_fn)(DuObject *);
 typedef DuObject *(*eval_fn)(DuObject *, DuObject *);
 typedef int(*len_fn)(DuObject *);
@@ -29,6 +31,7 @@
     const char *dt_name;
     int dt_typeindex;
     int dt_size;
+    trace_fn dt_trace;
     print_fn dt_print;
     eval_fn dt_eval;
     len_fn dt_is_true;
@@ -43,8 +46,9 @@
 #define DUTYPE_LIST          5
 #define DUTYPE_TUPLE         6
 #define DUTYPE_FRAME         7
-#define DUTYPE_CONTAINER     8
-#define _DUTYPE_TOTAL        9
+#define DUTYPE_FRAMENODE     8
+#define DUTYPE_CONTAINER     9
+#define _DUTYPE_TOTAL       10
 
 extern DuType DuNone_Type;
 extern DuType DuInt_Type;
@@ -92,6 +96,7 @@
 DuObject *DuList_GetItem(DuObject *list, int index);
 void DuList_SetItem(DuObject *list, int index, DuObject *newobj);
 DuObject *DuList_Pop(DuObject *list, int index);
+size_t _DuTuple_ByteSize(DuObject *tuple);
 
 DuObject *DuContainer_New(DuObject *obj);
 DuObject *DuContainer_GetRef(DuObject *container);
diff --git a/duhton/frame.c b/duhton/frame.c
--- a/duhton/frame.c
+++ b/duhton/frame.c
@@ -2,6 +2,7 @@
 #include <stdint.h>
 
 struct dictentry {
+    revision_t symbol_id;
     DuObject *symbol;
     DuObject *value;
     eval_fn builtin_macro;
@@ -11,13 +12,30 @@
 
 typedef struct {
     DuOBJECT_HEAD
-    int entry_count;
-    struct dictentry *entries;
+    int ob_count;
+    struct dictentry ob_items[1];
+} DuFrameNodeObject;
+
+void framenode_trace(DuFrameNodeObject *ob, void visit(gcptr *))
+{
+    int i;
+    for (i=ob->ob_count-1; i>=0; i--) {
+        struct dictentry *e = &ob->ob_items[i];
+        visit(&e->symbol);
+        visit(&e->value);
+        visit(&e->func_arglist);
+        visit(&e->func_progn);
+    }
+}
+
+
+typedef struct {
+    DuOBJECT_HEAD
+    DuFrameNodeObject *ob_nodes;
 } DuFrameObject;
 
 DuFrameObject Du_GlobalsFrame = {
     DuOBJECT_HEAD_INIT(DUTYPE_FRAME),
-    0,
     NULL,
 };
 
@@ -29,11 +47,11 @@
 DuObject *DuFrame_New()
 {
     DuFrameObject *ob = (DuFrameObject *)DuObject_New(&DuFrame_Type);
-    ob->entry_count = 0;
-    ob->entries = NULL;
+    ob->ob_nodes = NULL;
     return (DuObject *)ob;
 }
 
+#if 0
 DuObject *DuFrame_Copy(DuObject *frame)
 {
     DuFrame_Ensure("DuFrame_Copy", frame);
@@ -53,19 +71,11 @@
     }
     return (DuObject *)dst;
 }
+#endif
 
-void frame_free(DuFrameObject *ob)
+void frame_trace(DuFrameObject *ob, void visit(gcptr *))
 {
-    int i;
-    for (i=0; i<ob->entry_count; i++) {
-        struct dictentry *e = &ob->entries[i];
-        Du_DECREF(e->symbol);
-        if (e->value        != NULL) Du_DECREF(e->value       );
-        if (e->func_arglist != NULL) Du_DECREF(e->func_arglist);
-        if (e->func_progn   != NULL) Du_DECREF(e->func_progn  );
-    }
-    free(ob->entries);
-    free(ob);
+    visit((gcptr *)&ob->ob_nodes);
 }
 
 void frame_print(DuFrameObject *ob)
@@ -74,58 +84,88 @@
 }
 
 static struct dictentry *
-find_entry(DuFrameObject *frame, DuObject *symbol, int add_if_missing)
+find_entry(DuFrameObject *frame, DuObject *symbol, int write_mode)
 {
+    _du_read1(frame);
+    DuFrameNodeObject *ob = frame->ob_nodes;
+
+    _du_read1(ob);
     int left = 0;
-    int right = frame->entry_count;
-    struct dictentry *entries = frame->entries;
+    int right = ob->ob_count;
+    struct dictentry *entries = ob->ob_items;
+    revision_t search_id = stm_id(symbol);
+
     while (right > left) {
         int middle = (left + right) / 2;
-        DuObject *found = entries[middle].symbol;
-        if ((intptr_t)found < (intptr_t)symbol)
+        revision_t found_id = entries[middle].symbol_id;
+        if (search_id < found_id)
             right = middle;
-        else if (found == symbol)
+        else if (search_id == found_id) {
+            if (write_mode) {
+                _du_write1(ob);
+                entries = ob->ob_items;
+            }
             return entries + middle;
+        }
         else
             left = middle + 1;
     }
-    if (add_if_missing) {
+
+    if (!write_mode) {
+        return NULL;
+    }
+    else {
         int i;
-        int newcount = frame->entry_count + 1;
-        struct dictentry *newentries = malloc(sizeof(struct dictentry) *
-                                              newcount);
+        size_t size = (sizeof(DuFrameNodeObject) +
+                       (ob->ob_count + 1 - 1)*sizeof(struct dictentry));
+        DuFrameNodeObject *newob;
+
+        _du_save3(ob, symbol, frame);
+        newob = (DuFrameNodeObject *)stm_allocate(size, DUTYPE_FRAMENODE);
+        _du_restore3(ob, symbol, frame);
+
+        newob->ob_count = ob->ob_count + 1;
+        struct dictentry *newentries = newob->ob_items;
+        entries = ob->ob_items;
+
         for (i=0; i<left; i++)
             newentries[i] = entries[i];
+
         DuSymbol_Ensure("find_entry", symbol);
-        newentries[left].symbol = symbol; Du_INCREF(symbol);
+        newentries[left].symbol = symbol;
         newentries[left].value = NULL;
         newentries[left].builtin_macro = NULL;
         newentries[left].func_arglist = NULL;
         newentries[left].func_progn = NULL;
-        for (i=left+1; i<newcount; i++)
+
+        for (i=left+1; i<newob->ob_count; i++)
             newentries[i] = entries[i-1];
-        frame->entries = newentries;
-        frame->entry_count = newcount;
-        free(entries);
+
+        _du_write1(frame);
+        frame->ob_nodes = newob;
+
         return newentries + left;
     }
-    else
-        return NULL;
 }
 
 void DuFrame_SetBuiltinMacro(DuObject *frame, char *name, eval_fn func)
 {
     DuFrame_Ensure("DuFrame_SetBuiltinMacro", frame);
+
+    _du_save1(frame);
     DuObject *sym = DuSymbol_FromString(name);
+    _du_restore1(frame);
+
     struct dictentry *e = find_entry((DuFrameObject *)frame, sym, 1);
     e->builtin_macro = func;
-    Du_DECREF(sym);
 }
 
 static void
 _parse_arguments(DuObject *symbol, DuObject *arguments,
                  DuObject *formallist, DuObject *caller, DuObject *callee)
 {
+    abort();
+#if 0
     while (DuCons_Check(formallist)) {
         if (!DuCons_Check(arguments))
             Du_FatalError("call to '%s': not enough arguments",
@@ -140,11 +180,14 @@
     if (arguments != Du_None)
         Du_FatalError("call to '%s': too many arguments",
                       DuSymbol_AsString(symbol));
+#endif
 }
 
 DuObject *_DuFrame_EvalCall(DuObject *frame, DuObject *symbol,
                             DuObject *rest, int execute_now)
 {
+    stm_fatalerror("_DuFrame_EvalCall\n");
+#if 0
     struct dictentry *e;
     DuFrame_Ensure("_DuFrame_EvalCall", frame);
 
@@ -184,10 +227,13 @@
  not_defined:
     Du_FatalError("symbol not defined as a function: '%s'",
                   DuSymbol_AsString(symbol));
+#endif
 }
 
 DuObject *DuFrame_GetSymbol(DuObject *frame, DuObject *symbol)
 {
+    stm_fatalerror("DuFrame_GetSymbol\n");
+#if 0
     struct dictentry *e;
     DuFrame_Ensure("DuFrame_GetSymbol", frame);
 
@@ -198,28 +244,37 @@
     }
     else
         return NULL;
+#endif
 }
 
 void DuFrame_SetSymbol(DuObject *frame, DuObject *symbol, DuObject *value)
 {
+    stm_fatalerror("DuFrame_SetSymbol\n");
+#if 0
     struct dictentry *e;
     DuFrame_Ensure("DuFrame_SetSymbol", frame);
 
     e = find_entry((DuFrameObject *)frame, symbol, 1);
     if (e->value) Du_DECREF(e->value);
     e->value = value; Du_INCREF(value);
+#endif
 }
 
 void DuFrame_SetSymbolStr(DuObject *frame, char *name, DuObject *value)
 {
+    stm_fatalerror("DuFrame_SetSymbolStr\n");
+#if 0
     DuObject *sym = DuSymbol_FromString(name);
     DuFrame_SetSymbol(frame, sym, value);
     Du_DECREF(sym);
+#endif
 }
 
 void DuFrame_SetUserFunction(DuObject *frame, DuObject *symbol,
                              DuObject *arglist, DuObject *progn)
 {
+    stm_fatalerror("DuFrame_SetUserFunction\n");
+#if 0
     struct dictentry *e;
     DuFrame_Ensure("DuFrame_SetUserFunction", frame);
 
@@ -228,18 +283,27 @@
     if (e->func_progn)   Du_DECREF(e->func_progn);
     e->func_arglist = arglist; Du_INCREF(arglist);
     e->func_progn   = progn;   Du_INCREF(progn);
+#endif
 }
 
 void DuFrame_Ensure(char *where, DuObject *ob)
 {
     if (!DuFrame_Check(ob))
         Du_FatalError("%s: expected 'frame' argument, got '%s'",
-                      where, ob->ob_type->dt_name);
+                      where, Du_TYPE(ob)->dt_name);
 }
 
+DuType DuFrameNode_Type = {    /* internal type */
+    "framenode",
+    DUTYPE_FRAMENODE,
+    sizeof(DuFrameNodeObject),
+    (trace_fn)framenode_trace,
+};
+
 DuType DuFrame_Type = {
     "frame",
     DUTYPE_FRAME,
     sizeof(DuFrameObject),
+    (trace_fn)frame_trace,
     (print_fn)frame_print,
 };
diff --git a/duhton/glob.c b/duhton/glob.c
--- a/duhton/glob.c
+++ b/duhton/glob.c
@@ -563,6 +563,8 @@
 
 void Du_Initialize(void)
 {
+    stm_initialize();
+
     DuFrame_SetBuiltinMacro(Du_Globals, "progn", Du_Progn);
     DuFrame_SetBuiltinMacro(Du_Globals, "setq", du_setq);
     DuFrame_SetBuiltinMacro(Du_Globals, "print", du_print);
@@ -599,4 +601,5 @@
 
 void Du_Finalize(void)
 {
+    stm_finalize();
 }
diff --git a/duhton/intobject.c b/duhton/intobject.c
--- a/duhton/intobject.c
+++ b/duhton/intobject.c
@@ -21,6 +21,7 @@
     "int",
     DUTYPE_INT,
     sizeof(DuIntObject),
+    (trace_fn)NULL,
     (print_fn)int_print,
     (eval_fn)NULL,
     (len_fn)int_is_true,
diff --git a/duhton/listobject.c b/duhton/listobject.c
--- a/duhton/listobject.c
+++ b/duhton/listobject.c
@@ -17,6 +17,19 @@
 } DuListObject;
 
 
+void tuple_trace(DuTupleObject *ob, void visit(gcptr *))
+{
+    int i;
+    for (i=ob->ob_count-1; i>=0; i--) {
+        visit(&ob->ob_items[i]);
+    }
+}
+
+void list_trace(DuListObject *ob, void visit(gcptr *))
+{
+    visit((gcptr *)&ob->ob_tuple);
+}
+
 void list_print(DuListObject *ob)
 {
     int i;
@@ -139,10 +152,28 @@
     return _list_pop((DuListObject *)list, index);
 }
 
+size_t _DuTuple_ByteSize(DuObject *tuple)
+{
+    DuTupleObject *t = (DuTupleObject *)tuple;
+    return sizeof(DuTupleObject) + (t->ob_count - 1) * sizeof(DuObject *);
+}
+
+DuType DuTuple_Type = {    /* "tuple" is mostly an internal type here */
+    "tuple",
+    DUTYPE_TUPLE,
+    0,    /* dt_size */
+    (trace_fn)tuple_trace,
+    (print_fn)NULL,
+    (eval_fn)NULL,
+    (len_fn)NULL,
+    (len_fn)NULL,
+};
+
 DuType DuList_Type = {
     "list",
     DUTYPE_LIST,
     sizeof(DuListObject),
+    (trace_fn)list_trace,
     (print_fn)list_print,
     (eval_fn)NULL,
     (len_fn)NULL,
diff --git a/duhton/object.c b/duhton/object.c
--- a/duhton/object.c
+++ b/duhton/object.c
@@ -14,6 +14,24 @@
 };
 
 
+/* callback: get the size of an object */
+size_t stmcb_size(gcptr obj)
+{
+    if (_DuObject_TypeNum(obj) == DUTYPE_TUPLE)
+        return _DuTuple_ByteSize(obj);
+    else
+        return Du_TYPE(obj)->dt_size;
+}
+
+/* callback: trace the content of an object */
+void stmcb_trace(gcptr obj, void visit(gcptr *))
+{
+    trace_fn trace = Du_TYPE(obj)->dt_trace;
+    if (trace)
+        trace(obj, visit);
+}
+
+
 DuObject *DuObject_New(DuType *tp)
 {
     assert(tp->dt_size >= sizeof(DuObject));
@@ -36,6 +54,7 @@
     "NoneType",
     DUTYPE_NONE,
     sizeof(DuObject),
+    (trace_fn)NULL,
     none_print,
     (eval_fn)NULL,
     none_is_true,
diff --git a/duhton/symbol.c b/duhton/symbol.c
--- a/duhton/symbol.c
+++ b/duhton/symbol.c
@@ -1,4 +1,5 @@
 #include <string.h>
+#include <stdlib.h>
 #include "duhton.h"
 
 typedef struct _Du_Symbol {
@@ -13,6 +14,11 @@
     NULL};
 
 
+void symbol_trace(DuSymbolObject *ob, void visit(gcptr *))
+{
+    visit((gcptr *)&ob->next);
+}
+
 void symbol_print(DuSymbolObject *ob)
 {
     _du_read1(ob);
@@ -39,6 +45,7 @@
     "symbol",
     DUTYPE_SYMBOL,
     sizeof(DuSymbolObject),
+    (trace_fn)symbol_trace,
     (print_fn)symbol_print,
     (eval_fn)symbol_eval,
 };

From noreply at buildbot.pypy.org  Tue Jun 25 16:51:45 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Jun 2013 16:51:45 +0200 (CEST)
Subject: [pypy-commit] stmgc default: clean only used part of nursery
Message-ID: <20130625145145.928341C345E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r279:0f3222c773e9
Date: 2013-06-25 16:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/0f3222c773e9/

Log:	clean only used part of nursery

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -516,7 +516,8 @@
     dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
              d->nursery_end));
 #else
-    memset(d->nursery_base, 0, GC_NURSERY);
+    memset(d->nursery_base, 0,
+           d->nursery_current - d->nursery_base);
 #endif
     d->nursery_current = d->nursery_base;
 
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -3,6 +3,7 @@
 
 #ifndef GC_NURSERY
 #define GC_NURSERY        4194304    /* 4 MB */
+//#define GC_NURSERY        (1<<20)    /* 1 MB */
 #endif
 
 

From noreply at buildbot.pypy.org  Tue Jun 25 16:52:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 16:52:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix Du_Types[]
Message-ID: <20130625145241.155521C345E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r280:5be54701cb63
Date: 2013-06-25 16:52 +0200
http://bitbucket.org/pypy/stmgc/changeset/5be54701cb63/

Log:	Fix Du_Types[]

diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -38,6 +38,7 @@
     len_fn dt_length;
 } DuType;
 
+/* keep this list in sync with object.c's Du_Types[] */
 #define DUTYPE_INVALID       0
 #define DUTYPE_NONE          1
 #define DUTYPE_INT           2
@@ -57,6 +58,7 @@
 extern DuType DuList_Type;
 extern DuType DuTuple_Type;
 extern DuType DuFrame_Type;
+extern DuType DuFrameNode_Type;
 extern DuType DuContainer_Type;
 
 extern DuType *Du_Types[_DUTYPE_TOTAL];
diff --git a/duhton/object.c b/duhton/object.c
--- a/duhton/object.c
+++ b/duhton/object.c
@@ -3,14 +3,16 @@
 
 
 DuType *Du_Types[_DUTYPE_TOTAL] = {
-    NULL,
-    &DuNone_Type,
-    &DuInt_Type,
-    &DuSymbol_Type,
-    &DuCons_Type,
-    &DuList_Type,
-    &DuFrame_Type,
-    &DuContainer_Type,
+    NULL,                  // DUTYPE_INVALID       0
+    &DuNone_Type,          // DUTYPE_NONE          1
+    &DuInt_Type,           // DUTYPE_INT           2
+    &DuSymbol_Type,        // DUTYPE_SYMBOL        3
+    &DuCons_Type,          // DUTYPE_CONS          4
+    &DuList_Type,          // DUTYPE_LIST          5
+    &DuTuple_Type,         // DUTYPE_TUPLE         6
+    &DuFrame_Type,         // DUTYPE_FRAME         7
+    &DuFrameNode_Type,     // DUTYPE_FRAMENODE     8
+    &DuContainer_Type,     // DUTYPE_CONTAINER     9
 };
 
 

From noreply at buildbot.pypy.org  Tue Jun 25 17:01:57 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 17:01:57 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fixes fixes
Message-ID: <20130625150157.936B41C3436@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r281:84b179a63ac1
Date: 2013-06-25 17:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/84b179a63ac1/

Log:	fixes fixes

diff --git a/duhton/Makefile b/duhton/Makefile
--- a/duhton/Makefile
+++ b/duhton/Makefile
@@ -5,7 +5,7 @@
 	gcc -lrt -pthread -g -O2 -o duhton *.c ../c4/stmgc.c -Wall
 
 duhton_debug: *.c *.h ../c4/*.c ../c4/*.h
-	gcc -lrt -pthread -g -DDu_DEBUG -o duhton_debug *.c ../c4/stmgc.c -Wall
+	gcc -lrt -pthread -g -DDu_DEBUG -D_GC_DEBUG=2 -DGC_NURSERY=2048 -o duhton_debug *.c ../c4/stmgc.c -Wall
 
 clean:
 	rm -f duhton duhton_debug
diff --git a/duhton/compile.c b/duhton/compile.c
--- a/duhton/compile.c
+++ b/duhton/compile.c
@@ -33,7 +33,9 @@
             goto done;
 
         case '(':
+            _du_save1(list);
             item = _Du_Parse(f, level + 1, 0);
+            _du_restore1(list);
             c = fgetc(f);
             break;
 
@@ -69,12 +71,14 @@
                 } while (!(isspace(c) || c == '(' || c == ')' || c == EOF));
                 *p = '\0';
                 number = strtol(token, &end, 0);
+                _du_save1(list);
                 if (*end == '\0') {
                     item = DuInt_FromInt(number);
                 }
                 else {
                     item = DuSymbol_FromString(token);
                 }
+                _du_restore1(list);
                 break;
             }
         }
diff --git a/duhton/listobject.c b/duhton/listobject.c
--- a/duhton/listobject.c
+++ b/duhton/listobject.c
@@ -75,7 +75,10 @@
 
     _du_read1(olditems);
     int i, newcount = olditems->ob_count + 1;
+
+    _du_save3(ob, x, olditems);
     DuTupleObject *newitems = DuTuple_New(newcount);
+    _du_restore3(ob, x, olditems);
 
     for (i=0; i<newcount-1; i++)
         newitems->ob_items[i] = olditems->ob_items[i];

From noreply at buildbot.pypy.org  Tue Jun 25 17:12:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Jun 2013 17:12:44 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Generalize the tuple's stmcb_size()
Message-ID: <20130625151244.B81B51C0175@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r282:d1f22e7b8690
Date: 2013-06-25 17:12 +0200
http://bitbucket.org/pypy/stmgc/changeset/d1f22e7b8690/

Log:	Generalize the tuple's stmcb_size()

diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -23,6 +23,7 @@
 
 
 typedef void(*trace_fn)(DuObject *, void visit(gcptr *));
+typedef size_t(*bytesize_fn)(DuObject *);
 typedef void(*print_fn)(DuObject *);
 typedef DuObject *(*eval_fn)(DuObject *, DuObject *);
 typedef int(*len_fn)(DuObject *);
@@ -36,6 +37,7 @@
     eval_fn dt_eval;
     len_fn dt_is_true;
     len_fn dt_length;
+    bytesize_fn dt_bytesize;
 } DuType;
 
 /* keep this list in sync with object.c's Du_Types[] */
@@ -98,7 +100,6 @@
 DuObject *DuList_GetItem(DuObject *list, int index);
 void DuList_SetItem(DuObject *list, int index, DuObject *newobj);
 DuObject *DuList_Pop(DuObject *list, int index);
-size_t _DuTuple_ByteSize(DuObject *tuple);
 
 DuObject *DuContainer_New(DuObject *obj);
 DuObject *DuContainer_GetRef(DuObject *container);
diff --git a/duhton/frame.c b/duhton/frame.c
--- a/duhton/frame.c
+++ b/duhton/frame.c
@@ -28,6 +28,12 @@
     }
 }
 
+size_t framenode_bytesize(DuFrameNodeObject *ob)
+{
+    return (sizeof(DuFrameNodeObject) +
+            (ob->ob_count - 1) * sizeof(struct dictentry));
+}
+
 
 typedef struct {
     DuOBJECT_HEAD
@@ -100,6 +106,13 @@
     struct dictentry *entries = ob->ob_items;
     revision_t search_id = stm_id(symbol);
 
+#ifdef _GC_DEBUG
+    int j;
+    for (j = 0; j < right; j++) {
+        dprintf(("\t%p\n", (gcptr)entries[j].symbol_id));
+    }
+#endif
+
     while (right > left) {
         int middle = (left + right) / 2;
         revision_t found_id = entries[middle].symbol_id;
@@ -137,6 +150,7 @@
             newentries[i] = entries[i];
 
         DuSymbol_Ensure("find_entry", symbol);
+        dprintf(("NEW ENTRY ADDED WITH search_id = %p\n", (gcptr)search_id));
         newentries[left].symbol_id = search_id;
         newentries[left].symbol = symbol;
         newentries[left].value = NULL;
@@ -305,8 +319,13 @@
 DuType DuFrameNode_Type = {    /* internal type */
     "framenode",
     DUTYPE_FRAMENODE,
-    sizeof(DuFrameNodeObject),
+    0,    /* dt_size */
     (trace_fn)framenode_trace,
+    (print_fn)NULL,
+    (eval_fn)NULL,
+    (len_fn)NULL,
+    (len_fn)NULL,
+    (bytesize_fn)framenode_bytesize,
 };
 
 DuType DuFrame_Type = {
diff --git a/duhton/listobject.c b/duhton/listobject.c
--- a/duhton/listobject.c
+++ b/duhton/listobject.c
@@ -25,6 +25,11 @@
     }
 }
 
+size_t tuple_bytesize(DuTupleObject *ob)
+{
+    return sizeof(DuTupleObject) + (ob->ob_count - 1) * sizeof(DuObject *);
+}
+
 void list_trace(DuListObject *ob, void visit(gcptr *))
 {
     visit((gcptr *)&ob->ob_tuple);
@@ -155,12 +160,6 @@
     return _list_pop((DuListObject *)list, index);
 }
 
-size_t _DuTuple_ByteSize(DuObject *tuple)
-{
-    DuTupleObject *t = (DuTupleObject *)tuple;
-    return sizeof(DuTupleObject) + (t->ob_count - 1) * sizeof(DuObject *);
-}
-
 DuType DuTuple_Type = {    /* "tuple" is mostly an internal type here */
     "tuple",
     DUTYPE_TUPLE,
@@ -170,6 +169,7 @@
     (eval_fn)NULL,
     (len_fn)NULL,
     (len_fn)NULL,
+    (bytesize_fn)tuple_bytesize,
 };
 
 DuType DuList_Type = {
diff --git a/duhton/object.c b/duhton/object.c
--- a/duhton/object.c
+++ b/duhton/object.c
@@ -19,10 +19,11 @@
 /* callback: get the size of an object */
 size_t stmcb_size(gcptr obj)
 {
-    if (_DuObject_TypeNum(obj) == DUTYPE_TUPLE)
-        return _DuTuple_ByteSize(obj);
-    else
-        return Du_TYPE(obj)->dt_size;
+    DuType *tp = Du_TYPE(obj);
+    size_t result = tp->dt_size;
+    if (result == 0)
+        result = tp->dt_bytesize(obj);
+    return result;
 }
 
 /* callback: trace the content of an object */

From noreply at buildbot.pypy.org  Tue Jun 25 17:36:03 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 25 Jun 2013 17:36:03 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-nditer: Some progress on op_axes
Message-ID: <20130625153603.27ED71C32EB@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-nditer
Changeset: r64977:ea9a8efd21c6
Date: 2013-06-25 17:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ea9a8efd21c6/

Log:	Some progress on op_axes

diff --git a/pypy/module/micronumpy/interp_nditer.py b/pypy/module/micronumpy/interp_nditer.py
--- a/pypy/module/micronumpy/interp_nditer.py
+++ b/pypy/module/micronumpy/interp_nditer.py
@@ -269,6 +269,7 @@
         parse_func_flags(space, self, w_flags)
         self.op_flags = parse_op_arg(space, 'op_flags', w_op_flags,
                                      len(self.seq), parse_op_flag)
+        self.set_op_axes(space, w_op_axes)
         self.iters=[]
         self.shape = iter_shape = shape_agreement_multiple(space, self.seq)
         if self.tracked_index != "":
@@ -288,6 +289,22 @@
                 self.iters.append(BoxIterator(get_iter(space, self.order,
                                 self.seq[i], iter_shape), self.op_flags[i]))
 
+    def set_op_axes(self, space, w_op_axes):
+        if space.len_w(w_op_axes) != len(self.seq):
+            raise OperationError(space.w_ValueError, space.wrap("op_axes must be a tuple/list matching the number of ops"))
+        op_axes = space.listview(w_op_axes)
+        l = -1
+        for w_axis in op_axes:
+            if not space.is_(w_axis, space.w_None):
+                axis_len = space.len_w(w_axis)
+                if l == -1:
+                    l = axis_len
+                elif axis_len != l:
+                    raise OperationError(space.w_ValueError, space.wrap("Each entry of op_axes must have the same size"))
+                self.op_axes.append([space.int_w(x) if not space.is_(x, space.w_None) else space.w_None for x in space.listview(w_axis)])
+        if l == -1:
+            raise OperationError(space.w_ValueError, space.wrap("If op_axes is provided, at least one list of axes must be contained within it"))
+
     def descr_iter(self, space):
         return space.wrap(self)
 

From noreply at buildbot.pypy.org  Tue Jun 25 17:51:33 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Jun 2013 17:51:33 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fix validation of priv_from_prot
Message-ID: <20130625155133.6F2321C33FE@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r283:2e4480dc9707
Date: 2013-06-25 17:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/2e4480dc9707/

Log:	fix validation of priv_from_prot

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -330,6 +330,14 @@
             p = (gcptr)(((nodeptr)read_barrier(p))->next);
         break;
     case 9: // XXX: rare events
+        k = get_rand(10);
+        if (k == 1) {
+            push_roots();
+            stm_push_root(p);
+            stm_become_inevitable("fun");
+            p = stm_pop_root();
+            pop_roots();
+        }
         break;
     case 10: // only do a stm_read_barrier
         p = read_barrier(p);
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -652,12 +652,17 @@
             {
               /* such an object R might be listed in list_of_read_objects
                  before it was turned from protected to private */
-              continue;
+              if(((gcptr)v)->h_tid & GCFLAG_BACKUP_COPY)
+                continue;
+              /* the backup was stolen */
+              return 0; 
             }
           else if ((R->h_tid & (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
                             == (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
             {
-              /* such an object is identical to the one it points to */
+              /* such an object is identical to the one it points to
+               (stolen protected young object with h_revision pointing
+               to the new copy) */
               R = (gcptr)v;
               goto retry;
             }

From noreply at buildbot.pypy.org  Tue Jun 25 18:16:08 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Jun 2013 18:16:08 +0200 (CEST)
Subject: [pypy-commit] stmgc default: break the huge switch in demo_random
Message-ID: <20130625161608.6A17C1C0175@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <meierrem at student.ethz.ch>
Branch: 
Changeset: r284:7b5404748520
Date: 2013-06-25 18:15 +0200
http://bitbucket.org/pypy/stmgc/changeset/7b5404748520/

Log:	break the huge switch in demo_random

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -274,93 +274,97 @@
 
 }
 
-gcptr do_step(gcptr p)
+gcptr rare_events(gcptr p, gcptr _r, gcptr _sr)
 {
-    nodeptr w_r, w_sr, w_t;
-    gcptr _r, _sr, _t;
-    int num, k;
+    int k = get_rand(10);
+    if (k == 1) {
+        push_roots();
+        stm_push_root(p);
+        stm_become_inevitable("fun");
+        p = stm_pop_root();
+        pop_roots();
+    } 
+    else if (k < 4) {
+        push_roots();
+        stmgc_minor_collect();
+        pop_roots();
+        p = NULL;
+    }
+    return p;
+}
 
-    _t = NULL;
-    num = get_rand(td.num_roots);
-    _r = td.roots[num];
-    
-    num = get_rand(SHARED_ROOTS);
-    _sr = shared_roots[num];
-
-    k = get_rand(19);
-    check(p);
-
+gcptr simple_events(gcptr p, gcptr _r, gcptr _sr)
+{
+    nodeptr w_r;
+    int k = get_rand(8);
+    int num = get_rand(td.num_roots);
     switch (k) {
     case 0: // remove a root
         if (num > 0)
             del_root(num);
         break;
-    case 1: // set 'p' to point to a root
+    case 1: // add 'p' to roots
+        if (p && td.num_roots < MAXROOTS)
+            td.roots[td.num_roots++] = p;
+        break;
+    case 2: // set 'p' to point to a root
         if (_r)
             p = _r;
         break;
-    case 2: // add 'p' to roots
-        if (p && td.num_roots < MAXROOTS)
-            td.roots[td.num_roots++] = p;
-        break;
     case 3: // allocate fresh 'p'
         p = (gcptr)allocate_node();
         break;
-    case 4: // set 'p' as *next in one of the roots
+    case 4:  // read and validate 'p'
+        p = read_barrier(p);
+        break;
+    case 5: // only do a stm_write_barrier
+        p = write_barrier(p);
+        break;
+    case 6: // follow p->next
+        if (p)
+            p = (gcptr)(((nodeptr)read_barrier(p))->next);
+        break;
+    case 7: // set 'p' as *next in one of the roots
         check(_r);
         w_r = (nodeptr)write_barrier(_r);
         check((gcptr)w_r);
         check(p);
         w_r->next = (struct node*)p;
         break;
-    case 5:  // read and validate 'p'
-        read_barrier(p);
-        break;
-    case 6: // transaction break
-        if (td.interruptible)
-            return (gcptr)-1; // break current
-        transaction_break();
-        p = NULL;
-        break;
-    case 7: // only do a stm_write_barrier
-        p = write_barrier(p);
-        break;
-    case 8:
-        if (p)
-            p = (gcptr)(((nodeptr)read_barrier(p))->next);
-        break;
-    case 9: // XXX: rare events
-        k = get_rand(10);
-        if (k == 1) {
-            push_roots();
-            stm_push_root(p);
-            stm_become_inevitable("fun");
-            p = stm_pop_root();
-            pop_roots();
-        }
-        break;
-    case 10: // only do a stm_read_barrier
-        p = read_barrier(p);
-        break;
-    case 11:
+    }
+    return p;
+}
+
+gcptr shared_roots_events(gcptr p, gcptr _r, gcptr _sr)
+{
+    nodeptr w_sr;
+
+    int k = get_rand(3);
+    switch (k) {
+    case 0: // read_barrier on shared root
         read_barrier(_sr);
         break;
-    case 12:
+    case 1: // write_barrier on shared root
         write_barrier(_sr);
         break;
-    case 13:
+    case 2:
         w_sr = (nodeptr)write_barrier(_sr);
         w_sr->next = (nodeptr)shared_roots[get_rand(SHARED_ROOTS)];
         break;
-    case 14:
-        push_roots();
-        stmgc_minor_collect();
-        pop_roots();
-        p = NULL;
-        break;
-    case 15: /* test stm_id on (non-)shared roots */
+    }
+    return p;
+}
+
+gcptr id_hash_events(gcptr p, gcptr _r, gcptr _sr)
+{
+    nodeptr w_t;
+    int k = get_rand(4);
+    gcptr _t = NULL;
+
+    switch (k) {
+    case 0: /* test stm_id on (non-)shared roots */
         _t = _r;
-    case 16:
+    case 1:
         if (!_t)
             _t = _sr;
         w_t = (nodeptr)read_barrier(_t);
@@ -374,9 +378,9 @@
             assert(w_t->id == stm_id((gcptr)_t));
         }
         break;
-    case 17: /* test stm_hash on (non-)shared roots */
+    case 2: /* test stm_hash on (non-)shared roots */
         _t = _r;
-    case 18:
+    case 3:
         if (!_t)
             _t = _sr;
         w_t = (nodeptr)read_barrier(_t);
@@ -400,6 +404,40 @@
 }
 
 
+
+gcptr do_step(gcptr p)
+{
+    gcptr _r, _sr;
+    int num, k;
+
+    num = get_rand(td.num_roots);
+    _r = td.roots[num];
+    
+    num = get_rand(SHARED_ROOTS);
+    _sr = shared_roots[num];
+
+    k = get_rand(9);
+    check(p);
+
+    if (k < 3)
+        p = simple_events(p, _r, _sr);
+    else if (k < 5)
+        p = shared_roots_events(p, _r, _sr);
+    else if (k < 7)
+        p = id_hash_events(p, _r, _sr);
+    else if (k < 8)
+        p = rare_events(p, _r, _sr);
+    else if (get_rand(3) == 1) {
+        // transaction break
+        if (td.interruptible)
+            return (gcptr)-1; // break current
+        transaction_break();
+        p = NULL;
+    }
+    return p;
+}
+
+
 void transaction_break()
 {
     push_roots();

From noreply at buildbot.pypy.org  Tue Jun 25 18:29:44 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 25 Jun 2013 18:29:44 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: rollback of last commit
Message-ID: <20130625162944.6CA2E1C0175@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r475:6c5a57856c52
Date: 2013-06-25 18:28 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/6c5a57856c52/

Log:	rollback of last commit update of the CI image

diff too long, truncating to 2000 out of 79931 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -221,7 +221,7 @@
     "lang-smalltalk",
     executables=[
         Executable("rsqueakvm", "./targetimageloadingsmalltalk-c"),
-        # Executable("rsqueakvm-nojit", "./targetimageloadingsmalltalk-nojit-c")
+        Executable("rsqueakvm-nojit", "./targetimageloadingsmalltalk-nojit-c")
     ],
     arguments=["images/%s.image" % SqueakImage, '-m', 'runSPyBenchmarks']
 )
diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
--- a/images/Squeak4.5-12568.changes
+++ b/images/Squeak4.5-12568.changes
@@ -36,4 +36,4 @@
 Workspace allInstances do: [:w | w topView delete].
 ReleaseBuilderFor4dot4 prepareNewBuild.
 Smalltalk snapshot: true andQuit: true.
-!

----End fileIn of a stream----!

----SNAPSHOT----{31 March 2013 . 3:27:34 pm} Squeak4.5-12327.image priorSource: 7430688!
!Installer methodsFor: 'squeakmap' stamp: 'fbs 1/28/2013 19:25' prior: 57597950!
packageAndVersionFrom: pkg
	| p |
	p := ReadStream on: pkg .
	^{(p upTo: $(). p upTo: $)} collect: [:s | s withBlanksTrimmed].! !

"Installer-Core"!
!Categorizer methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 16:58'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!ClassCategoryReader methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:21'!
scanFrom: aStream environment: anEnvironment
	"File in methods from the stream, aStream."
	| methodText |
	[methodText := aStream nextChunkText.
	 methodText size > 0] whileTrue:
		[class 
			compile: methodText 
			environment: anEnvironment
			classified: category
			withStamp: changeStamp 
			notifying: nil]! !
!ClassCommentReader methodsFor: 'as yet unclassified' stamp: 'cwp 6/20/2012 17:22'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!Metaclass methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:29'!
bindingOf: varName environment: anEnvironment 
	^ thisClass classBindingOf: varName environment: anEnvironment! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 20:03' prior: 22505876!
\\ aNumber 
	"Primitive. Take the receiver modulo the argument. The result is the
	remainder rounded towards negative infinity, of the receiver divided
	by the argument. Fail if the argument is 0. Fail if either the argument
	or the result is not a SmallInteger or a LargePositiveInteger less than
	2-to-the-30th (1073741824). Optional. See Object documentation whatIsAPrimitive."

	<primitive: 31>
	aNumber isInteger
		ifTrue:
			[| neg qr q r |
			neg := self negative == aNumber negative == false.
			qr := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: neg).
			q := qr first normalize.
			r := qr last normalize.
			^(q negative
				ifTrue: [r isZero not]
				ifFalse: [q isZero and: [neg]])
					ifTrue: [r + aNumber]
					ifFalse: [r]].
	^super \\ aNumber
	! !
!LargePositiveInteger methodsFor: 'converting' stamp: 'nice 1/27/2012 22:41' prior: 37616324!
asFloat
	"Answer a Float that best approximates the value of the receiver.
	This algorithm is optimized to process only the significant digits of a LargeInteger.
	And it does honour IEEE 754 round to nearest even mode in case of excess precision (see details below)."
	
	"How numbers are rounded in IEEE 754 default rounding mode:
	A shift is applied so that the highest 53 bits are placed before the floating point to form a mantissa.
	The trailing bits form the fraction part placed after the floating point.
	This fractional number must be rounded to the nearest integer.
	If fraction part is 2r0.1, exactly between two consecutive integers, there is a tie.
	The nearest even integer is chosen in this case.
	Examples (First 52bits of mantissa are omitted for brevity):
	2r0.00001 is rounded downward to 2r0
	2r1.00001 is rounded downward to 2r1
	2r0.1 is a tie and rounded to 2r0 (nearest even)
	2r1.1 is a tie and rounded to 2r10 (nearest even)
	2r0.10001 is rounded upward to 2r1
	2r1.10001 is rounded upward to 2r10
	Thus, if the next bit after floating point is 0, the mantissa is left unchanged.
	If next bit after floating point is 1, an odd mantissa is always rounded upper.
	An even mantissa is rounded upper only if the fraction part is not a tie."
	
	"Algorihm details:
	The floating point hardware can perform the rounding correctly with several excess bits as long as there is a single inexact operation.
	This can be obtained by splitting the mantissa plus excess bits in two part with less bits than Float precision.
	Note 1: the inexact flag in floating point hardware must not be trusted because in some cases the operations would be exact but would not take into account some bits that were truncated before the Floating point operations.
	Note 2: the floating point hardware is presumed configured in default rounding mode."
	
	| mantissa shift excess result n |

	"Check how many bits excess the maximum precision of a Float mantissa."
	excess := self highBitOfMagnitude - Float precision.
	excess > 7
		ifTrue:
			["Remove the excess bits but seven."
			mantissa := self bitShiftMagnitude: 7 - excess.
			shift := excess - 7.
			"An even mantissa with a single excess bit immediately following would be truncated.
			But this would not be correct if above shift has truncated some extra bits.
			Check this case, and round excess bits upper manually."
			((mantissa digitAt: 1) = 2r01000000 and: [self anyBitOfMagnitudeFrom: 1 to: shift])
				ifTrue: [mantissa := mantissa + 1]]
		ifFalse:
			[mantissa := self.
			shift := 0].

	"There will be a single inexact round off at last iteration"
	result := (mantissa digitAt: (n := mantissa digitLength)) asFloat.
	[(n := n - 1) > 0] whileTrue: [
		result := 256.0 * result + (mantissa digitAt: n) asFloat].
	^result timesTwoPower: shift.! !
!LargePositiveInteger methodsFor: 'private' stamp: 'nice 12/30/2012 14:25'!
primitiveQuo: anInteger 
	"Primitive. Divide the receiver by the argument and return the result.
	Round the result down towards zero to make it a whole integer. Fail if
	the argument is 0. Fail if either the argument or the result is not a
	SmallInteger or a LargePositiveInteger less than 2-to-the-30th (1073741824). Optional. See
	Object documentation whatIsAPrimitive."

	<primitive: 33>
	^nil! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 14:34'!
rem: aNumber 
	"Remainder defined in terms of quo:. See super rem:.
	This is defined only to speed up case of very large integers."

	(self primitiveQuo: aNumber)
		ifNotNil: [:quo | ^self - (quo * aNumber)].
	 aNumber isInteger
		ifTrue:
			[| ng rem |
			ng := self negative == aNumber negative == false.
			rem := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: ng) at: 2.
			^ rem normalize].
	^super rem: aNumber! !
!LargeNegativeInteger methodsFor: 'converting' stamp: 'nice 1/1/2013 15:42' prior: 37616204!
asFloat
	^super asFloat negated! !
!UndefinedObject methodsFor: 'class hierarchy' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:32'!
bindingOf: varName environment: anEnvironment
	^superclass bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:30'!
classBindingOf: varName environment: anEnvironment
	^self bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:37'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor
	"Postprocesses a literal scanned by Scanner scanToken (esp. xLitQuote).
	If scannedLiteral is not an association, answer it.
	Else, if it is of the form:
		nil->#NameOfMetaclass
	answer nil->theMetaclass, if any has that name, else report an error.
	Else, if it is of the form:
		#NameOfGlobalVariable->anythiEng
	answer the global, class, or pool association with that nameE, if any, else
	add it to Undeclared a answer the new Association."

	| key value |
	(scannedLiteral isVariableBinding)
		ifFalse: [^ scannedLiteral].
	key := scannedLiteral key.
	value := scannedLiteral value.
	key ifNil: "###<metaclass soleInstance name>"
		[(self bindingOf: value environment: anEnvironment) ifNotNil:
			[:assoc|
			(assoc value isKindOf: Behavior) ifTrue: 
				[^ nil->assoc value class]].
			 requestor notify: 'No such metaclass'.
			 ^false].
	(key isSymbol) ifTrue: "##<global var name>"
		[(self bindingOf: key environment: anEnvironment) ifNotNil:
			[:assoc | ^assoc].
		^ anEnvironment undeclared: key].
	requestor notify: '## must be followed by a non-local variable name'.
	^false

"	Form literalScannedAs: 14 notifying: nil 14
	Form literalScannedAs: #OneBitForm notiEfying: nil  OneBitForm
	Form literalScannedAs: ##OneBitForm notifying: nil  OneBitForm->a Form
	Form literalScannedAs: ##Form notifying: nil   Form->Form
	Form literalScannedAs: ###Form notifying: nil   nilE->Form class
"! !
!Fraction methodsFor: 'converting' stamp: 'nice 11/21/2011 22:34' prior: 37619655!
asFloat
	"Answer a Float that closely approximates the value of the receiver.
	This implementation will answer the closest floating point number to the receiver.
	In case of a tie, it will use the IEEE 754 round to nearest even mode.
	In case of overflow, it will answer +/- Float infinity."

	| a b mantissa exponent hasTruncatedBits lostBit n ha hb hm |
	a := numerator abs.
	b := denominator.	"denominator is always positive"
	ha := a highBitOfMagnitude.
	hb := b highBitOfMagnitude.
	
	"Number of bits to keep in mantissa plus one to handle rounding."
	n := 1 + Float precision.

	"If both numerator and denominator are represented exactly in floating point number,
	then fastest thing to do is to use hardwired float division."
	(ha < n and: [hb < n]) ifTrue: [^numerator asFloat / denominator asFloat].

	"Shift the fraction by a power of two exponent so as to obtain a mantissa with n bits.
	First guess is rough, the mantissa might have n+1 bits."
	exponent := ha - hb - n.
	exponent >= 0
		ifTrue: [b := b bitShift: exponent]
		ifFalse: [a := a bitShift: exponent negated].
	mantissa := a quo: b.
	hasTruncatedBits := a > (mantissa * b).
	hm := mantissa highBit.
	
	"Check for gradual underflow, in which case the mantissa will loose bits.
	Keep at least one bit to let underflow preserve the sign of zero."
	lostBit := Float emin - (exponent + hm - 1).
	lostBit > 0 ifTrue: [n := n - lostBit max: 1].

	"Remove excess bits in the mantissa."
	hm > n
		ifTrue:
			[exponent := exponent + hm - n.
			hasTruncatedBits := hasTruncatedBits or: [mantissa anyBitOfMagnitudeFrom: 1 to: hm - n].
			mantissa := mantissa bitShift: n - hm].

	"Check if mantissa must be rounded upward.
	The case of tie (mantissa odd & hasTruncatedBits not)
	will be handled by Integer>>asFloat."
	(hasTruncatedBits and: [mantissa odd])
		ifTrue: [mantissa := mantissa + 1].

	^ (self positive
			ifTrue: [mantissa asFloat]
			ifFalse: [mantissa asFloat negated])
		timesTwoPower: exponent! !
!Float methodsFor: 'arithmetic' stamp: 'nice 12/20/2012 23:16' prior: 20878776!
negated
	"Answer a Number that is the negation of the receiver.
	Implementation note: this version cares of negativeZero."

	^-1.0 * self! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:21'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor
	^ self 
		compile: text 
		environment: anEnvironment 
		classified: category 
		withStamp: changeStamp 
		notifying: requestor 
		logSource: self acceptsLoggingOfCompilation! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 12/27/2012 13:17'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor logSource: logSource
	| methodAndNode context methodNode |
	context := CompilationCue
		source: text
		class: self
		environment: anEnvironment
		category: category
		requestor: requestor.
	methodNode := self newCompiler compile: context ifFail: [^ nil].
	methodAndNode := CompiledMethodWithNode 
		generateMethodFromNode: methodNode 
		trailer: self defaultMethodTrailer.

	logSource ifTrue: [
		self logMethodSource: text forMethodWithNode: methodAndNode 
			inCategory: category withStamp: changeStamp notifying: requestor.
	].
	self addAndClassifySelector: methodAndNode selector withMethod: methodAndNode 
		method inProtocol: category notifying: requestor.
	self instanceSide noteCompilationOf: methodAndNode selector meta: self isClassSide.
	^ methodAndNode selector! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:47'!
bindingOf: varName environment: anEnvironment
	"Answer the binding of some variable resolved in the scope of the receiver"
	| aSymbol binding |
	aSymbol := varName asSymbol.

	"First look in classVar dictionary."
	binding := self classPool bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Next look in shared pools."
	self sharedPools do:[:pool | 
		binding := pool bindingOf: aSymbol.
		binding ifNotNil:[^binding].
	].

	"Next look in declared environment."
	binding := anEnvironment bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Finally look higher up the superclass chain and fail at the end."
	superclass == nil
		ifTrue: [^ nil]
		ifFalse: [^ superclass bindingOf: aSymbol].

! !

"Kernel"!

ParseNode subclass: #Encoder
	instanceVariableNames: 'scopeTable nTemps supered requestor class selector literalStream selectorSet litIndSet litSet sourceRanges globalSourceRanges addedSelectorAndMethodClassLiterals optimizedSelectors cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Encoder commentStamp: 'cwp 12/26/2012 23:29' prior: 36323851!
I encode names and literals into tree nodes with byte codes for the compiler. Byte codes for literals are not assigned until the tree-sizing pass of the compiler, because only then is it known which literals are actually needed. I also keep track of sourceCode ranges during parsing and code generation so I can provide an inverse map for the debugger.!

Scanner subclass: #Parser
	instanceVariableNames: 'here hereType hereMark hereEnd prevMark prevEnd encoder requestor parseNode failBlock requestorOffset tempsMark doitFlag properties category queriedUnusedTemporaries cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Parser commentStamp: 'cwp 12/26/2012 23:34' prior: 38557958!
I parse Smalltalk syntax and create a MethodNode that is the root of the parse tree. I look one token ahead.!

Object subclass: #CompilationCue
	instanceVariableNames: 'source context receiver class environment category requestor'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

Object subclass: #Compiler
	instanceVariableNames: 'sourceStream requestor class category context parser cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Compiler commentStamp: 'cwp 12/26/2012 23:17' prior: 59257505!
The compiler accepts Smalltalk source code and compiles it with respect to a given class. The user of the compiler supplies a context so that temporary variables are accessible during compilation. If there is an error, a requestor (usually a kind of StringHolderController) is sent the message notify:at:in: so that the error message can be displayed. If there is no error, then the result of compilation is a MethodNode, which is the root of a parse tree whose nodes are kinds of ParseNodes. The parse tree can be sent messages to (1) generate code for a CompiledMethod (this is done for compiling methods or evaluating expressions); (2) pretty-print the code (for formatting); or (3) produce a map from object code back to source code (used by debugger program-counter selection). See also Parser, Encoder, ParseNode.!
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:34'!
init: aCue notifying: anObject
	"The use of the variable requestor is a bit confusing here. This is
	*not* the original requestor, which is available through the cue.
	It's the Parser instance that is using the encoder."

	self setCue: aCue.
	requestor := anObject.
	nTemps := 0.
	supered := false.
	self initScopeAndLiteralTables.
	cue getClass variablesAndOffsetsDo:
		[:variable "<String|CFieldDefinition>" :offset "<Integer|nil>" |
		offset isNil
			ifTrue: [scopeTable at: variable name put: (FieldNode new fieldDefinition: variable)]
			ifFalse: [scopeTable
						at: variable
						put: (offset >= 0
								ifTrue: [InstanceVariableNode new
											name: variable index: offset]
								ifFalse: [MaybeContextInstanceVariableNode new
											name: variable index: offset negated])]].
	cue context ~~ nil ifTrue:
		[| homeNode |
		 homeNode := self bindTemp: self doItInContextName.
		 "0th temp = aContext passed as arg"
		 cue context tempNames withIndexDo:
			[:variable :index|
			scopeTable
				at: variable
				put: (MessageAsTempNode new
						receiver: homeNode
						selector: #namedTempAt:
						arguments: (Array with: (self encodeLiteral: index))
						precedence: 3
						from: self)]].
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/26/2012 23:30'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy instance variables for methods that
	don't use cue yet"
	class := cue getClass.! !
!Dictionary methodsFor: '*Compiler' stamp: 'cwp 6/22/2012 09:17'!
bindingOf: varName ifAbsent: aBlock

	^self associationAt: varName ifAbsent: aBlock! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:37'!
init: sourceStream cue: aCue failBlock: aBlock

	self setCue: aCue.
	failBlock := aBlock.
	requestorOffset := 0.
	super scan: sourceStream.
	prevMark := hereMark := mark.
	self advance
! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/26/2012 23:41'!
parse: sourceStream cue: aCue noPattern: noPattern ifFail: aBlock 
	"Answer a MethodNode for the argument, sourceStream, that is the root of
	 a parse tree. Parsing is done with respect to the CompilationCue to 
	 resolve variables. Errors in parsing are reported to the cue's requestor; 
	 otherwise aBlock is evaluated. The argument noPattern is a Boolean that is
	 true if the the sourceStream does not contain a method header (i.e., for DoIts)."

	| methNode repeatNeeded myStream s p subSelection |
	myStream := sourceStream.
	[repeatNeeded := false.
	 p := myStream position.
	 s := myStream upToEnd.
	 myStream position: p.
	 subSelection := aCue requestor notNil and: [aCue requestor selectionInterval = (p + 1 to: p + s size)].
	 self encoder init: aCue notifying: self.
	 self init: myStream cue: aCue failBlock: [^ aBlock value].
	 doitFlag := noPattern.
	 failBlock:= aBlock.
	 [methNode := self method: noPattern context: cue context] 
		on: ReparseAfterSourceEditing 
		do:	[ :ex |
			repeatNeeded := true.
			myStream := subSelection
							ifTrue:
								[ReadStream
									on: cue requestor text string
									from: cue requestor selectionInterval first
									to: cue requestor selectionInterval last]
							ifFalse:
								[ReadStream on: cue requestor text string]].
	 repeatNeeded] whileTrue:
		[encoder := self encoder class new].
	methNode sourceText: s.
	^methNode
! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:35'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy variables for methods that don't use cue yet."
	requestor := cue requestor.
	category := cue category.! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
class: aClass
	^ self 
		context: nil
		class: aClass
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
context: aContext class: aClass requestor: anObject
	^ self
		source: nil
		context: aContext
		receiver: nil
		class: aClass
		environment: (aClass ifNotNil: [aClass environment])
		category: nil
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aTextOrStream class: aClass environment: anEnvironment category: aString requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		receiver: nil
		class: aClass
		environment: anEnvironment
		category: aString
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
source: aTextOrStream context: aContext class: aClass category: aString requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		receiver: (aContext ifNotNil: [aContext receiver])
		class: aClass 
		environment: (aClass ifNotNil: [aClass environment])
		category: aString 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream context: aContext class: aClass requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		class: aClass 
		category: nil 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:55'!
source: aTextOrStream context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	^ self basicNew
		initializeWithSource: aTextOrStream 
		context: aContext 
		receiver: recObject 
		class: aClass 
		environment: anEnvironment 
		category: aString 
		requestor: reqObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aString environment: anEnvironment
	^ self 
		source: aString
		context: nil
		receiver: nil
		class: UndefinedObject
		environment: anEnvironment
		category: nil
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		class: nil
		requestor: anObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/20/2012 09:39'!
bindingOf: aSymbol
	^ class bindingOf: aSymbol environment: environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
category
	^ category! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 12/26/2012 23:19'!
context
	^ context! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
environment
	^ environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
getClass
	^ class! !
!CompilationCue methodsFor: 'initialization' stamp: 'cwp 12/26/2012 23:16'!
initializeWithSource: aTextOrString context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	self initialize.
	source := aTextOrString isStream ifTrue: [aTextOrString contents] ifFalse: [aTextOrString].
	context := aContext.
	receiver := recObject.
	class := aClass.
	environment := anEnvironment.
	category := aString.
	requestor := reqObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: anObject notifying: anEncoder
	^ class literalScannedAs: anObject environment: environment notifying: anEncoder! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
receiver
	^ receiver! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
requestor
	^ requestor! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
source
	^ source! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:44'!
sourceStream
	^ source readStream! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 6/20/2012 17:25'!
evaluate: aString environment: anEnvironment
	^ self 
		evaluate: aString 
		environment: anEnvironment 
		logged: false! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 12/27/2012 12:36'!
evaluate: aString environment: anEnvironment logged: aBoolean
	| cue |
	cue := CompilationCue
		source: aString
		environment: anEnvironment.
		
	^ self new
		evaluate: aString
		cue: cue
		ifFail: [^ nil]
		logged: aBoolean! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 13:18'!
compile: aCue ifFail: failBlock 
	"Answer a MethodNode. If the MethodNode can not be created, notify 
	the requestor in the contxt. If the requestor is nil, evaluate failBlock 
	instead. The MethodNode is the root  of a parse tree. It can be told 
	to generate a CompiledMethod to be installed in the method dictionary 
	of the class specified by the context."
	
	self setCue: aCue.
	self source: cue source.
	^self
		translate: sourceStream
		noPattern: false
		ifFail: failBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:06'!
evaluate: textOrStream cue: aCue ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	a method. Finally, the compiled method is invoked from here via 	withArgs:executeMethod:, hence the system no longer creates Doit method
	litter on errors."

	| methodNode method value toLog itsSelection itsSelectionString |
	self setCue: aCue.
	self source: textOrStream.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].

	method := self interactive
				ifTrue: [methodNode generateWithTempNames]
				ifFalse: [methodNode generate].

	value := cue receiver
				withArgs: (cue context ifNil: [#()] ifNotNil: [{cue context}])
				executeMethod: method.

	logFlag ifTrue:
		[toLog := ((cue requestor respondsTo: #selection)  
			and:[(itsSelection := cue requestor selection) notNil
			and:[(itsSelectionString := itsSelection asString) isEmptyOrNil not]])
				ifTrue:[itsSelectionString]
				ifFalse:[sourceStream contents].
		SystemChangeNotifier uniqueInstance evaluated: toLog context: cue context].
	^ value
! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:20'!
setCue: aCue
	cue := aCue.
	
	"Set legacy instance variables for methods that don't use cue yet."
	requestor := cue requestor.
	class := cue getClass.
	category := cue category.
	context := cue context.! !
!Compiler methodsFor: 'private' stamp: 'cwp 6/19/2012 21:58'!
source: textOrStream
	sourceStream := (textOrStream isKindOf: PositionableStream)
		ifTrue: [ textOrStream ]
		ifFalse: [ ReadStream on: textOrStream asString ]! !

"Compiler"!
!SmartRefStream class methodsFor: 'i/o' stamp: 'cwp 6/20/2012 17:42'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!SmartRefStream methodsFor: 'read write' stamp: 'cwp 6/20/2012 17:41'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!ImageSegment methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:23'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!PseudoClass methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| aString newTranslations assoc currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[Locale
		currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:26'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| newTranslations currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[| aString assoc |
	Locale currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!ObjectScanner methodsFor: 'scanning' stamp: 'cwp 6/20/2012 17:39'!
scanFrom: aByteStream environment: anEnvironment
	"This should probably be reimplemented using an environment
	for compilation. For now, don't change anything"
	^ self scanFrom: aByteStream! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 09:16'!
bindingOf: varName ifAbsent: aBlock
	"SystemDictionary includes Symbols only"
	^super bindingOf: varName asSymbol ifAbsent: aBlock! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 15:48'!
undeclared
	^ self at: #Undeclared! !

"System"!
!ExceptionTests methodsFor: 'testing-outer' stamp: 'fbs 1/1/2013 22:14' prior: 40840955!
expectedFailures
	^ #().! !

"Tests"!

ReleaseBuilder subclass: #ReleaseBuilderFor4dot5
	instanceVariableNames: ''
	classVariableNames: ''
	poolDictionaries: ''
	category: 'ReleaseBuilder'!
!ReleaseBuilderFor4dot5 commentStamp: 'fbs 1/1/2013 20:25' prior: 0!
The release builder for Squeak 4.5!
!ReleaseBuilder class methodsFor: 'scripts' stamp: 'fbs 12/31/2012 20:43'!
transferCurrentPackagesAsUser: username password: password
	"Copy the packages currently loaded in the image from the trunk repository to my releaseRepository."
	| trunkRep releaseRep |
	trunkRep := self trunkRepository.
	releaseRep := self releaseRepository
		user: username;
		password: password;
		yourself.
	MCWorkingCopy allManagers do:
		[ : eachWorkingCopy | eachWorkingCopy ancestors do:
			[ : eachVersionInfo | (releaseRep includesVersionNamed: eachVersionInfo versionName) ifFalse:
				[ (trunkRep versionWithInfo: eachVersionInfo)
					ifNil: [ Warning signal: eachVersionInfo name , ' not found in ', trunkRep ]
					ifNotNilDo: [ : ver | releaseRep storeVersion: ver ] ] ] ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
openWelcomeWorkspaces
	TheWorldMainDockingBar instance
		showWelcomeText: #squeakUserInterface
		label: 'Squeak User Interface'
		in: (40 @ 40 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #workingWithSqueak
		label: 'Working With Squeak'
		in: (80 @ 80 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #licenseInformation
		label: 'License Information'
		in: (120 @ 120 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeFutureDirections
		label: 'Future Directions'
		in: (160 @ 160 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeToSqueak
		label: 'Welcome to Squeak 4.5'
		in: (200 @ 200 extent: 500 @ 300)! !
!ReleaseBuilderFor4dot5 class methodsFor: 'scripts' stamp: 'fbs 1/1/2013 20:22'!
prepareNewBuild
	super prepareNewBuild.
	MCMockPackageInfo initialize.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:24'!
releaseRepository
	"At release time, change 'trunk' to 'squeak45'."
	^ MCHttpRepository
		location: 'http://source.squeak.org/trunk'
		user: 'squeak'
		password: 'squeak'! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:22'!
setDisplayExtent: extent
"Uncomment next line when the primitives become available in the Squeak VM."
"	DisplayScreen hostWindowSize: extent."
	Display extent = extent ifFalse: [ Warning signal: 'Display extent not set to ', extent ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
setPreferences
	Preferences 
		installBrightWindowColors ;
		setPreference: #scrollBarsWithoutMenuButton toValue: true ;
		setPreference: #swapMouseButtons toValue: true ;
		setPreference: #annotationPanes toValue: true ;
		setPreference: #showSplitterHandles toValue: false ;
		setPreference: #showBoundsInHalo toValue: true ;
		setPreference: #alternateHandlesLook toValue: false ;
		setPreference: #roundedMenuCorners toValue: false ;
		setPreference: #roundedWindowCorners toValue: false.
	PluggableButtonMorph roundedButtonCorners: false.
	FillInTheBlankMorph roundedDialogCorners: false.
	Workspace shouldStyle: false.
	NetNameResolver enableIPv6: true.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
switchToNewRepository
	| old44Repository |
	MCMcmUpdater defaultUpdateURL: self releaseRepository description.
	old44Repository := MCRepositoryGroup default repositories
				detect: [:each | each description includesSubString: 'squeak44'] ifNone: [nil].
	old44Repository
		ifNotNil: [MCRepositoryGroup default removeRepository: old44Repository].
	MCRepositoryGroup default addRepository: self releaseRepository! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
versionString
	^ 'Squeak4.5'.! !

ReleaseBuilder class removeSelector: #transferCurrentPackages!

"ReleaseBuilder"!
!Environment class methodsFor: 'as yet unclassified' stamp: 'cwp 1/1/2013 18:52' prior: 40834114!
initialize
	self install! !

"Environments"!
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:59' prior: 52081878!
initPattern: aString notifying: req return: aBlock

	| result |
	self
		init: (ReadStream on: aString asString)
		cue: (CompilationCue source: aString requestor: req)
		failBlock: [^nil].
	encoder := self.
	result := aBlock value: (self pattern: false inContext: nil).
	encoder := failBlock := nil.  "break cycles"
	^result! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:01' prior: 34175471!
parse: sourceStream class: class category: aCategory noPattern: noPattern context: aContext notifying: req ifFail: aBlock 
	| c |
	c := CompilationCue
			source: sourceStream
			context: aContext
			class: class
			category: aCategory
			requestor: req.
	^ self 
		parse: sourceStream 
		cue: c 
		noPattern: noPattern 
		ifFail: aBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:11' prior: 34183963!
evaluate: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	 a method. If aContext is not nil, the text can refer to temporaries in that
	 context (the Debugger uses this). If aRequestor is not nil, then it will receive
	 a notify:at: message before the attempt to evaluate is aborted. Finally, the 
	 compiled method is invoked from here via withArgs:executeMethod:, hence
	 the system no longer creates Doit method litter on errors."
	
	| theClass |
	theClass := ((aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class).
	self setCue: (CompilationCue
		source: textOrStream
		context: aContext
		receiver: receiver
		class: theClass
		environment: theClass environment
		category: nil
		requestor: aRequestor).
	^ self evaluate: textOrStream cue: cue ifFail: failBlock logged: logFlag! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:17' prior: 34185488!
from: textOrStream class: aClass classified: aCategory context: aContext notifying: req
	self source: textOrStream.
	self setCue: 
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			category: aCategory
			requestor: req)! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:55' prior: 50781309!
from: textOrStream class: aClass context: aContext notifying: req
	self source: textOrStream.
	self setCue:
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			requestor: req)
! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/27/2012 09:41' prior: 50996506!
init: aClass context: aContext notifying: anObject
	| c |
	c := CompilationCue
		context: aContext 
		class: aClass 
		requestor: nil.
	self init: c notifying: anObject! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:58' prior: 39061698!
temps: tempVars literals: lits class: cl 
	"Initialize this encoder for decompilation."

	self setCue: (CompilationCue class: cl).
	supered := false.
	nTemps := tempVars size.
	tempVars do: [:node | scopeTable at: node name put: node].
	literalStream := WriteStream on: (Array new: lits size).
	literalStream nextPutAll: lits.
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32.! !

"Compiler"!
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:48' prior: 36026010!
addClassVarName: aString 
	"Add the argument, aString, as a class variable of the receiver.
	Signal an error if the first character of aString is not capitalized,
	or if it is already a variable named in the class."
	| symbol oldState |
	oldState := self copy.
	aString first canBeGlobalVarInitial
		ifFalse: [^self error: aString, ' class variable name should be capitalized; proceed to include anyway.'].
	symbol := aString asSymbol.
	self withAllSubclasses do: 
		[:subclass | 
		(self canFindWithoutEnvironment: symbol) ifTrue: [
			(DuplicateVariableError new)
				superclass: superclass; "fake!!!!!!"
				variable: aString;
				signal: aString, ' is already defined']].
	classPool == nil ifTrue: [classPool := Dictionary new].
	(classPool includesKey: symbol) ifFalse: 
		["Pick up any refs in Undeclared"
		classPool declare: symbol from: environment undeclared.
		SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: oldState to: self]! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:48' prior: 54782024!
bindingOf: varName
	^ self bindingOf: varName environment: self environment! !
!Class methodsFor: 'organization' stamp: 'cwp 6/25/2012 18:25' prior: 54785804!
category
	"Answer the system organization category for the receiver. First check whether the
	category name stored in the ivar is still correct and only if this fails look it up
	(latter is much more expensive)"

	category ifNotNil: [ :symbol |
		((self environment organization listAtCategoryNamed: symbol) includes: self name)
			ifTrue: [ ^symbol ] ].
	category := self environment organization categoryOfElement: self name.
	^category! !
!Class methodsFor: 'initialize-release' stamp: 'cwp 6/22/2012 15:49' prior: 36027730!
declare: varString 
	"Declare class variables common to all instances. Answer whether 
	recompilation is advisable."

	| newVars conflicts |
	
	newVars := 
		(Scanner new scanFieldNames: varString)
			collect: [:x | x asSymbol].
	newVars do:
		[:var | var first canBeGlobalVarInitial
			ifFalse: [self error: var, ' class variable name should be capitalized; proceed to include anyway.']].
	conflicts := false.
	classPool == nil 
		ifFalse: [(classPool keys reject: [:x | newVars includes: x]) do: 
					[:var | self removeClassVarName: var]].
	(newVars reject: [:var | self classPool includesKey: var])
		do: [:var | "adding"
			"check if new vars defined elsewhere"
			(self canFindWithoutEnvironment: var) ifTrue: [
					(DuplicateVariableError new)
						superclass: superclass; "fake!!!!!!"
						variable: var;
						signal: var, ' is already defined'.
					conflicts := true]].
	newVars size > 0
		ifTrue: 
			[classPool := self classPool.
			"in case it was nil"
			newVars do: [:var | classPool declare: var from: environment undeclared]].
	^conflicts! !
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:49' prior: 54802475!
removeClassVarName: aString 
	"Remove the class variable whose name is the argument, aString, from 
	the names defined in the receiver, a class. Create an error notification if 
	aString is not a class variable or if it is still being used in the code of 
	the class."

	| aSymbol |
	aSymbol := aString asSymbol.
	(classPool includesKey: aSymbol)
		ifFalse: [^self error: aString, ' is not a class variable'].
	self withAllSubclasses do:[:subclass |
		(Array with: subclass with: subclass class) do:[:classOrMeta |
			(classOrMeta whichSelectorsReferTo: (classPool associationAt: aSymbol))
				isEmpty ifFalse: [
					InMidstOfFileinNotification signal ifTrue: [
						Transcript cr; show: self name, ' (' , aString , ' is Undeclared) '.
						^ environment undeclared declare: aSymbol from: classPool].
					(self confirm: (aString,' is still used in code of class ', classOrMeta name,
						'.\Is it okay to move it to Undeclared?') withCRs)
						ifTrue:[^Undeclared declare: aSymbol from: classPool]
						ifFalse:[^self]]]].
	classPool removeKey: aSymbol.
	classPool isEmpty ifTrue: [classPool := nil].
! !
!Class methodsFor: 'class name' stamp: 'cwp 6/22/2012 15:49' prior: 54796206!
rename: aString 
	"The new name of the receiver is the argument, aString."

	| oldName newName |
	(newName := aString asSymbol) = (oldName := self name)
		ifTrue: [^ self].
	(self environment includesKey: newName)
		ifTrue: [^ self error: newName , ' already exists'].
	(environment undeclared includesKey: newName)
		ifTrue: [self inform: 'There are references to, ' , aString printString , '
from Undeclared. Check them after this change.'].
	name := newName.
	self environment renameClass: self from: oldName! !
!ClassBuilder methodsFor: 'class definition' stamp: 'cwp 6/22/2012 01:05' prior: 39054430!
name: className inEnvironment: env subclassOf: newSuper type: type instanceVariableNames: instVarString classVariableNames: classVarString poolDictionaries: poolString category: category unsafe: unsafe
	"Define a new class in the given environment.
	If unsafe is true do not run any validation checks.
	This facility is provided to implement important system changes."
	| oldClass instVars classVars copyOfOldClass newClass |
 
	environ := env.
	instVars := Scanner new scanFieldNames: instVarString.
	classVars := (Scanner new scanFieldNames: classVarString) collect: [:x | x asSymbol].

	"Validate the proposed name"
	unsafe ifFalse:[(self validateClassName: className) ifFalse:[^nil]].
	oldClass := env at: className ifAbsent:[nil].
	oldClass isBehavior 
		ifFalse: [oldClass := nil]  "Already checked in #validateClassName:"
		ifTrue: [
			copyOfOldClass := oldClass copy.
			copyOfOldClass superclass addSubclass: copyOfOldClass].
	
	
	[ | newCategory needNew force organization oldCategory |
	unsafe ifFalse:[
		"Run validation checks so we know that we have a good chance for recompilation"
		(self validateSuperclass: newSuper forSubclass: oldClass) ifFalse:[^nil].
		(self validateInstvars: instVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateClassvars: classVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateSubclassFormat: type from: oldClass forSuper: newSuper extra: instVars size) ifFalse:[^nil]].

	"See if we need a new subclass"
	needNew := self needsSubclassOf: newSuper type: type instanceVariables: instVars from: oldClass.
	needNew == nil ifTrue:[^nil]. "some error"

	(needNew and:[unsafe not]) ifTrue:[
		"Make sure we don't redefine any dangerous classes"
		(self tooDangerousClasses includes: oldClass name) ifTrue:[
			self error: oldClass name, ' cannot be changed'.
		].
		"Check if the receiver should not be redefined"
		(oldClass ~~ nil and:[oldClass shouldNotBeRedefined]) ifTrue:[
			self notify: oldClass name asText allBold, 
						' should not be redefined. \Proceed to store over it.' withCRs]].

	needNew ifTrue:[
		"Create the new class"
		newClass := self 
			newSubclassOf: newSuper 
			type: type 
			instanceVariables: instVars
			from: oldClass.
		newClass == nil ifTrue:[^nil]. "Some error"
		newClass setName: className.
		newClass environment: environ.
	] ifFalse:[
		"Reuse the old class"
		newClass := oldClass.
	].

	"Install the class variables and pool dictionaries... "
	force := (newClass declare: classVarString) | (newClass sharing: poolString).

	"... classify ..."
	newCategory := category asSymbol.
	organization := environ ifNotNil:[environ organization].
	oldClass isNil ifFalse: [oldCategory := (organization categoryOfElement: oldClass name) asSymbol].
	organization classify: newClass name under: newCategory suppressIfDefault: true.
	
	"... recompile ..."
	newClass := self recompile: force from: oldClass to: newClass mutate: false.

	"... export if not yet done ..."
	(environ at: newClass name ifAbsent:[nil]) == newClass ifFalse:[
		[environ at: newClass name put: newClass]
			on: AttemptToWriteReadOnlyGlobal do:[:ex| ex resume: true].
		environ flushClassNameCache.
	].


	newClass doneCompiling.
	"... notify interested clients ..."
	oldClass isNil ifTrue: [
		SystemChangeNotifier uniqueInstance classAdded: newClass inCategory: newCategory.
		^ newClass].
	newCategory ~= oldCategory 
		ifTrue: [SystemChangeNotifier uniqueInstance class: newClass recategorizedFrom: oldCategory to: category]
		ifFalse: [SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: copyOfOldClass to: newClass.].
] ensure: 
		[copyOfOldClass ifNotNil: [copyOfOldClass superclass removeSubclass: copyOfOldClass].
		Behavior flushObsoleteSubclasses.
		].
	^newClass! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 22:57' prior: 18572019!
superclass: newSuper
	subclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat 
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class."
	| env |
	env := EnvironmentRequest signal ifNil: [newSuper environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: newSuper
		type: newSuper typeOfClass
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:01' prior: 50629912!
superclass: aClass
	variableByteSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable byte-sized nonpointer variables."
	| oldClassOrNil actualType env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a byte subclass of a class with named fields'].
	(aClass isVariable and: [aClass isWords])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with word fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with pointer fields'].
	oldClassOrNil := aClass environment at: t ifAbsent:[nil].
	actualType := (oldClassOrNil notNil
				   and: [oldClassOrNil typeOfClass == #compiledMethod])
					ifTrue: [#compiledMethod]
					ifFalse: [#bytes].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: actualType
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:03' prior: 18573442!
superclass: aClass
	variableSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable pointer variables."
	
	| env |
	aClass isBits ifTrue: 
		[^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #variable
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18574098!
superclass: aClass
	variableWordSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable word-sized nonpointer variables."
	| env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a word subclass of a class with named fields'].
	(aClass isVariable and: [aClass isBytes])
		ifTrue: [^self error: 'cannot make a word subclass of a class with byte fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a word subclass of a class with pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #words
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18575028!
superclass: aClass
	weakSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class (the receiver) in which the subclass is to 
	have weak indexable pointer variables."
	| env |
	aClass isBits 
		ifTrue: [^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #weak
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !

"Kernel"!
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 59135029!
ambiguousSelector: aString inRange: anInterval
	| correctedSelector userSelection offset intervalWithOffset |
	
	self interactive ifFalse: [
		"In non interactive mode, compile with backward comapatibility: $- is part of literal argument"
		Transcript cr; store: encoder classEncoding; nextPutAll:#'>>';store: encoder selector; show: ' would send ' , token , '-'.
		^super ambiguousSelector: aString inRange: anInterval].
	
	"handle the text selection"
	userSelection := cue requestor selectionInterval.
	intervalWithOffset := anInterval first + requestorOffset to: anInterval last + requestorOffset.
	cue requestor selectFrom: intervalWithOffset first to: intervalWithOffset last.
	cue requestor select.

	"Build the menu with alternatives"
	correctedSelector := AmbiguousSelector 
			signalName: aString
			inRange: intervalWithOffset.
	correctedSelector ifNil: [^self fail].

	"Execute the selected action"
	offset := self substituteWord: correctedSelector wordInterval: intervalWithOffset offset: 0.
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + offset.
	token := (correctedSelector readStream upTo: Character space) asSymbol! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 38558136!
collectTemporaryDeclarationsFrom: methodNode
	| tempsMarks str |
	tempsMarks := OrderedCollection new.
	str := cue requestor text asString.
	methodNode accept: (ParseNodeEnumerator
		ofBlock: [ :aNode | 
			| mark |
			(aNode class canUnderstand: #tempsMark) 
				ifTrue: 
					[mark := aNode tempsMark.
					(mark notNil and: [ mark between: 1 and: str size ] and: [ (str at: mark) = $| ])
						ifTrue: [ tempsMarks addLast: aNode ]]]).
	(tempsMark notNil and: [ tempsMark between: 1 and: str size ] and: [ (str at: tempsMark) = $| ])
						ifTrue: [ tempsMarks addLast: self ].
	^ tempsMarks sorted: [ :a :b | a tempsMark > b tempsMark ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 52096606!
correctSelector: proposedKeyword wordIntervals: spots exprInterval: expInt ifAbort: abortAction
	"Correct the proposedKeyword to some selector symbol, correcting the original text if such action is indicated.  abortAction is invoked if the proposedKeyword couldn't be converted into a valid selector.  Spots is an ordered collection of intervals within the test stream of the for each of the keyword parts."

	| correctSelector userSelection |
	"If we can't ask the user, assume that the keyword will be defined later"
	self interactive ifFalse: [^proposedKeyword asSymbol].

	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spots first first to: spots last last.
	cue requestor select.

	correctSelector := UnknownSelector name: proposedKeyword.
	correctSelector ifNil: [^abortAction value].

	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.

	self substituteSelector: correctSelector keywords wordIntervals: spots.
	^(proposedKeyword last ~~ $:
	   and: [correctSelector last == $:])
		ifTrue: [abortAction value]
		ifFalse: [correctSelector]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 33907242!
correctVariable: proposedVariable interval: spot 
	"Correct the proposedVariable to a known variable, or declare it as a new
	variable if such action is requested.  We support declaring lowercase
	variables as temps or inst-vars, and uppercase variables as Globals or 
	ClassVars, depending on whether the context is nil (class=UndefinedObject).
	Spot is the interval within the test stream of the variable.
	rr 3/4/2004 10:26 : adds the option to define a new class. "

	"Check if this is an i-var, that has been corrected already (ugly)"

	"Display the pop-up menu"

	| binding userSelection action |
	(encoder classEncoding instVarNames includes: proposedVariable) ifTrue: 
		[^InstanceVariableNode new 
			name: proposedVariable
			index: (encoder classEncoding allInstVarNames indexOf: proposedVariable)].

	"If we can't ask the user for correction, make it undeclared"
	self interactive ifFalse: [^encoder undeclared: proposedVariable].

	"First check to see if the requestor knows anything about the variable"
	(binding := cue requestor bindingOf: proposedVariable)
		ifNotNil: [^encoder global: binding name: proposedVariable].
	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spot first to: spot last.
	cue requestor select.

	"Build the menu with alternatives"
	action := UndeclaredVariable 
				signalFor: self
				name: proposedVariable
				inRange: spot.
	action ifNil: [^self fail].

	"Execute the selected action"
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.
	^action value! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:19' prior: 34172921!
declareUndeclaredTemps: methodNode
	"Declare any undeclared temps, declaring them at the smallest enclosing scope."

	| undeclared userSelection blocksToVars |
	(undeclared := encoder undeclaredTemps) isEmpty ifTrue:
		[^self].
	userSelection := cue requestor selectionInterval.
	blocksToVars := IdentityDictionary new.
	undeclared do:
		[:var|
		(blocksToVars
			at: (var tag == #method
					ifTrue: [methodNode block]
					ifFalse: [methodNode accept: (VariableScopeFinder new ofVariable: var)])
			ifAbsentPut: [SortedCollection new]) add: var name].
	(blocksToVars removeKey: methodNode block ifAbsent: []) ifNotNil:
		[:rootVars|
		rootVars do: [:varName| self pasteTempAtMethodLevel: varName]].
	(blocksToVars keys sorted: [:a :b| a tempsMark < b tempsMark]) do:
		[:block| | decl |
		decl := (blocksToVars at: block) reduce: [:a :b| a, ' ', b].
		block temporaries isEmpty
			ifTrue:
				[self substituteWord: ' | ', decl, ' |'
					wordInterval: (block tempsMark + 1 to: block tempsMark)
					offset: requestorOffset]
			ifFalse:
				[self substituteWord: decl, ' '
					wordInterval: (block tempsMark to: block tempsMark - 1)
					offset: requestorOffset]].
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + requestorOffset.
	ReparseAfterSourceEditing signal! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 11:45' prior: 37183770!
defineClass: className 
	"prompts the user to define a new class,  
	asks for it's category, and lets the users edit further  
	the definition"
	| sym cat def d2 |
	sym := className asSymbol.
	cat := UIManager default request: 'Enter class category : ' initialAnswer: self encoder classEncoding theNonMetaClass category.
	cat
		ifEmpty: [cat := 'Unknown'].
	def := 'Object subclass: #' , sym , '
		instanceVariableNames: '''' 
		classVariableNames: ''''
		poolDictionaries: ''''
		category: ''' , cat , ''''.
	d2 := UIManager default request: 'Edit class definition : ' initialAnswer: def.
	d2
		ifEmpty: [d2 := def].
	Compiler evaluate: d2.
	^ encoder
		global: (cue environment bindingOf: sym)
		name: sym! !
!Parser methodsFor: 'primitives' stamp: 'cwp 12/27/2012 11:46' prior: 37184567!
externalFunctionDeclaration
	"Parse the function declaration for a call to an external library."
	| descriptorClass callType modifier retType externalName args argType module fn |
	descriptorClass := cue environment
		valueOf: #ExternalFunction 
		ifAbsent: [^ false].
	callType := descriptorClass callingConventionFor: here.
	callType == nil ifTrue:[^false].
	[modifier := descriptorClass callingConventionModifierFor: token.
	 modifier notNil] whileTrue:
		[self advance.
		 callType := callType bitOr: modifier].
	"Parse return type"
	self advance.
	retType := self externalType: descriptorClass.
	retType == nil ifTrue:[^self expected:'return type'].
	"Parse function name or index"
	externalName := here.
	(self match: #string) 
		ifTrue:[externalName := externalName asSymbol]
		ifFalse:[(self match:#number) ifFalse:[^self expected:'function name or index']].
	(self matchToken: #'(') ifFalse:[^self expected:'argument list'].
	args := WriteStream on: Array new.
	[here == #')'] whileFalse:[
		argType := self externalType: descriptorClass.
		argType == nil ifTrue:[^self expected:'argument'].
		argType isVoid & argType isPointerType not ifFalse:[args nextPut: argType].
	].
	(self matchToken: #')') ifFalse:[^self expected:')'].
	(self matchToken: 'module:') ifTrue:[
		module := here.
		(self match: #string) ifFalse:[^self expected: 'String'].
		module := module asSymbol].
	Smalltalk at: #ExternalLibraryFunction ifPresent:[:xfn|
		fn := xfn name: externalName 
				module: module 
				callType: callType
				returnType: retType
				argumentTypes: args contents.
		self allocateLiteral: fn.
	].
	(self matchToken: 'error:')
		ifTrue:
			[| errorCodeVariable |
			 errorCodeVariable := here.
			(hereType == #string
			 or: [hereType == #word]) ifFalse:[^self expected: 'error code (a variable or string)'].
			 self advance.
			 self addPragma: (Pragma keyword: #primitive:error: arguments: (Array with: 120 with: errorCodeVariable)).
			 fn ifNotNil: [fn setErrorCodeName: errorCodeVariable]]
		ifFalse:
			[self addPragma: (Pragma keyword: #primitive: arguments: #(120))].
	^true
! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:19' prior: 58306169!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:22' prior: 58137223!
notify: string at: location
	cue requestor isNil
		ifTrue: [(encoder == self or: [encoder isNil]) ifTrue: [^ self fail  "failure setting up syntax error"].
				SyntaxErrorNotification
					inClass: encoder classEncoding
					category: cue category
					withCode: 
						(source contents asText
							copyReplaceFrom: location
							to: location - 1
							with: ((string , ' ->') asText allBold 
								addAttribute: TextColor red; yourself))
					doitFlag: doitFlag
					errorMessage: string
					location: location]
		ifFalse: [cue requestor
					notify: string , ' ->'
					at: location
					in: source].
	^self fail! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:17' prior: 34177108!
pasteTempAtMethodLevel: name
	| insertion delta theTextString characterBeforeMark |

	theTextString := cue requestor text string.
	characterBeforeMark := theTextString at: tempsMark-1 ifAbsent: [$ ].
	(theTextString at: tempsMark) = $| ifTrue: [
  		"Paste it before the second vertical bar"
		insertion := name, ' '.
		characterBeforeMark isSeparator ifFalse: [insertion := ' ', insertion].
		delta := 0.
	] ifFalse: [
		"No bars - insert some with CR, tab"
		insertion := '| ' , name , ' |',String cr.
		delta := 2.	"the bar and CR"
		characterBeforeMark = Character tab ifTrue: [
			insertion := insertion , String tab.
			delta := delta + 1.	"the tab"
		].
	].
	tempsMark := tempsMark +
		(self substituteWord: insertion
			wordInterval: (tempsMark to: tempsMark-1)
			offset: 0) - delta! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:16' prior: 52095305!
queryUndefined
	| varStart varName | 
	varName := parseNode key.
	varStart := self endOfLastToken + requestorOffset - varName size + 1.
	cue requestor selectFrom: varStart to: varStart + varName size - 1; select.
	(UndefinedVariable name: varName) ifFalse: [^ self fail]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38599341!
removeEmptyTempDeclarationsFrom: methodNode

	| sourceCode madeChanges tempsMarkHolder |
	sourceCode := cue requestor text asString.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	madeChanges := false.
	tempsMarkHolder do: [ :currentBlock | | tempsMarkChar0 tempsMarkChar1 tempsMarkChar2 end start |
		tempsMarkChar0 := (sourceCode at: currentBlock tempsMark).
		tempsMarkChar1 := (sourceCode at: currentBlock tempsMark - 1).
		tempsMarkChar2 := (sourceCode at: currentBlock tempsMark - 2).
		tempsMarkChar0 = $| & tempsMarkChar1 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 1].
		tempsMarkChar0 = $| & tempsMarkChar1 = $  & tempsMarkChar2 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 2].
		
		start notNil & end notNil ifTrue: [
			| lineStart lineEnd |
			lineStart := 1 + (sourceCode 
				lastIndexOf: Character cr 
				startingAt: start - 1
				ifAbsent: [ 0 ]).
			lineEnd := sourceCode 
				indexOf: Character cr
				startingAt: end + 1
				ifAbsent: [ sourceCode size ].
			((sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: lineStart) >= start 
				and: [ (sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: end + 1) > lineEnd ]) ifTrue: [
					start := lineStart.
					end := lineEnd ].
			cue requestor correctFrom: start to: end with: ''.
			madeChanges := true.
			currentBlock tempsMark: nil ] ].
	madeChanges ifTrue: [ReparseAfterSourceEditing signal]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38561281!
removeUnusedTemporaryNamed: temp from: str lookingAt: currentBlock movingTempMarksOf: someBlocks

	| start end |
	end := currentBlock tempsMark - 1.
	["Beginning at right temp marker..."
	start := end - temp size + 1.
	end < temp size or: [ (str at: start) = $| ]
		or: [ temp = (str copyFrom: start to: end) 
			and: [ ((str at: start - 1) = $| | (str at: start - 1) isSeparator) 
				& ((str at: end + 1) = $| | (str at: end + 1) isSeparator) ] ]]
		whileFalse: [ 
			"Search left for the unused temp"
			end := cue requestor nextTokenFrom: end direction: -1 ].
	(end < temp size or: [ (str at: start) = $| ])
		ifFalse: 
			[(str at: start - 1) = $ 
				ifTrue: [ start := start - 1 ].
			cue requestor correctFrom: start to: end with: ''.
			someBlocks do: [ :aBlock | aBlock tempsMark: aBlock tempsMark - (end - start + 1)].
			^true ].
	^false! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 38562194!
removeUnusedTemps: methodNode
	"Scan for unused temp names, and prompt the user about the prospect of removing each one found"

	| madeChanges tempsMarkHolder unusedTempNames tempMarkHoldersToChange |
	madeChanges := false.
	tempMarkHoldersToChange := OrderedCollection new.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	unusedTempNames := encoder unusedTempNames select: 
		[ :temp | (encoder lookupVariable: temp ifAbsent: [ ]) isUndefTemp 
				and: [ self queriedUnusedTemporaries at: temp ifAbsentPut: [UnusedVariable name: temp] ]].
	tempsMarkHolder do: [ :currentBlock | 
		tempMarkHoldersToChange add: currentBlock.
		unusedTempNames do: 
			[ :temp |
			(self 
				removeUnusedTemporaryNamed: temp 
				from: cue requestor text asString 
				lookingAt: currentBlock
				movingTempMarksOf: tempMarkHoldersToChange) ifTrue: [ madeChanges := true ]]].
	madeChanges
		ifTrue: [ self removeEmptyTempDeclarationsFrom: methodNode.
			ReparseAfterSourceEditing signal ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 34179326!
substituteWord: correctWord wordInterval: spot offset: o
	"Substitute the correctSelector into the (presumed interactive) receiver.
	 Update requestorOffset based on the delta size and answer the updated offset."

	cue requestor correctFrom: spot first + o to: spot last + o with: correctWord.
	requestorOffset := requestorOffset + correctWord size - spot size.
	^o + correctWord size - spot size! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34179807!
temporaries
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'
! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34180638!
temporariesIn: methodSelector
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance in: methodSelector)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:11' prior: 53971863!
compiledMethodFor: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code
	 into a method, and answers it.  If receiver is not nil, then the text can
	 refer to instance variables of that receiver (the Inspector uses this).
	 If aContext is not nil, the text can refer to temporaries in that context
	 (the Debugger uses this). If aRequestor is not nil, then it will receive a 
	 notify:at: message before the attempt to evaluate is aborted."

	| methodNode method theClass |
	theClass := (aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class.
	self from: textOrStream class: theClass context: aContext notifying: aRequestor.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].
	method := self interactive ifTrue: [ 	methodNode generateWithTempNames ] 
		ifFalse: [methodNode generate].
		
	logFlag ifTrue:
		[SystemChangeNotifier uniqueInstance evaluated: sourceStream contents context: aContext].
	^method! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:33' prior: 34363593!
format: aStream noPattern: noPattern ifFail: failBlock
	^(self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]) preen! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 10:08' prior: 58306325!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Compiler methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:10' prior: 50779387!
notify: aString at: location
	"Refer to the comment in Object|notify:."

	^ cue requestor == nil
		ifTrue: [SyntaxErrorNotification
					inClass: cue getClass
					category: cue category
					withCode: 
						(sourceStream contents
							copyReplaceFrom: location
							to: location - 1
							with: aString)
					doitFlag: false
					errorMessage: aString
					location: location]
		ifFalse: [cue requestor
					notify: aString
					at: location
					in: sourceStream]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:34' prior: 50777201!
parse: textOrStream in: aClass notifying: req
	"Compile the argument, textOrStream, with respect to the class, aClass, and
	 answer the MethodNode that is the root of the resulting parse tree.  Notify the
	 argument, req, if an error occurs. The failBlock is defaulted to an empty block."

	self from: textOrStream class: aClass context: nil notifying: req.
	^self parser
		parse: sourceStream
		cue: cue
		noPattern: false
		ifFail: []! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:09' prior: 36332471!
parser

	parser ifNil: [parser := (cue getClass ifNil: [self class]) newParser].
	^parser! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:37' prior: 50780779!
translate: aStream noPattern: noPattern ifFail: failBlock
	^self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:37' prior: 19124095!
translate: aStream noPattern: noPattern ifFail: failBlock parser: parser
	| tree |
	tree := parser
			parse: aStream
			cue: cue 
			noPattern: noPattern
			ifFail: [^ failBlock value].
	^ tree! !
!Encoder methodsFor: 'results' stamp: 'cwp 12/27/2012 10:26' prior: 50999892!
associationForClass
	| assoc |
	assoc := self environment associationAt: cue getClass name ifAbsent: [nil].
	^assoc value == cue getClass
		ifTrue: [assoc]
		ifFalse: [Association new value: cue getClass]! !
!Encoder methodsFor: 'temps' stamp: 'cwp 12/27/2012 10:25' prior: 20148386!
bindTemp: name in: methodSelector
	"Declare a temporary; error not if a field or class variable."
	scopeTable at: name ifPresent:[:node|
		"When non-interactive raise the error only if its a duplicate"
		(node isTemp or:[requestor interactive])
			ifTrue:[^self notify:'Name is already defined']
			ifFalse:[Transcript 
				show: '(', name, ' is shadowed in "' , cue getClass printString , '>>' , methodSelector printString , '")']].
	^self reallyBind: name! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:25' prior: 20149084!
classEncoding
	"This is a hack so that the parser may findout what class it was parsing for when it wants to create a syntax error view."
	^ cue getClass! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:39' prior: 20138819!
encodeLiteral: object

	^self
		name: object
		key: (cue  literalScannedAs: object notifying: self)
		class: LiteralNode
		type: LdLitType
		set: litSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 20139010!
encodeSelector: aSelector

	^self
		name: aSelector
		key: aSelector
		class: SelectorNode
		type: SendType
		set: selectorSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 58545123!
environment
	"Answer the environment of the current compilation context,
	 be it in a class or global (e.g. a workspace)"
	^cue environment! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 11:41' prior: 50994497!
lookupInPools: varName ifFound: assocBlock

	^Symbol
		hasInterned: varName
		ifTrue:
			[:sym|
			(cue bindingOf: sym)
				ifNil: [^false]
				ifNotNil: [:assoc| assocBlock value: assoc]]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 51004306!
possibleNamesFor: proposedName
	| results |
	results := cue getClass 
		possibleVariablesFor: proposedName 
		continuedFrom: nil.
	^ proposedName correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 50995012!
possibleVariablesFor: proposedVariable

	| results |
	results := proposedVariable correctAgainstDictionary: scopeTable
								continuedFrom: nil.
	proposedVariable first canBeGlobalVarInitial ifTrue:
		[ results := cue getClass possibleVariablesFor: proposedVariable
						continuedFrom: results ].
	^ proposedVariable correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:42' prior: 51002830!
undeclared: name
	| sym |
	requestor interactive ifTrue:
		[requestor requestor == #error: ifTrue:
			[requestor error: 'Undeclared'].
		 ^self notify: 'Undeclared'].
	"Allow knowlegeable clients to squash the undeclared warning if they want (e.g.
	 Diffing pretty printers that are simply formatting text).  As this breaks
	 compilation it should only be used by clients that want to discard the result
	 of the compilation.  To squash the warning use e.g.
		[Compiler format: code in: class notifying: nil decorated: false]
			on: UndeclaredVariableWarning
			do: [:ex| ex resume: false]"
	sym := name asSymbol.
	^(UndeclaredVariableWarning new name: name selector: selector class: cue getClass) signal
		ifTrue:
			[| undeclared |
			undeclared := cue environment undeclared.
			undeclared at: sym put: nil.
			self global: (undeclared associationAt: sym) name: sym]
		ifFalse:
			[self global: (Association key: sym) name: sym]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:23' prior: 51006007!
warnAboutShadowed: name
	requestor addWarning: name,' is shadowed'.
	selector ifNotNil:
		[Transcript cr; show: cue getClass name,'>>', selector, '(', name,' is shadowed)']! !

"Compiler"!
!SmalltalkImage methodsFor: 'housekeeping' stamp: 'cwp 6/22/2012 15:56' prior: 58497062!
cleanOutUndeclared 
	globals undeclared removeUnreferencedKeys! !
!SmalltalkImage methodsFor: 'special objects' stamp: 'cwp 6/22/2012 09:01' prior: 40515090!
recreateSpecialObjectsArray
	"Smalltalk recreateSpecialObjectsArray"
	
	"To external package developers:
	**** DO NOT OVERRIDE THIS METHOD.  *****
	If you are writing a plugin and need additional special object(s) for your own use, 
	use addGCRoot() function and use own, separate special objects registry "
	
	"The Special Objects Array is an array of objects used by the Squeak virtual machine.
	 Its contents are critical and accesses to it by the VM are unchecked, so don't even
	 think of playing here unless you know what you are doing."
	| newArray |
	newArray := Array new: 56.
	"Nil false and true get used throughout the interpreter"
	newArray at: 1 put: nil.
	newArray at: 2 put: false.
	newArray at: 3 put: true.
	"This association holds the active process (a ProcessScheduler)"
	newArray at: 4 put: (self bindingOf: #Processor).
	"Numerous classes below used for type checking and instantiation"
	newArray at: 5 put: Bitmap.
	newArray at: 6 put: SmallInteger.
	newArray at: 7 put: ByteString.
	newArray at: 8 put: Array.
	newArray at: 9 put: Smalltalk.
	newArray at: 10 put: Float.
	newArray at: 11 put: MethodContext.
	newArray at: 12 put: BlockContext.
	newArray at: 13 put: Point.
	newArray at: 14 put: LargePositiveInteger.
	newArray at: 15 put: Display.
	newArray at: 16 put: Message.
	newArray at: 17 put: CompiledMethod.
	newArray at: 18 put: (self specialObjectsArray at: 18).
	"(low space Semaphore)"
	newArray at: 19 put: Semaphore.
	newArray at: 20 put: Character.
	newArray at: 21 put: #doesNotUnderstand:.
	newArray at: 22 put: #cannotReturn:.
	newArray at: 23 put: nil. "This is the process signalling low space."
	"An array of the 32 selectors that are compiled as special bytecodes,
	 paired alternately with the number of arguments each takes."
	newArray at: 24 put: #(	#+ 1 #- 1 #< 1 #> 1 #<= 1 #>= 1 #= 1 #~= 1
							#* 1 #/ 1 #\\ 1 #@ 1 #bitShift: 1 #// 1 #bitAnd: 1 #bitOr: 1
							#at: 1 #at:put: 2 #size 0 #next 0 #nextPut: 1 #atEnd 0 #== 1 #class 0
							#blockCopy: 1 #value 0 #value: 1 #do: 1 #new 0 #new: 1 #x 0 #y 0 ).
	"An array of the 255 Characters in ascii order.
	 Cog inlines table into machine code at: prim so do not regenerate it."
	newArray at: 25 put: (self specialObjectsArray at: 25).
	newArray at: 26 put: #mustBeBoolean.
	newArray at: 27 put: ByteArray.
	newArray at: 28 put: Process.
	"An array of up to 31 classes whose instances will have compact headers"
	newArray at: 29 put: self compactClassesArray.
	newArray at: 30 put: (self specialObjectsArray at: 30). "(delay Semaphore)"
	newArray at: 31 put: (self specialObjectsArray at: 31). "(user interrupt Semaphore)"
	"Entries 32 - 34 unreferenced. Previously these contained prototype instances to be copied for fast initialization"
	newArray at: 32 put: nil. "was (Float new: 2)"
	newArray at: 33 put: nil. "was (LargePositiveInteger new: 4)"
	newArray at: 34 put: nil. "was Point new"
	newArray at: 35 put: #cannotInterpret:.
	"Note: This must be fixed once we start using context prototypes (yeah, right)"
	"(MethodContext new: CompiledMethod fullFrameSize)."
	newArray at: 36 put: (self specialObjectsArray at: 36). "Is the prototype MethodContext (unused by the VM)"
	newArray at: 37 put: BlockClosure.
	"(BlockContext new: CompiledMethod fullFrameSize)."
	newArray at: 38 put: (self specialObjectsArray at: 38). "Is the prototype BlockContext (unused by the VM)"
	"array of objects referred to by external code"
	newArray at: 39 put: (self specialObjectsArray at: 39).	"preserve external semaphores"
	newArray at: 40 put: nil. "Reserved for Mutex in Cog VMs"
	newArray at: 41 put: nil. "Reserved for a LinkedList instance for overlapped calls in CogMT"
	"finalization Semaphore"
	newArray at: 42 put: ((self specialObjectsArray at: 42) ifNil: [Semaphore new]).
	newArray at: 43 put: LargeNegativeInteger.
	"External objects for callout.
	 Note: Written so that one can actually completely remove the FFI."
	newArray at: 44 put: (self at: #ExternalAddress ifAbsent: []).
	newArray at: 45 put: (self at: #ExternalStructure ifAbsent: []).
	newArray at: 46 put: (self at: #ExternalData ifAbsent: []).
	newArray at: 47 put: (self at: #ExternalFunction ifAbsent: []).
	newArray at: 48 put: (self at: #ExternalLibrary ifAbsent: []).
	newArray at: 49 put: #aboutToReturn:through:.
	newArray at: 50 put: #run:with:in:.
	"51 reserved for immutability message"
	"newArray at: 51 put: #attemptToAssign:withIndex:."
	newArray at: 52 put: #(nil "nil => generic error" #'bad receiver'
							#'bad argument' #'bad index'
							#'bad number of arguments'
							#'inappropriate operation'  #'unsupported operation'
							#'no modification' #'insufficient object memory'
							#'insufficient C memory' #'not found' #'bad method'
							#'internal error in named primitive machinery'
							#'object may move').
	"53 to 55 are for Alien"
	newArray at: 53 put: (self at: #Alien ifAbsent: []).
	newArray at: 54 put: #invokeCallback:stack:registers:jmpbuf:.
	newArray at: 55 put: (self at: #UnsafeAlien ifAbsent: []).

	"Weak reference finalization"
	newArray at: 56 put: (self at: #WeakFinalizationList ifAbsent: []).

	"Now replace the interpreter's reference in one atomic operation"
	self specialObjectsArray becomeForward: newArray
	! !
!SmalltalkImage methodsFor: 'shrinking' stamp: 'cwp 6/22/2012 15:57' prior: 37288071!
unloadAllKnownPackages
	"Unload all packages we know how to unload and reload"

	"Prepare unloading"
	Smalltalk zapMVCprojects.
	Flaps disableGlobalFlaps: false.
	StandardScriptingSystem removeUnreferencedPlayers.
	Project removeAllButCurrent.
	#('Morphic-UserObjects' 'EToy-UserObjects' 'Morphic-Imported' )
		do: [:each | SystemOrganization removeSystemCategory: each].
	Smalltalk at: #ServiceRegistry ifPresent:[:aClass|
		SystemChangeNotifier uniqueInstance
			noMoreNotificationsFor: aClass.
	].
	World removeAllMorphs.

	"Go unloading"
	#(	'ReleaseBuilder' 'ScriptLoader'
		'311Deprecated' '39Deprecated'
		'Universes' 'SMLoader' 'SMBase' 'Installer-Core'
		'VersionNumberTests' 'VersionNumber'
		'Services-Base' 'PreferenceBrowser' 'Nebraska'
		'ToolBuilder-MVC' 'ST80'
		'CollectionsTests' 'GraphicsTests' 'KernelTests'  'MorphicTests' 
		'MultilingualTests' 'NetworkTests' 'ToolsTests' 'TraitsTests'
		'SystemChangeNotification-Tests' 'FlexibleVocabularies' 
		'EToys' 'Protocols' 'XML-Parser' 'Tests' 'SUnitGUI'
		'Help-Squeak' 'HelpSystem' 'SystemReporter'
	) do: [:pkgName| 
			(MCPackage named: pkgName) unload.
			MCMcmUpdater disableUpdatesOfPackage: pkgName.
			].
	"Traits use custom unload"
	Smalltalk at: #Trait ifPresent:[:aClass| aClass unloadTraits].

	"Post-unload cleanup"
	MCWorkingCopy flushObsoletePackageInfos.
	SystemOrganization removeSystemCategory: 'UserObjects'.
	Presenter defaultPresenterClass: nil.
	World dumpPresenter.
	ScheduledControllers := nil.
	Preferences removePreference: #allowEtoyUserCustomEvents.
	SystemOrganization removeEmptyCategories.
	ChangeSet removeChangeSetsNamedSuchThat:[:cs | (cs == ChangeSet current) not].
	globals undeclared removeUnreferencedKeys.
	StandardScriptingSystem initialize.
	MCFileBasedRepository flushAllCaches.
	MCDefinition clearInstances.
	Behavior flushObsoleteSubclasses.
	ChangeSet current clear.
	ChangeSet current name: 'Unnamed1'.
	Smalltalk flushClassNameCache.
	Smalltalk at: #Browser ifPresent:[:br| br initialize].
	DebuggerMethodMap voidMapCache.
	DataStream initialize.
	AppRegistry removeObsolete.
	FileServices removeObsolete.
	Preferences removeObsolete.
	TheWorldMenu removeObsolete.
	Smalltalk garbageCollect.
	Symbol compactSymbolTable.
	TheWorldMainDockingBar updateInstances.
	MorphicProject defaultFill: (Color gray: 0.9).
	World color: (Color gray: 0.9).
! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34' prior: 40472775!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:27' prior: 40496770!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!SystemDictionary methodsFor: 'dictionary access' stamp: 'cwp 6/22/2012 15:58' prior: 30574136!
at: aKey put: anObject 
	"Override from Dictionary to check Undeclared and fix up
	references to undeclared variables."
	| index element |
	(self includesKey: aKey) ifFalse: 
		[self declare: aKey from: (self at: #Undeclared).
		self flushClassNameCache].
	super at: aKey put: anObject.
	^ anObject! !

"System"!


From noreply at buildbot.pypy.org  Tue Jun 25 19:01:44 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 25 Jun 2013 19:01:44 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: accept only bytestrings, not unicode
Message-ID: <20130625170144.9C42C1C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64978:bb292772f94a
Date: 2013-06-25 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/bb292772f94a/

Log:	accept only bytestrings, not unicode

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -322,8 +322,11 @@
         return i
 
 
- at unwrap_spec(s=str)
-def loads(space, s):
+def loads(space, w_s):
+    if space.isinstance_w(w_s, space.w_unicode):
+        raise OperationError(space.w_TypeError,
+                             space.wrap("Expected utf8-encoded str, got unicode"))
+    s = space.str_w(w_s)
     decoder = JSONDecoder(space, s)
     w_res = decoder.decode_any(0)
     i = decoder.skip_whitespace(decoder.pos)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -15,6 +15,11 @@
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._fastjson": True}
 
+    def test_raise_on_unicode(self):
+        import _fastjson
+        raises(TypeError, _fastjson.loads, u"42")
+
+
     def test_decode_constants(self):
         import _fastjson
         assert _fastjson.loads('null') is None
@@ -143,3 +148,4 @@
         assert _fastjson.loads('[1, 2]') == [1, 2]
         raises(ValueError, "_fastjson.loads('[1: 2]')")
         raises(ValueError, "_fastjson.loads('[1, 2')")
+

From noreply at buildbot.pypy.org  Tue Jun 25 19:01:45 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 25 Jun 2013 19:01:45 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: handle surrogate pairs
Message-ID: <20130625170145.C1B711C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64979:72939f271dd6
Date: 2013-06-25 18:37 +0200
http://bitbucket.org/pypy/pypy/changeset/72939f271dd6/

Log:	handle surrogate pairs

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -311,16 +311,29 @@
         i += 4
         hexdigits = self.getslice(start, i)
         try:
-            uchr = unichr(int(hexdigits, 16))
+            val = int(hexdigits, 16)
+            if val & 0xfc00 == 0xd800:
+                # surrogate pair
+                val = self.decode_surrogate_pair(i, val)
+                i += 6
         except ValueError:
             self._raise("Invalid \uXXXX escape (char %d)", i-1)
             return # help the annotator to know that we'll never go beyond
                    # this point
         #
+        uchr = unichr(val)
         utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
         builder.append(utf8_ch)
         return i
 
+    def decode_surrogate_pair(self, i, highsurr):
+        if self.ll_chars[i] != '\\' or self.ll_chars[i+1] != 'u':
+            self._raise("Unpaired high surrogate at char %d", i)
+        i += 2
+        hexdigits = self.getslice(i, i+4)
+        lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
+        return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
+
 
 def loads(space, w_s):
     if space.isinstance_w(w_s, space.w_unicode):
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -149,3 +149,8 @@
         raises(ValueError, "_fastjson.loads('[1: 2]')")
         raises(ValueError, "_fastjson.loads('[1, 2')")
 
+    def test_big_unicode_decode(self):
+        import _fastjson
+        expected = u'z\U0001d120x'
+        res = _fastjson.loads('"z\\ud834\\udd20x"')
+        assert res == expected

From noreply at buildbot.pypy.org  Tue Jun 25 19:01:47 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 25 Jun 2013 19:01:47 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: use the _fastjson speedup, if present
Message-ID: <20130625170147.2D8501C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64980:b0370b823d98
Date: 2013-06-25 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/b0370b823d98/

Log:	use the _fastjson speedup, if present

diff --git a/lib-python/2.7/json/__init__.py b/lib-python/2.7/json/__init__.py
--- a/lib-python/2.7/json/__init__.py
+++ b/lib-python/2.7/json/__init__.py
@@ -105,6 +105,12 @@
 
 __author__ = 'Bob Ippolito <bob at redivi.com>'
 
+try:
+    # PyPy speedup, the interface is different than CPython's _json
+    import _fastjson
+except ImportError:
+    _fastjson = None
+
 from .decoder import JSONDecoder
 from .encoder import JSONEncoder
 
@@ -241,7 +247,6 @@
 _default_decoder = JSONDecoder(encoding=None, object_hook=None,
                                object_pairs_hook=None)
 
-
 def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None,
         parse_int=None, parse_constant=None, object_pairs_hook=None, **kw):
     """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
@@ -323,7 +328,10 @@
     if (cls is None and encoding is None and object_hook is None and
             parse_int is None and parse_float is None and
             parse_constant is None and object_pairs_hook is None and not kw):
-        return _default_decoder.decode(s)
+        if _fastjson and not isinstance(s, unicode):
+            return _fastjson.loads(s)
+        else:
+            return _default_decoder.decode(s)
     if cls is None:
         cls = JSONDecoder
     if object_hook is not None:

From noreply at buildbot.pypy.org  Tue Jun 25 19:01:48 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 25 Jun 2013 19:01:48 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: text and fix for the + sign
Message-ID: <20130625170148.4DB691C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r64981:89c55c007418
Date: 2013-06-25 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/89c55c007418/

Log:	text and fix for the + sign

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -152,6 +152,8 @@
         if self.ll_chars[i] == '-':
             sign = -1
             i += 1
+        elif self.ll_chars[i] == '+':
+            i += 1
         i, intval, _ = self.parse_digits(i)
         return i, sign * intval
 
@@ -314,8 +316,8 @@
             val = int(hexdigits, 16)
             if val & 0xfc00 == 0xd800:
                 # surrogate pair
+                i += 6
                 val = self.decode_surrogate_pair(i, val)
-                i += 6
         except ValueError:
             self._raise("Invalid \uXXXX escape (char %d)", i-1)
             return # help the annotator to know that we'll never go beyond
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -111,6 +111,7 @@
         check('42E0', 42.0)
         check('42E3', 42000.0)
         check('42E-1', 4.2)
+        check('42E+1', 420.0)
         check('42.123E3', 42123.0)
 
     def test_decode_numeric_invalid(self):
@@ -125,7 +126,6 @@
         error('12E')
         error('12E-')
 
-
     def test_decode_object(self):
         import _fastjson
         assert _fastjson.loads('{}') == {}
@@ -154,3 +154,4 @@
         expected = u'z\U0001d120x'
         res = _fastjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
+

From noreply at buildbot.pypy.org  Tue Jun 25 19:28:41 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Jun 2013 19:28:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Ignore __pycache__ directories
Message-ID: <20130625172841.996DA1C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r64982:b1419a3f5663
Date: 2013-06-25 10:28 -0700
http://bitbucket.org/pypy/pypy/changeset/b1419a3f5663/

Log:	Ignore __pycache__ directories

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -6,6 +6,7 @@
 .idea
 .project
 .pydevproject
+__pycache__
 
 syntax: regexp
 ^testresult$

From noreply at buildbot.pypy.org  Tue Jun 25 21:36:40 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 25 Jun 2013 21:36:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: optimize unicode(some_unicode) to just
 return some_unicode
Message-ID: <20130625193640.6A3EE1C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64983:e2f91e30de83
Date: 2013-06-25 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/e2f91e30de83/

Log:	optimize unicode(some_unicode) to just return some_unicode

diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -25,6 +25,12 @@
         w_str = self.space.wrap(u'abcd')
         assert self.space.listview_unicode(w_str) == list(u"abcd")
 
+    def test_new_shortcut(self):
+        space = self.space
+        w_uni = self.space.wrap(u'abcd')
+        w_new = space.call_method(
+                space.w_unicode, "__new__", space.w_unicode, w_uni)
+        assert w_new is w_uni
 
 class AppTestUnicodeStringStdOnly:
     def test_compares(self):
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -296,13 +296,16 @@
     encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
     # convoluted logic for the case when unicode subclass has a __unicode__
     # method, we need to call this method
-    if (space.is_w(space.type(w_obj), space.w_unicode) or
+    is_precisely_unicode = space.is_w(space.type(w_obj), space.w_unicode)
+    if (is_precisely_unicode or
         (space.isinstance_w(w_obj, space.w_unicode) and
          space.findattr(w_obj, space.wrap('__unicode__')) is None)):
         if encoding is not None or errors is not None:
             raise OperationError(space.w_TypeError,
                                  space.wrap('decoding Unicode is not supported'))
         w_value = w_obj
+        if is_precisely_unicode and space.is_w(w_unicodetype, space.w_unicode):
+            return w_value
     else:
         if encoding is None and errors is None:
             w_value = unicode_from_object(space, w_obj)

From noreply at buildbot.pypy.org  Tue Jun 25 21:36:41 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 25 Jun 2013 21:36:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20130625193641.C2BFA1C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r64984:666a4888acee
Date: 2013-06-25 21:34 +0200
http://bitbucket.org/pypy/pypy/changeset/666a4888acee/

Log:	merge

diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -25,6 +25,12 @@
         w_str = self.space.wrap(u'abcd')
         assert self.space.listview_unicode(w_str) == list(u"abcd")
 
+    def test_new_shortcut(self):
+        space = self.space
+        w_uni = self.space.wrap(u'abcd')
+        w_new = space.call_method(
+                space.w_unicode, "__new__", space.w_unicode, w_uni)
+        assert w_new is w_uni
 
 class AppTestUnicodeStringStdOnly:
     def test_compares(self):
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -296,13 +296,16 @@
     encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
     # convoluted logic for the case when unicode subclass has a __unicode__
     # method, we need to call this method
-    if (space.is_w(space.type(w_obj), space.w_unicode) or
+    is_precisely_unicode = space.is_w(space.type(w_obj), space.w_unicode)
+    if (is_precisely_unicode or
         (space.isinstance_w(w_obj, space.w_unicode) and
          space.findattr(w_obj, space.wrap('__unicode__')) is None)):
         if encoding is not None or errors is not None:
             raise OperationError(space.w_TypeError,
                                  space.wrap('decoding Unicode is not supported'))
         w_value = w_obj
+        if is_precisely_unicode and space.is_w(w_unicodetype, space.w_unicode):
+            return w_value
     else:
         if encoding is None and errors is None:
             w_value = unicode_from_object(space, w_obj)

From noreply at buildbot.pypy.org  Tue Jun 25 22:20:23 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Jun 2013 22:20:23 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: test, implement complex.round()
Message-ID: <20130625202023.BCDCA1C34F2@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64986:1b5d8b97c785
Date: 2013-06-25 22:08 +0300
http://bitbucket.org/pypy/pypy/changeset/1b5d8b97c785/

Log:	test, implement complex.round()

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -347,6 +347,8 @@
         bround = b.round()
         assert (bround == [1., 0.]).all()
         assert bround.dtype is dtype('float16')
+        c = array([10.5+11.5j, -15.2-100.3456j, 0.2343+11.123456j])
+        assert (c.round(0) == [10.+12.j, -15-100j, 0+11j]).all()
 
 
     def test_copysign(self):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1378,6 +1378,15 @@
         except ZeroDivisionError:
             return rfloat.NAN, rfloat.NAN
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        ans = list(self.unbox(v))
+        if isfinite(ans[0]):
+            ans[0] = rfloat.round_double(ans[0], decimals,  half_even=True)
+        if isfinite(ans[1]):
+            ans[1] = rfloat.round_double(ans[1], decimals,  half_even=True)
+        return self.box_complex(*ans)
+
     # No floor, ceil, trunc in numpy for complex
     #@simple_unary_op
     #def floor(self, v):

From noreply at buildbot.pypy.org  Tue Jun 25 22:20:22 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Jun 2013 22:20:22 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: implement ndarray.round,
 add failing tests for scalar.round
Message-ID: <20130625202022.9214A1C34F1@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64985:c6d01b824d87
Date: 2013-06-25 21:48 +0300
http://bitbucket.org/pypy/pypy/changeset/c6d01b824d87/

Log:	implement ndarray.round, add failing tests for scalar.round

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -561,13 +561,23 @@
     @unwrap_spec(decimals=int)
     def descr_round(self, space, decimals=0, w_out=None):
         if space.is_none(w_out):
-            w_out = None
+            if self.get_dtype().is_bool_type():
+                #numpy promotes bool.round() to float16. Go figure.
+                w_out = W_NDimArray.from_shape(self.get_shape(),
+                       interp_dtype.get_dtype_cache(space).w_float16dtype)
+            else:
+                w_out = None
         elif not isinstance(w_out, W_NDimArray):
             raise OperationError(space.w_TypeError, space.wrap(
                 "return arrays must be of ArrayType"))
         out = interp_dtype.dtype_agreement(space, [self], self.get_shape(),
                                            w_out)
-        loop.round(space, self, self.get_shape(), decimals, out)
+        if out.get_dtype().is_bool_type() and self.get_dtype().is_bool_type():
+            calc_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
+        else:
+            calc_dtype = out.get_dtype()
+
+        loop.round(space, self, calc_dtype, self.get_shape(), decimals, out)
         return out
 
     def descr_searchsorted(self, space, w_v, w_side='left'):
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -12,7 +12,6 @@
 from pypy.module.micronumpy.iter import PureShapeIterator
 from pypy.module.micronumpy import constants
 from pypy.module.micronumpy.support import int_w
-from rpython.rlib.rfloat import round_double
 
 call2_driver = jit.JitDriver(name='numpy_call2',
                              greens = ['shapelen', 'func', 'calc_dtype',
@@ -576,16 +575,14 @@
 round_driver = jit.JitDriver(greens = ['shapelen', 'dtype'],
                                     reds = 'auto')
 
-def round(space, arr, shape, decimals, out):
+def round(space, arr, dtype, shape, decimals, out):
     arr_iter = arr.create_iter(shape)
-    dtype = out.get_dtype()
     shapelen = len(shape)
     out_iter = out.create_iter(shape)
     while not arr_iter.done():
         round_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
-        #w_v = dtype.itemtype.round(arr_iter.getitem().convert_to(dtype),
-        #             decimals)
-        w_v = arr_iter.getitem().convert_to(dtype)
+        w_v = dtype.itemtype.round(arr_iter.getitem().convert_to(dtype),
+                     decimals)
         out_iter.setitem(w_v)
         arr_iter.next()
         out_iter.next()
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -21,3 +21,19 @@
 
         a = zeros(3)
         assert loads(dumps(sum(a))) == sum(a)
+
+    def test_round(self):
+        from numpypy import int32, float64, complex128, bool
+        i = int32(1337)
+        f = float64(13.37)
+        c = complex128(13 + 37.j)
+        b = bool(0)
+        assert i.round(decimals=-2) == 1300
+        assert i.round(decimals=1) == 1337
+        assert c.round() == c
+        assert f.round() == 13.
+        assert f.round(decimals=-1) == 10.
+        assert f.round(decimals=1) == 13.4
+        exc = raises(AttributeError, 'b.round()')
+        assert exc.value[0] == "'bool' object has no attribute 'round'"
+
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -335,10 +335,18 @@
         assert all([math.copysign(1, f(-abs(float("nan")))) == -1 for f in floor, ceil, trunc])
 
     def test_round(self):
-        from numpypy import array
+        from numpypy import array, dtype
         ninf, inf = float("-inf"), float("inf")
         a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
         assert ([ninf, -1.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == a.round()).all()
+        i = array([-1000, -100, -1, 0, 1, 111, 1111, 11111], dtype=int)
+        assert (i == i.round()).all()
+        assert (i.round(decimals=4) == i).all()
+        assert (i.round(decimals=-4) == [0, 0, 0, 0, 0, 0, 0, 10000]).all()
+        b = array([True, False], dtype=bool)
+        bround = b.round()
+        assert (bround == [1., 0.]).all()
+        assert bround.dtype is dtype('float16')
 
 
     def test_copysign(self):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -37,7 +37,7 @@
         return self.box(
             func(
                 self,
-                self.for_computation(raw)
+                self.for_computation(raw),
             )
         )
     return dispatcher
@@ -521,6 +521,20 @@
             return v
         return 0
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.unbox(v)
+        if decimals < 0:
+            factor = int(10 ** -decimals)
+            #int does floor division, we want toward zero
+            if raw < 0:
+                ans = - (-raw / factor * factor)
+            else:
+                ans = raw / factor * factor
+        else:
+            ans = raw
+        return self.box(ans)
+
     @raw_unary_op
     def signbit(self, v):
         return v < 0
@@ -798,6 +812,16 @@
     def ceil(self, v):
         return math.ceil(v)
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.unbox(v)
+        if rfloat.isinf(raw):
+            return v
+        elif rfloat.isnan(raw):
+            return v
+        ans = rfloat.round_double(raw, decimals, half_even=True)
+        return self.box(ans)
+
     @simple_unary_op
     def trunc(self, v):
         if v < 0:

From noreply at buildbot.pypy.org  Tue Jun 25 22:20:25 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Jun 2013 22:20:25 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: implement round for scalars
Message-ID: <20130625202025.11EB91C34F1@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64987:737e0485a18b
Date: 2013-06-25 22:33 +0300
http://bitbucket.org/pypy/pypy/changeset/737e0485a18b/

Log:	implement round for scalars

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -235,6 +235,11 @@
         w_values = space.newtuple([self])
         return convert_to_array(space, w_values)
 
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0):
+        v = self.convert_to(self.get_dtype(space))
+        return self.get_dtype(space).itemtype.round(v, decimals)
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -501,6 +506,7 @@
     any = interp2app(W_GenericBox.descr_any),
     all = interp2app(W_GenericBox.descr_all),
     ravel = interp2app(W_GenericBox.descr_ravel),
+    round = interp2app(W_GenericBox.descr_round),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,

From noreply at buildbot.pypy.org  Tue Jun 25 22:20:26 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Jun 2013 22:20:26 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: fix translation
Message-ID: <20130625202026.727B41C34F1@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64988:0e990018147d
Date: 2013-06-25 23:05 +0300
http://bitbucket.org/pypy/pypy/changeset/0e990018147d/

Log:	fix translation

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -523,9 +523,12 @@
 
     @specialize.argtype(1)
     def round(self, v, decimals=0):
-        raw = self.unbox(v)
+        raw = self.for_computation(self.unbox(v))
         if decimals < 0:
-            factor = int(10 ** -decimals)
+            # No ** in rpython
+            factor = 1
+            for i in xrange(-decimals):
+                factor *=10
             #int does floor division, we want toward zero
             if raw < 0:
                 ans = - (-raw / factor * factor)
@@ -814,7 +817,7 @@
 
     @specialize.argtype(1)
     def round(self, v, decimals=0):
-        raw = self.unbox(v)
+        raw = self.for_computation(self.unbox(v))
         if rfloat.isinf(raw):
             return v
         elif rfloat.isnan(raw):
@@ -1380,12 +1383,12 @@
 
     @specialize.argtype(1)
     def round(self, v, decimals=0):
-        ans = list(self.unbox(v))
+        ans = list(self.for_computation(self.unbox(v)))
         if isfinite(ans[0]):
             ans[0] = rfloat.round_double(ans[0], decimals,  half_even=True)
         if isfinite(ans[1]):
             ans[1] = rfloat.round_double(ans[1], decimals,  half_even=True)
-        return self.box_complex(*ans)
+        return self.box_complex(ans[0], ans[1])
 
     # No floor, ceil, trunc in numpy for complex
     #@simple_unary_op

From noreply at buildbot.pypy.org  Tue Jun 25 23:46:23 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Jun 2013 23:46:23 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: close branch about to be merged
Message-ID: <20130625214623.BF8861C32EB@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64990:c8a81e74f66b
Date: 2013-06-26 00:44 +0300
http://bitbucket.org/pypy/pypy/changeset/c8a81e74f66b/

Log:	close branch about to be merged


From noreply at buildbot.pypy.org  Tue Jun 25 23:46:22 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Jun 2013 23:46:22 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-round: update whatsnew
Message-ID: <20130625214622.81D3C1C2FC2@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-round
Changeset: r64989:b652e52b9f52
Date: 2013-06-26 00:44 +0300
http://bitbucket.org/pypy/pypy/changeset/b652e52b9f52/

Log:	update whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -57,3 +57,4 @@
 Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
 
 .. branch: dtype-isnative
+.. branch: ndarray-round

From noreply at buildbot.pypy.org  Tue Jun 25 23:46:25 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Jun 2013 23:46:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge ndarray-round which added round()
	to ndarray
Message-ID: <20130625214625.3AE681C33FE@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r64991:79d8f7de40f1
Date: 2013-06-26 00:45 +0300
http://bitbucket.org/pypy/pypy/changeset/79d8f7de40f1/

Log:	merge ndarray-round which added round() to ndarray

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -57,3 +57,4 @@
 Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
 
 .. branch: dtype-isnative
+.. branch: ndarray-round
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -235,6 +235,11 @@
         w_values = space.newtuple([self])
         return convert_to_array(space, w_values)
 
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0):
+        v = self.convert_to(self.get_dtype(space))
+        return self.get_dtype(space).itemtype.round(v, decimals)
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -501,6 +506,7 @@
     any = interp2app(W_GenericBox.descr_any),
     all = interp2app(W_GenericBox.descr_all),
     ravel = interp2app(W_GenericBox.descr_ravel),
+    round = interp2app(W_GenericBox.descr_round),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -558,9 +558,27 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "resize not implemented yet"))
 
-    def descr_round(self, space, w_decimals=0, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "round not implemented yet"))
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0, w_out=None):
+        if space.is_none(w_out):
+            if self.get_dtype().is_bool_type():
+                #numpy promotes bool.round() to float16. Go figure.
+                w_out = W_NDimArray.from_shape(self.get_shape(),
+                       interp_dtype.get_dtype_cache(space).w_float16dtype)
+            else:
+                w_out = None
+        elif not isinstance(w_out, W_NDimArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                "return arrays must be of ArrayType"))
+        out = interp_dtype.dtype_agreement(space, [self], self.get_shape(),
+                                           w_out)
+        if out.get_dtype().is_bool_type() and self.get_dtype().is_bool_type():
+            calc_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
+        else:
+            calc_dtype = out.get_dtype()
+
+        loop.round(space, self, calc_dtype, self.get_shape(), decimals, out)
+        return out
 
     def descr_searchsorted(self, space, w_v, w_side='left'):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -975,6 +993,7 @@
     byteswap = interp2app(W_NDimArray.descr_byteswap),
     choose   = interp2app(W_NDimArray.descr_choose),
     clip     = interp2app(W_NDimArray.descr_clip),
+    round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
 
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -173,7 +173,7 @@
         iter = x_iter
     shapelen = len(shape)
     while not iter.done():
-        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, 
+        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
                                         arr_dtype=arr_dtype)
         w_cond = arr_iter.getitem()
         if arr_dtype.itemtype.bool(w_cond):
@@ -188,7 +188,7 @@
     return out
 
 axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce',
-                                    greens=['shapelen', 
+                                    greens=['shapelen',
                                             'func', 'dtype',
                                             'identity'],
                                     reds='auto')
@@ -228,7 +228,7 @@
     arg_driver = jit.JitDriver(name='numpy_' + op_name,
                                greens = ['shapelen', 'dtype'],
                                reds = 'auto')
-    
+
     def argmin_argmax(arr):
         result = 0
         idx = 1
@@ -265,7 +265,7 @@
      result.shape == [3, 5, 2, 4]
      broadcast shape should be [3, 5, 2, 7, 4]
      result should skip dims 3 which is len(result_shape) - 1
-        (note that if right is 1d, result should 
+        (note that if right is 1d, result should
                   skip len(result_shape))
      left should skip 2, 4 which is a.ndims-1 + range(right.ndims)
           except where it==(right.ndims-2)
@@ -283,9 +283,9 @@
     righti = right.create_dot_iter(broadcast_shape, right_skip)
     while not outi.done():
         dot_driver.jit_merge_point(dtype=dtype)
-        lval = lefti.getitem().convert_to(dtype) 
-        rval = righti.getitem().convert_to(dtype) 
-        outval = outi.getitem().convert_to(dtype) 
+        lval = lefti.getitem().convert_to(dtype)
+        rval = righti.getitem().convert_to(dtype)
+        outval = outi.getitem().convert_to(dtype)
         v = dtype.itemtype.mul(lval, rval)
         value = dtype.itemtype.add(v, outval).convert_to(dtype)
         outi.setitem(value)
@@ -355,7 +355,7 @@
         setitem_filter_driver.jit_merge_point(shapelen=shapelen,
                                               index_dtype=index_dtype,
                                               arr_dtype=arr_dtype,
-                                             ) 
+                                             )
         if index_iter.getitem_bool():
             arr_iter.setitem(value_iter.getitem())
             value_iter.next()
@@ -572,6 +572,21 @@
         out_iter.next()
         min_iter.next()
 
+round_driver = jit.JitDriver(greens = ['shapelen', 'dtype'],
+                                    reds = 'auto')
+
+def round(space, arr, dtype, shape, decimals, out):
+    arr_iter = arr.create_iter(shape)
+    shapelen = len(shape)
+    out_iter = out.create_iter(shape)
+    while not arr_iter.done():
+        round_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+        w_v = dtype.itemtype.round(arr_iter.getitem().convert_to(dtype),
+                     decimals)
+        out_iter.setitem(w_v)
+        arr_iter.next()
+        out_iter.next()
+
 diagonal_simple_driver = jit.JitDriver(greens = ['axis1', 'axis2'],
                                        reds = 'auto')
 
@@ -613,4 +628,4 @@
         out_iter.setitem(arr.getitem_index(space, indexes))
         iter.next()
         out_iter.next()
-       
+
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -21,3 +21,19 @@
 
         a = zeros(3)
         assert loads(dumps(sum(a))) == sum(a)
+
+    def test_round(self):
+        from numpypy import int32, float64, complex128, bool
+        i = int32(1337)
+        f = float64(13.37)
+        c = complex128(13 + 37.j)
+        b = bool(0)
+        assert i.round(decimals=-2) == 1300
+        assert i.round(decimals=1) == 1337
+        assert c.round() == c
+        assert f.round() == 13.
+        assert f.round(decimals=-1) == 10.
+        assert f.round(decimals=1) == 13.4
+        exc = raises(AttributeError, 'b.round()')
+        assert exc.value[0] == "'bool' object has no attribute 'round'"
+
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -286,7 +286,7 @@
 
         skip('sign of nan is non-determinant')
         assert (signbit([float('nan'), float('-nan'), -float('nan')]) ==
-            [False, True, True]).all()    
+            [False, True, True]).all()
 
     def test_reciprocal(self):
         from numpypy import array, reciprocal, complex64, complex128
@@ -334,6 +334,23 @@
         assert all([math.copysign(1, f(abs(float("nan")))) == 1 for f in floor, ceil, trunc])
         assert all([math.copysign(1, f(-abs(float("nan")))) == -1 for f in floor, ceil, trunc])
 
+    def test_round(self):
+        from numpypy import array, dtype
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -1.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == a.round()).all()
+        i = array([-1000, -100, -1, 0, 1, 111, 1111, 11111], dtype=int)
+        assert (i == i.round()).all()
+        assert (i.round(decimals=4) == i).all()
+        assert (i.round(decimals=-4) == [0, 0, 0, 0, 0, 0, 0, 10000]).all()
+        b = array([True, False], dtype=bool)
+        bround = b.round()
+        assert (bround == [1., 0.]).all()
+        assert bround.dtype is dtype('float16')
+        c = array([10.5+11.5j, -15.2-100.3456j, 0.2343+11.123456j])
+        assert (c.round(0) == [10.+12.j, -15-100j, 0+11j]).all()
+
+
     def test_copysign(self):
         from numpypy import array, copysign
 
@@ -364,7 +381,7 @@
             assert b[i] == res
 
     def test_exp2(self):
-        import math 
+        import math
         from numpypy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
@@ -759,8 +776,8 @@
              complex(inf, inf), complex(inf, ninf), complex(0, inf),
              complex(ninf, ninf), complex(nan, 0), complex(0, nan),
              complex(nan, nan)]
-        assert (isfinite(a) == [True, True, False, False, False, 
-                        False, False, False, False, False]).all() 
+        assert (isfinite(a) == [True, True, False, False, False,
+                        False, False, False, False, False]).all()
 
     def test_logical_ops(self):
         from numpypy import logical_and, logical_or, logical_xor, logical_not
@@ -864,7 +881,7 @@
         #numpy returns (a.real*b.real + a.imag*b.imag) / abs(b)**2
         expect = [3., -23., 1.]
         for i in range(len(a)):
-            assert b[i] == expect[i] 
+            assert b[i] == expect[i]
         b = floor_divide(a[0], 0.)
         assert math.isnan(b.real)
         assert b.imag == 0.
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -37,7 +37,7 @@
         return self.box(
             func(
                 self,
-                self.for_computation(raw)
+                self.for_computation(raw),
             )
         )
     return dispatcher
@@ -521,6 +521,23 @@
             return v
         return 0
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.for_computation(self.unbox(v))
+        if decimals < 0:
+            # No ** in rpython
+            factor = 1
+            for i in xrange(-decimals):
+                factor *=10
+            #int does floor division, we want toward zero
+            if raw < 0:
+                ans = - (-raw / factor * factor)
+            else:
+                ans = raw / factor * factor
+        else:
+            ans = raw
+        return self.box(ans)
+
     @raw_unary_op
     def signbit(self, v):
         return v < 0
@@ -798,6 +815,16 @@
     def ceil(self, v):
         return math.ceil(v)
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.for_computation(self.unbox(v))
+        if rfloat.isinf(raw):
+            return v
+        elif rfloat.isnan(raw):
+            return v
+        ans = rfloat.round_double(raw, decimals, half_even=True)
+        return self.box(ans)
+
     @simple_unary_op
     def trunc(self, v):
         if v < 0:
@@ -1354,6 +1381,15 @@
         except ZeroDivisionError:
             return rfloat.NAN, rfloat.NAN
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        ans = list(self.for_computation(self.unbox(v)))
+        if isfinite(ans[0]):
+            ans[0] = rfloat.round_double(ans[0], decimals,  half_even=True)
+        if isfinite(ans[1]):
+            ans[1] = rfloat.round_double(ans[1], decimals,  half_even=True)
+        return self.box_complex(ans[0], ans[1])
+
     # No floor, ceil, trunc in numpy for complex
     #@simple_unary_op
     #def floor(self, v):

From noreply at buildbot.pypy.org  Wed Jun 26 09:03:52 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 09:03:52 +0200 (CEST)
Subject: [pypy-commit] pypy faster-str-of-bigint: use a string builder
Message-ID: <20130626070352.150701C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-str-of-bigint
Changeset: r64993:c18866b5ecce
Date: 2013-06-25 22:21 +0200
http://bitbucket.org/pypy/pypy/changeset/c18866b5ecce/

Log:	use a string builder

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -2191,15 +2191,26 @@
 
 _FORMAT_DECIMAL_MINDIGITS = 9 # fits in 32 bits, there may be a better choice for this
 
-def _format_decimal_recursive(x, i, output, pts):
-    if i < 0: # bottomed out with min_digit sized pieces
-        if output or x.gt(NULLRBIGINT):
+def _format_decimal_recursive(x, i, output, pts, negative):
+    # bottomed out with min_digit sized pieces
+    # use str of ints
+    if i < 0:
+        # this checks whether any digit has been appended yet
+        # the extra char for negative rbigints is the '-'
+        if output.getlength() == negative:
+            if x.sign == 0:
+                pass
+            else:
+                s = str(x.toint())
+                output.append(s)
+        else:
             s = str(x.toint())
-            output.append("0"*(_FORMAT_DECIMAL_MINDIGITS - len(s)) + s) # note that this appends in inorder
+            output.append_multiple_char("0", _FORMAT_DECIMAL_MINDIGITS - len(s))
+            output.append(s)
     else:
         top,bot = x.divmod(pts[i]) # split the number
-        _format_decimal_recursive(top,i-1, output, pts)
-        _format_decimal_recursive(bot,i-1, output, pts)
+        _format_decimal_recursive(top, i-1, output, pts, negative)
+        _format_decimal_recursive(bot, i-1, output, pts, negative)
 
 def _format_decimal_new(x, addL=False):
     if x.sign == 0:
@@ -2213,17 +2224,20 @@
     two = rbigint.fromint(2)
 
     pts = [ten.pow(rbigint.fromint(_FORMAT_DECIMAL_MINDIGITS))]
+    stringsize = _FORMAT_DECIMAL_MINDIGITS
     while pts[-1].lt(x):
         pts.append(pts[-1].pow(two))
+        stringsize *= 2
     pts.pop() # remove first 10**2**i greater than x
-    output = []
 
-    _format_decimal_recursive(x,len(pts)-1, output, pts)
-    # strip leading zeros, we can probably do this more elegantly
-    output = "".join(output)
-    while len(output) > 1 and output[0] == "0":
-        output = output[1:]
-    return "-" * negative + output + "L" * addL
+    output = StringBuilder(stringsize)
+    if negative:
+        output.append('-')
+    _format_decimal_recursive(x,len(pts)-1, output, pts, negative)
+
+    if addL:
+        output.append('L')
+    return output.build()
 
 
 def _bitwise(a, op, b): # '&', '|', '^'

From noreply at buildbot.pypy.org  Wed Jun 26 09:03:50 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 09:03:50 +0200 (CEST)
Subject: [pypy-commit] pypy faster-str-of-bigint: preliminary working version
Message-ID: <20130626070350.BE0B71C00F4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-str-of-bigint
Changeset: r64992:d70fc9b223e1
Date: 2013-06-25 21:33 +0200
http://bitbucket.org/pypy/pypy/changeset/d70fc9b223e1/

Log:	preliminary working version

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -434,11 +434,11 @@
 
     @jit.elidable
     def repr(self):
-        return _format_decimal(self, addL=True)
+        return _format_decimal_new(self, addL=True)
 
     @jit.elidable
     def str(self):
-        return _format_decimal(self)
+        return _format_decimal_new(self)
 
     @jit.elidable
     def eq(self, other):
@@ -2189,6 +2189,43 @@
     return builder.build()
 
 
+_FORMAT_DECIMAL_MINDIGITS = 9 # fits in 32 bits, there may be a better choice for this
+
+def _format_decimal_recursive(x, i, output, pts):
+    if i < 0: # bottomed out with min_digit sized pieces
+        if output or x.gt(NULLRBIGINT):
+            s = str(x.toint())
+            output.append("0"*(_FORMAT_DECIMAL_MINDIGITS - len(s)) + s) # note that this appends in inorder
+    else:
+        top,bot = x.divmod(pts[i]) # split the number
+        _format_decimal_recursive(top,i-1, output, pts)
+        _format_decimal_recursive(bot,i-1, output, pts)
+
+def _format_decimal_new(x, addL=False):
+    if x.sign == 0:
+        if addL:
+            return "0L"
+        return "0"
+    negative = x.sign < 0
+    if negative:
+        x = x.neg()
+    ten = rbigint.fromint(10)
+    two = rbigint.fromint(2)
+
+    pts = [ten.pow(rbigint.fromint(_FORMAT_DECIMAL_MINDIGITS))]
+    while pts[-1].lt(x):
+        pts.append(pts[-1].pow(two))
+    pts.pop() # remove first 10**2**i greater than x
+    output = []
+
+    _format_decimal_recursive(x,len(pts)-1, output, pts)
+    # strip leading zeros, we can probably do this more elegantly
+    output = "".join(output)
+    while len(output) > 1 and output[0] == "0":
+        output = output[1:]
+    return "-" * negative + output + "L" * addL
+
+
 def _bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -31,11 +31,14 @@
         assert rbigint.frombool(True).tolong() == 1
 
     def test_str(self):
-        for i in range(100):
-            n = 3 ** i
-            r1 = rbigint.fromlong(n)
+        n = 1
+        r1 = rbigint.fromint(1)
+        three = rbigint.fromint(3)
+        for i in range(300):
+            n *= 3
+            r1 = r1.mul(three)
             assert r1.str() == str(n)
-            r2 = rbigint.fromlong(-n)
+            r2 = r1.neg()
             assert r2.str() == str(-n)
 
     def test_floordiv(self):

From noreply at buildbot.pypy.org  Wed Jun 26 09:03:53 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 09:03:53 +0200 (CEST)
Subject: [pypy-commit] pypy faster-str-of-bigint: remove old version
Message-ID: <20130626070353.578B91C00F4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-str-of-bigint
Changeset: r64994:070cdcee14ca
Date: 2013-06-25 22:52 +0200
http://bitbucket.org/pypy/pypy/changeset/070cdcee14ca/

Log:	remove old version

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -434,11 +434,11 @@
 
     @jit.elidable
     def repr(self):
-        return _format_decimal_new(self, addL=True)
+        return _format_decimal(self, addL=True)
 
     @jit.elidable
     def str(self):
-        return _format_decimal_new(self)
+        return _format_decimal(self)
 
     @jit.elidable
     def eq(self, other):
@@ -2094,101 +2094,6 @@
     return ''.join(s[p:])
 
 
-DECIMAL_SHIFT = 0      # computed as max(E such that 10**E fits in a digit)
-while 10 ** (DECIMAL_SHIFT + 1) <= 2 ** SHIFT:
-    DECIMAL_SHIFT += 1
-DECIMAL_BASE = 10 ** DECIMAL_SHIFT
-
-# an RPython trick: this creates a nested sequence of calls that are
-# all inlined into each other, making an unrolled loop.  Moreover the
-# calls are done in the "wrong" order to be written as a regular loop:
-# the first digit that is append-ed to the builder is the most
-# significant one (corresponding to the innermost call).
-_succ = specialize.memo()(lambda n: n + 1)
- at specialize.arg(3)
-def _add_decimal_digits(builder, value, ndigits, digit_index=1):
-    assert value >= 0
-    if digit_index < ndigits:
-        assert digit_index < DECIMAL_SHIFT
-        _add_decimal_digits(builder, value // 10, ndigits, _succ(digit_index))
-        builder.append(chr(ord('0') + value % 10))
-    else:
-        assert value < 10
-        builder.append(chr(ord('0') + value))
-_add_decimal_digits._always_inline_ = True
-
-
-def _format_decimal(a, addL=False):
-    """ Optimized version of _format(a, BASE10, '', 'L' if addL else ''). """
-    if a.sign == 0:
-        if addL:
-            return "0L"
-        else:
-            return "0"
-
-    size_a = a.numdigits()
-    negative = a.sign < 0
-
-    # quick and dirty upper bound for the number of digits
-    # required to express a in base DECIMAL_BASE:
-    #
-    #    #digits = 1 + floor(log2(a) / log2(DECIMAL_BASE))
-    #
-    # But log2(a) < size_a * PyLong_SHIFT, and
-    # log2(DECIMAL_BASE) = log2(10) * DECIMAL_SHIFT
-    #                    > 3 * DECIMAL_SHIFT
-
-    size = 1 + size_a * SHIFT // (3 * DECIMAL_SHIFT)
-    pout = [NULLDIGIT] * size
-
-    # convert array of base _PyLong_BASE digits in pin to an array of
-    # base _PyLong_DECIMAL_BASE digits in pout, following Knuth (TAOCP,
-    # Volume 2 (3rd edn), section 4.4, Method 1b).
-    size = 0
-    for i in range(size_a-1, -1, -1):
-        hi = a.digit(i)
-        for j in range(size):
-            z = (_widen_digit(pout[j]) << SHIFT) | hi
-            hi = _store_digit(z // DECIMAL_BASE)
-            pout[j] = _store_digit(z - _widen_digit(hi) * DECIMAL_BASE)
-        assert hi >= 0
-        while hi:
-            pout[size] = hi % DECIMAL_BASE
-            hi //= DECIMAL_BASE
-            size += 1
-    sizem1 = size - 1
-    assert sizem1 >= 0
-
-    # calculate exact length of output string, and allocate
-    decimal_digits_in_last_part = 1
-    rem = pout[sizem1]
-    tenpow = 10
-    while rem >= tenpow:
-        tenpow *= 10
-        decimal_digits_in_last_part += 1
-    strlen = (addL + negative +
-              decimal_digits_in_last_part + (sizem1) * DECIMAL_SHIFT)
-
-    builder = StringBuilder(strlen)
-
-    # start with the negative sign, if needed
-    if negative:
-        builder.append('-')
-
-    # pout[size-1] produces 'decimal_digits_in_last_part' digits.
-    # Then the remaining pout[size-2] through pout[0] contribute exactly
-    # DECIMAL_SHIFT digits each.
-    decimal_digits = decimal_digits_in_last_part
-    for i in range(sizem1, -1, -1):
-        _add_decimal_digits(builder, pout[i], decimal_digits)
-        decimal_digits = DECIMAL_SHIFT
-
-    # done
-    if addL:
-        builder.append('L')
-    return builder.build()
-
-
 _FORMAT_DECIMAL_MINDIGITS = 9 # fits in 32 bits, there may be a better choice for this
 
 def _format_decimal_recursive(x, i, output, pts, negative):
@@ -2208,11 +2113,11 @@
             output.append_multiple_char("0", _FORMAT_DECIMAL_MINDIGITS - len(s))
             output.append(s)
     else:
-        top,bot = x.divmod(pts[i]) # split the number
+        top, bot = x.divmod(pts[i]) # split the number
         _format_decimal_recursive(top, i-1, output, pts, negative)
         _format_decimal_recursive(bot, i-1, output, pts, negative)
 
-def _format_decimal_new(x, addL=False):
+def _format_decimal(x, addL=False):
     if x.sign == 0:
         if addL:
             return "0L"

From noreply at buildbot.pypy.org  Wed Jun 26 09:03:54 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 09:03:54 +0200 (CEST)
Subject: [pypy-commit] pypy faster-str-of-bigint: generalize _format_decimal
	to _format
Message-ID: <20130626070354.865681C00F4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-str-of-bigint
Changeset: r64995:794b370b7d1c
Date: 2013-06-25 23:34 +0200
http://bitbucket.org/pypy/pypy/changeset/794b370b7d1c/

Log:	generalize _format_decimal to _format

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -434,11 +434,11 @@
 
     @jit.elidable
     def repr(self):
-        return _format_decimal(self, addL=True)
+        return self.format(BASE10, suffix="L")
 
     @jit.elidable
     def str(self):
-        return _format_decimal(self)
+        return self.format(BASE10)
 
     @jit.elidable
     def eq(self, other):
@@ -2093,58 +2093,65 @@
                      # hint for the annotator for the slice below)
     return ''.join(s[p:])
 
+_FORMAT_MINDIGITS = 5 # 36 ** 5 fits in 32 bits, there may be a better choice for this
 
-_FORMAT_DECIMAL_MINDIGITS = 9 # fits in 32 bits, there may be a better choice for this
+def _format_int(val, digits):
+    base = len(digits)
+    j = 0
+    out = []
+    while val:
+        out.append(digits[val % base])
+        val //= base
+    out.reverse()
+    return "".join(out)
 
-def _format_decimal_recursive(x, i, output, pts, negative):
+
+def _format_recursive(x, i, output, pts, digits, size_prefix):
     # bottomed out with min_digit sized pieces
     # use str of ints
     if i < 0:
         # this checks whether any digit has been appended yet
-        # the extra char for negative rbigints is the '-'
-        if output.getlength() == negative:
+        if output.getlength() == size_prefix:
             if x.sign == 0:
                 pass
             else:
-                s = str(x.toint())
+                s = _format_int(x.toint(), digits)
                 output.append(s)
         else:
-            s = str(x.toint())
-            output.append_multiple_char("0", _FORMAT_DECIMAL_MINDIGITS - len(s))
+            s = _format_int(x.toint(), digits)
+            output.append_multiple_char(digits[0], _FORMAT_MINDIGITS - len(s))
             output.append(s)
     else:
         top, bot = x.divmod(pts[i]) # split the number
-        _format_decimal_recursive(top, i-1, output, pts, negative)
-        _format_decimal_recursive(bot, i-1, output, pts, negative)
+        _format_recursive(top, i-1, output, pts, digits, size_prefix)
+        _format_recursive(bot, i-1, output, pts, digits, size_prefix)
 
-def _format_decimal(x, addL=False):
+def _format(x, digits, prefix='', suffix=''):
     if x.sign == 0:
-        if addL:
-            return "0L"
-        return "0"
+        return prefix + "0" + suffix
+    base = len(digits)
     negative = x.sign < 0
     if negative:
         x = x.neg()
-    ten = rbigint.fromint(10)
+    rbase = rbigint.fromint(base)
     two = rbigint.fromint(2)
 
-    pts = [ten.pow(rbigint.fromint(_FORMAT_DECIMAL_MINDIGITS))]
-    stringsize = _FORMAT_DECIMAL_MINDIGITS
+    pts = [rbase.pow(rbigint.fromint(_FORMAT_MINDIGITS))]
+    stringsize = _FORMAT_MINDIGITS
     while pts[-1].lt(x):
         pts.append(pts[-1].pow(two))
         stringsize *= 2
-    pts.pop() # remove first 10**2**i greater than x
+    pts.pop() # remove first base**2**i greater than x
 
     output = StringBuilder(stringsize)
     if negative:
         output.append('-')
-    _format_decimal_recursive(x,len(pts)-1, output, pts, negative)
+    output.append(prefix)
+    _format_recursive(x,len(pts)-1, output, pts, digits, output.getlength())
 
-    if addL:
-        output.append('L')
+    output.append(suffix)
     return output.build()
 
-
 def _bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 

From noreply at buildbot.pypy.org  Wed Jun 26 09:10:31 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 09:10:31 +0200 (CEST)
Subject: [pypy-commit] pypy faster-str-of-bigint: kill the generic part of
 the previous _format and salvage the special case for
Message-ID: <20130626071031.5C0D11C00F4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-str-of-bigint
Changeset: r64996:49cce796c701
Date: 2013-06-26 09:09 +0200
http://bitbucket.org/pypy/pypy/changeset/49cce796c701/

Log:	kill the generic part of the previous _format and salvage the
	special case for powers of two

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1979,10 +1979,7 @@
     Convert a bigint object to a string, using a given conversion base.
     Return a string object.
     """
-    size_a = a.numdigits()
 
-    base = len(digits)
-    assert base >= 2 and base <= 36
 
     # Compute a rough upper bound for the length of the string
     i = base
@@ -2002,8 +1999,15 @@
     if a.sign == 0:
         p -= 1
         s[p] = '0'
-    elif (base & (base - 1)) == 0:
+
+def _format_base2(a, digits, prefix, suffix):
         # JRH: special case for power-of-2 bases
+        output = StringBuilder()
+        if a.sign < 0:
+            output.append('-')
+        output.append(prefix)
+        base = len(digits)
+        size_a = a.numdigits()
         accum = 0
         accumbits = 0  # # of bits in accum
         basebits = 1   # # of bits in base-1
@@ -2019,11 +2023,10 @@
             accum |= a.widedigit(i) << accumbits
             accumbits += SHIFT
             assert accumbits >= basebits
+            out = []
             while 1:
                 cdigit = intmask(accum & (base - 1))
-                p -= 1
-                assert p >= 0
-                s[p] = digits[cdigit]
+                out.append(digits[cdigit])
                 accumbits -= basebits
                 accum >>= basebits
                 if i < size_a - 1:
@@ -2032,66 +2035,11 @@
                 else:
                     if accum <= 0:
                         break
-                        
+            out.reverse()
+            output.append("".join(out))
             i += 1
-    else:
-        # Not 0, and base not a power of 2.  Divide repeatedly by
-        # base, but for speed use the highest power of base that
-        # fits in a digit.
-        size = size_a
-        pin = a # just for similarity to C source which uses the array
-        # powbase <- largest power of base that fits in a digit.
-        powbase = _widen_digit(base)  # powbase == base ** power
-        power = 1
-        while 1:
-            newpow = powbase * base
-            if newpow >> SHIFT:  # doesn't fit in a digit
-                break
-            powbase = newpow
-            power += 1
-
-        # Get a scratch area for repeated division.
-        scratch = rbigint([NULLDIGIT] * size, 1, size)
-
-        # Repeatedly divide by powbase.
-        while 1:
-            ntostore = power
-            rem = _inplace_divrem1(scratch, pin, powbase, size)
-            pin = scratch  # no need to use a again
-            if pin._digits[size - 1] == NULLDIGIT:
-                size -= 1
-
-            # Break rem into digits.
-            assert ntostore > 0
-            while 1:
-                nextrem = rem // base
-                c = rem - nextrem * base
-                p -= 1
-                assert p >= 0
-                s[p] = digits[c]
-                rem = nextrem
-                ntostore -= 1
-                # Termination is a bit delicate:  must not
-                # store leading zeroes, so must get out if
-                # remaining quotient and rem are both 0.
-                if not (ntostore and (size or rem)):
-                    break
-            if size == 0:
-                break
-
-    j = len(prefix)
-    while j > 0:
-        p -= 1
-        j -= 1
-        s[p] = prefix[j]
-
-    if a.sign < 0:
-        p -= 1
-        s[p] = '-'
-
-    assert p >= 0    # otherwise, buffer overflow (this is also a
-                     # hint for the annotator for the slice below)
-    return ''.join(s[p:])
+        output.append(suffix)
+        return output.build()
 
 _FORMAT_MINDIGITS = 5 # 36 ** 5 fits in 32 bits, there may be a better choice for this
 
@@ -2130,6 +2078,9 @@
     if x.sign == 0:
         return prefix + "0" + suffix
     base = len(digits)
+    assert base >= 2 and base <= 36
+    if (base & (base - 1)) == 0:
+        return _format_base2(x, digits, prefix, suffix)
     negative = x.sign < 0
     if negative:
         x = x.neg()

From noreply at buildbot.pypy.org  Wed Jun 26 09:19:55 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 09:19:55 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: some typos
Message-ID: <20130626071955.BDEDC1C00F4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kill-gen-store-back-in
Changeset: r64997:e88e833654b6
Date: 2013-06-26 09:16 +0200
http://bitbucket.org/pypy/pypy/changeset/e88e833654b6/

Log:	some typos

diff --git a/pypy/doc/jit/virtualizable.rst b/pypy/doc/jit/virtualizable.rst
--- a/pypy/doc/jit/virtualizable.rst
+++ b/pypy/doc/jit/virtualizable.rst
@@ -14,7 +14,7 @@
 if we're dealing with frames, virtuals are often not good enough. Frames
 can escape and they can also be allocated already at the moment we enter the
 JIT. In such cases we need some extra object that can still be optimized away,
-despite existing on heap.
+despite existing on the heap.
 
 Solution
 --------
@@ -22,7 +22,8 @@
 We introduce virtualizables. They're objects that exist on the heap, but their
 fields are not always in sync with whatever happens in the assembler. One
 example is that virtualizable fields can store virtual objects without
-forcing them. It's very useful for frames. The definition looks like this::
+forcing them. This is very useful for frames. Declaring an object to be
+virtualizable works like this:
 
     class Frame(object):
        _virtualizable2_ = ['locals[*]', 'stackdepth']
@@ -39,16 +40,17 @@
 
 * Each array access must be with a known positive index that cannot raise
   an ``IndexError``. Using ``no = jit.hint(no, promote=True)`` might be useful
-  to get a constant-number access.
+  to get a constant-number access. This is only safe if the index is actually
+  constant or changing rarely within the context of the user's code.
 
-* If you allocate a new virtualizable in the JIT, it has to be done like this
+* If you initialize a new virtualizable in the JIT, it has to be done like this
   (for example if we're in ``Frame.__init__``)::
 
     self = hint(self, access_directly=True, fresh_virtualizable=True)
 
   that way you can populate the fields directly.
 
-* If you use virtualizable outside of the JIT - it's very expensive and
+* If you use virtualizable outside of the JIT &#8211; it's very expensive and
   sometimes aborts tracing. Consider it carefully as to how do it only for
   debugging purposes and not every time (e.g. ``sys._getframe`` call).
 

From noreply at buildbot.pypy.org  Wed Jun 26 11:41:41 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 11:41:41 +0200 (CEST)
Subject: [pypy-commit] pypy faster-str-of-bigint: reinstate the use of an
 array for the result of _format_base2_notzero
Message-ID: <20130626094141.3BA2D1C00F4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-str-of-bigint
Changeset: r64998:c10714392297
Date: 2013-06-26 11:40 +0200
http://bitbucket.org/pypy/pypy/changeset/c10714392297/

Log:	reinstate the use of an array for the result of
	_format_base2_notzero it's faster and I couldn't be bothered to find
	out how to compute the result not in reverse

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1974,59 +1974,38 @@
 BASE10 = '0123456789'
 BASE16 = '0123456789abcdef'
 
-def _format(a, digits, prefix='', suffix=''):
-    """
-    Convert a bigint object to a string, using a given conversion base.
-    Return a string object.
-    """
-
-
-    # Compute a rough upper bound for the length of the string
-    i = base
-    bits = 0
-    while i > 1:
-        bits += 1
-        i >>= 1
-    i = 5 + len(prefix) + len(suffix) + (size_a*SHIFT + bits-1) // bits
-    s = [chr(0)] * i
-    p = i
-    j = len(suffix)
-    while j > 0:
-        p -= 1
-        j -= 1
-        s[p] = suffix[j]
-
-    if a.sign == 0:
-        p -= 1
-        s[p] = '0'
-
-def _format_base2(a, digits, prefix, suffix):
+def _format_base2_notzero(a, digits, prefix='', suffix=''):
+        base = len(digits)
         # JRH: special case for power-of-2 bases
-        output = StringBuilder()
-        if a.sign < 0:
-            output.append('-')
-        output.append(prefix)
-        base = len(digits)
-        size_a = a.numdigits()
         accum = 0
         accumbits = 0  # # of bits in accum
-        basebits = 1   # # of bits in base-1
+        basebits = 0
         i = base
-        while 1:
+        while i > 1:
+            basebits += 1
             i >>= 1
-            if i <= 1:
-                break
-            basebits += 1
+
+        # Compute a rough upper bound for the length of the string
+        size_a = a.numdigits()
+        i = 5 + len(prefix) + len(suffix) + (size_a*SHIFT + basebits-1) // basebits
+        result = [chr(0)] * i
+        next_char_index = i
+        j = len(suffix)
+        while j > 0:
+            next_char_index -= 1
+            j -= 1
+            result[next_char_index] = suffix[j]
 
         i = 0
         while i < size_a:
             accum |= a.widedigit(i) << accumbits
             accumbits += SHIFT
             assert accumbits >= basebits
-            out = []
             while 1:
                 cdigit = intmask(accum & (base - 1))
-                out.append(digits[cdigit])
+                next_char_index -= 1
+                assert next_char_index >= 0
+                result[next_char_index] = digits[cdigit]
                 accumbits -= basebits
                 accum >>= basebits
                 if i < size_a - 1:
@@ -2035,11 +2014,20 @@
                 else:
                     if accum <= 0:
                         break
-            out.reverse()
-            output.append("".join(out))
             i += 1
-        output.append(suffix)
-        return output.build()
+        j = len(prefix)
+        while j > 0:
+            next_char_index -= 1
+            j -= 1
+            result[next_char_index] = prefix[j]
+
+        if a.sign < 0:
+            next_char_index -= 1
+            result[next_char_index] = '-'
+
+        assert next_char_index >= 0    # otherwise, buffer overflow (this is also a
+                         # hint for the annotator for the slice below)
+        return ''.join(result[next_char_index:])
 
 _FORMAT_MINDIGITS = 5 # 36 ** 5 fits in 32 bits, there may be a better choice for this
 
@@ -2080,7 +2068,7 @@
     base = len(digits)
     assert base >= 2 and base <= 36
     if (base & (base - 1)) == 0:
-        return _format_base2(x, digits, prefix, suffix)
+        return _format_base2_notzero(x, digits, prefix, suffix)
     negative = x.sign < 0
     if negative:
         x = x.neg()

From noreply at buildbot.pypy.org  Wed Jun 26 12:08:20 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 12:08:20 +0200 (CEST)
Subject: [pypy-commit] pypy faster-str-of-bigint: close to-be-merged branch
Message-ID: <20130626100820.4A7F61C345E@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-str-of-bigint
Changeset: r64999:85222d0ab7be
Date: 2013-06-26 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/85222d0ab7be/

Log:	close to-be-merged branch


From noreply at buildbot.pypy.org  Wed Jun 26 12:08:21 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 12:08:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge faster-str-of-bigint
Message-ID: <20130626100821.9B8951C3462@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r65000:902241cca7dc
Date: 2013-06-26 12:07 +0200
http://bitbucket.org/pypy/pypy/changeset/902241cca7dc/

Log:	merge faster-str-of-bigint

	speed up str(long) significantly, thanks to an algorithm proposed by
	Nathan Hurst on pypy-dev:

	http://mail.python.org/pipermail/pypy-dev/2013-May/011433.html

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -434,11 +434,11 @@
 
     @jit.elidable
     def repr(self):
-        return _format_decimal(self, addL=True)
+        return self.format(BASE10, suffix="L")
 
     @jit.elidable
     def str(self):
-        return _format_decimal(self)
+        return self.format(BASE10)
 
     @jit.elidable
     def eq(self, other):
@@ -1974,45 +1974,27 @@
 BASE10 = '0123456789'
 BASE16 = '0123456789abcdef'
 
-def _format(a, digits, prefix='', suffix=''):
-    """
-    Convert a bigint object to a string, using a given conversion base.
-    Return a string object.
-    """
-    size_a = a.numdigits()
-
-    base = len(digits)
-    assert base >= 2 and base <= 36
-
-    # Compute a rough upper bound for the length of the string
-    i = base
-    bits = 0
-    while i > 1:
-        bits += 1
-        i >>= 1
-    i = 5 + len(prefix) + len(suffix) + (size_a*SHIFT + bits-1) // bits
-    s = [chr(0)] * i
-    p = i
-    j = len(suffix)
-    while j > 0:
-        p -= 1
-        j -= 1
-        s[p] = suffix[j]
-
-    if a.sign == 0:
-        p -= 1
-        s[p] = '0'
-    elif (base & (base - 1)) == 0:
+def _format_base2_notzero(a, digits, prefix='', suffix=''):
+        base = len(digits)
         # JRH: special case for power-of-2 bases
         accum = 0
         accumbits = 0  # # of bits in accum
-        basebits = 1   # # of bits in base-1
+        basebits = 0
         i = base
-        while 1:
+        while i > 1:
+            basebits += 1
             i >>= 1
-            if i <= 1:
-                break
-            basebits += 1
+
+        # Compute a rough upper bound for the length of the string
+        size_a = a.numdigits()
+        i = 5 + len(prefix) + len(suffix) + (size_a*SHIFT + basebits-1) // basebits
+        result = [chr(0)] * i
+        next_char_index = i
+        j = len(suffix)
+        while j > 0:
+            next_char_index -= 1
+            j -= 1
+            result[next_char_index] = suffix[j]
 
         i = 0
         while i < size_a:
@@ -2021,9 +2003,9 @@
             assert accumbits >= basebits
             while 1:
                 cdigit = intmask(accum & (base - 1))
-                p -= 1
-                assert p >= 0
-                s[p] = digits[cdigit]
+                next_char_index -= 1
+                assert next_char_index >= 0
+                result[next_char_index] = digits[cdigit]
                 accumbits -= basebits
                 accum >>= basebits
                 if i < size_a - 1:
@@ -2032,162 +2014,82 @@
                 else:
                     if accum <= 0:
                         break
-                        
             i += 1
+        j = len(prefix)
+        while j > 0:
+            next_char_index -= 1
+            j -= 1
+            result[next_char_index] = prefix[j]
+
+        if a.sign < 0:
+            next_char_index -= 1
+            result[next_char_index] = '-'
+
+        assert next_char_index >= 0    # otherwise, buffer overflow (this is also a
+                         # hint for the annotator for the slice below)
+        return ''.join(result[next_char_index:])
+
+_FORMAT_MINDIGITS = 5 # 36 ** 5 fits in 32 bits, there may be a better choice for this
+
+def _format_int(val, digits):
+    base = len(digits)
+    j = 0
+    out = []
+    while val:
+        out.append(digits[val % base])
+        val //= base
+    out.reverse()
+    return "".join(out)
+
+
+def _format_recursive(x, i, output, pts, digits, size_prefix):
+    # bottomed out with min_digit sized pieces
+    # use str of ints
+    if i < 0:
+        # this checks whether any digit has been appended yet
+        if output.getlength() == size_prefix:
+            if x.sign == 0:
+                pass
+            else:
+                s = _format_int(x.toint(), digits)
+                output.append(s)
+        else:
+            s = _format_int(x.toint(), digits)
+            output.append_multiple_char(digits[0], _FORMAT_MINDIGITS - len(s))
+            output.append(s)
     else:
-        # Not 0, and base not a power of 2.  Divide repeatedly by
-        # base, but for speed use the highest power of base that
-        # fits in a digit.
-        size = size_a
-        pin = a # just for similarity to C source which uses the array
-        # powbase <- largest power of base that fits in a digit.
-        powbase = _widen_digit(base)  # powbase == base ** power
-        power = 1
-        while 1:
-            newpow = powbase * base
-            if newpow >> SHIFT:  # doesn't fit in a digit
-                break
-            powbase = newpow
-            power += 1
+        top, bot = x.divmod(pts[i]) # split the number
+        _format_recursive(top, i-1, output, pts, digits, size_prefix)
+        _format_recursive(bot, i-1, output, pts, digits, size_prefix)
 
-        # Get a scratch area for repeated division.
-        scratch = rbigint([NULLDIGIT] * size, 1, size)
+def _format(x, digits, prefix='', suffix=''):
+    if x.sign == 0:
+        return prefix + "0" + suffix
+    base = len(digits)
+    assert base >= 2 and base <= 36
+    if (base & (base - 1)) == 0:
+        return _format_base2_notzero(x, digits, prefix, suffix)
+    negative = x.sign < 0
+    if negative:
+        x = x.neg()
+    rbase = rbigint.fromint(base)
+    two = rbigint.fromint(2)
 
-        # Repeatedly divide by powbase.
-        while 1:
-            ntostore = power
-            rem = _inplace_divrem1(scratch, pin, powbase, size)
-            pin = scratch  # no need to use a again
-            if pin._digits[size - 1] == NULLDIGIT:
-                size -= 1
+    pts = [rbase.pow(rbigint.fromint(_FORMAT_MINDIGITS))]
+    stringsize = _FORMAT_MINDIGITS
+    while pts[-1].lt(x):
+        pts.append(pts[-1].pow(two))
+        stringsize *= 2
+    pts.pop() # remove first base**2**i greater than x
 
-            # Break rem into digits.
-            assert ntostore > 0
-            while 1:
-                nextrem = rem // base
-                c = rem - nextrem * base
-                p -= 1
-                assert p >= 0
-                s[p] = digits[c]
-                rem = nextrem
-                ntostore -= 1
-                # Termination is a bit delicate:  must not
-                # store leading zeroes, so must get out if
-                # remaining quotient and rem are both 0.
-                if not (ntostore and (size or rem)):
-                    break
-            if size == 0:
-                break
+    output = StringBuilder(stringsize)
+    if negative:
+        output.append('-')
+    output.append(prefix)
+    _format_recursive(x,len(pts)-1, output, pts, digits, output.getlength())
 
-    j = len(prefix)
-    while j > 0:
-        p -= 1
-        j -= 1
-        s[p] = prefix[j]
-
-    if a.sign < 0:
-        p -= 1
-        s[p] = '-'
-
-    assert p >= 0    # otherwise, buffer overflow (this is also a
-                     # hint for the annotator for the slice below)
-    return ''.join(s[p:])
-
-
-DECIMAL_SHIFT = 0      # computed as max(E such that 10**E fits in a digit)
-while 10 ** (DECIMAL_SHIFT + 1) <= 2 ** SHIFT:
-    DECIMAL_SHIFT += 1
-DECIMAL_BASE = 10 ** DECIMAL_SHIFT
-
-# an RPython trick: this creates a nested sequence of calls that are
-# all inlined into each other, making an unrolled loop.  Moreover the
-# calls are done in the "wrong" order to be written as a regular loop:
-# the first digit that is append-ed to the builder is the most
-# significant one (corresponding to the innermost call).
-_succ = specialize.memo()(lambda n: n + 1)
- at specialize.arg(3)
-def _add_decimal_digits(builder, value, ndigits, digit_index=1):
-    assert value >= 0
-    if digit_index < ndigits:
-        assert digit_index < DECIMAL_SHIFT
-        _add_decimal_digits(builder, value // 10, ndigits, _succ(digit_index))
-        builder.append(chr(ord('0') + value % 10))
-    else:
-        assert value < 10
-        builder.append(chr(ord('0') + value))
-_add_decimal_digits._always_inline_ = True
-
-
-def _format_decimal(a, addL=False):
-    """ Optimized version of _format(a, BASE10, '', 'L' if addL else ''). """
-    if a.sign == 0:
-        if addL:
-            return "0L"
-        else:
-            return "0"
-
-    size_a = a.numdigits()
-    negative = a.sign < 0
-
-    # quick and dirty upper bound for the number of digits
-    # required to express a in base DECIMAL_BASE:
-    #
-    #    #digits = 1 + floor(log2(a) / log2(DECIMAL_BASE))
-    #
-    # But log2(a) < size_a * PyLong_SHIFT, and
-    # log2(DECIMAL_BASE) = log2(10) * DECIMAL_SHIFT
-    #                    > 3 * DECIMAL_SHIFT
-
-    size = 1 + size_a * SHIFT // (3 * DECIMAL_SHIFT)
-    pout = [NULLDIGIT] * size
-
-    # convert array of base _PyLong_BASE digits in pin to an array of
-    # base _PyLong_DECIMAL_BASE digits in pout, following Knuth (TAOCP,
-    # Volume 2 (3rd edn), section 4.4, Method 1b).
-    size = 0
-    for i in range(size_a-1, -1, -1):
-        hi = a.digit(i)
-        for j in range(size):
-            z = (_widen_digit(pout[j]) << SHIFT) | hi
-            hi = _store_digit(z // DECIMAL_BASE)
-            pout[j] = _store_digit(z - _widen_digit(hi) * DECIMAL_BASE)
-        assert hi >= 0
-        while hi:
-            pout[size] = hi % DECIMAL_BASE
-            hi //= DECIMAL_BASE
-            size += 1
-    sizem1 = size - 1
-    assert sizem1 >= 0
-
-    # calculate exact length of output string, and allocate
-    decimal_digits_in_last_part = 1
-    rem = pout[sizem1]
-    tenpow = 10
-    while rem >= tenpow:
-        tenpow *= 10
-        decimal_digits_in_last_part += 1
-    strlen = (addL + negative +
-              decimal_digits_in_last_part + (sizem1) * DECIMAL_SHIFT)
-
-    builder = StringBuilder(strlen)
-
-    # start with the negative sign, if needed
-    if negative:
-        builder.append('-')
-
-    # pout[size-1] produces 'decimal_digits_in_last_part' digits.
-    # Then the remaining pout[size-2] through pout[0] contribute exactly
-    # DECIMAL_SHIFT digits each.
-    decimal_digits = decimal_digits_in_last_part
-    for i in range(sizem1, -1, -1):
-        _add_decimal_digits(builder, pout[i], decimal_digits)
-        decimal_digits = DECIMAL_SHIFT
-
-    # done
-    if addL:
-        builder.append('L')
-    return builder.build()
-
+    output.append(suffix)
+    return output.build()
 
 def _bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -31,11 +31,14 @@
         assert rbigint.frombool(True).tolong() == 1
 
     def test_str(self):
-        for i in range(100):
-            n = 3 ** i
-            r1 = rbigint.fromlong(n)
+        n = 1
+        r1 = rbigint.fromint(1)
+        three = rbigint.fromint(3)
+        for i in range(300):
+            n *= 3
+            r1 = r1.mul(three)
             assert r1.str() == str(n)
-            r2 = rbigint.fromlong(-n)
+            r2 = r1.neg()
             assert r2.str() == str(-n)
 
     def test_floordiv(self):

From noreply at buildbot.pypy.org  Wed Jun 26 12:09:27 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Jun 2013 12:09:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: what's new item
Message-ID: <20130626100927.84D201C00F4@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r65001:c07f3a751c7c
Date: 2013-06-26 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/c07f3a751c7c/

Log:	what's new item

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -58,3 +58,6 @@
 
 .. branch: dtype-isnative
 .. branch: ndarray-round
+
+.. branch: faster-str-of-bigint
+Improve performance of str(long).

From noreply at buildbot.pypy.org  Wed Jun 26 12:52:52 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 12:52:52 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: reremoved nojit benchmarks
Message-ID: <20130626105252.D845C1C00F4@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r476:e35f642fd164
Date: 2013-06-26 12:45 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/e35f642fd164/

Log:	reremoved nojit benchmarks

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -221,7 +221,7 @@
     "lang-smalltalk",
     executables=[
         Executable("rsqueakvm", "./targetimageloadingsmalltalk-c"),
-        Executable("rsqueakvm-nojit", "./targetimageloadingsmalltalk-nojit-c")
+        # Executable("rsqueakvm-nojit", "./targetimageloadingsmalltalk-nojit-c")
     ],
     arguments=["images/%s.image" % SqueakImage, '-m', 'runSPyBenchmarks']
 )

From noreply at buildbot.pypy.org  Wed Jun 26 12:52:55 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 12:52:55 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed a bug when suspending
 an already suspended process
Message-ID: <20130626105255.193AD1C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r478:d661709c25eb
Date: 2013-06-26 12:52 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/d661709c25eb/

Log:	fixed a bug when suspending an already suspended process

diff --git a/spyvm/wrapper.py b/spyvm/wrapper.py
--- a/spyvm/wrapper.py
+++ b/spyvm/wrapper.py
@@ -108,9 +108,10 @@
             self.store_suspended_context(w_current_frame)
             return ProcessWrapper(self.space, w_process).activate()
         else:
-            process_list = ProcessListWrapper(self.space, self.my_list())
-            process_list.remove(self._w_self)
-            self.store_my_list(self.space.w_nil)
+            if self.my_list() is not self.space.w_nil:
+                process_list = ProcessListWrapper(self.space, self.my_list())
+                process_list.remove(self._w_self)
+                self.store_my_list(self.space.w_nil)
             return w_current_frame
 
 class LinkedListWrapper(Wrapper):

From noreply at buildbot.pypy.org  Wed Jun 26 12:52:54 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 12:52:54 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed a bug when creating
 nameless classes and using them (Behavior new new)
Message-ID: <20130626105254.0031D1C0A3F@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r477:eb1af4f5f97f
Date: 2013-06-26 12:51 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/eb1af4f5f97f/

Log:	fixed a bug when creating nameless classes and using them (Behavior
	new new)

diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -187,10 +187,12 @@
             # metaclasses hold a pointer to the real class in the last
             # slot. This is pos 6 in mini.image and higher in squeak3.9
             w_realclass = w_self._fetch(w_self.size() - 1)
-            assert isinstance(w_realclass, model.W_PointersObject)
-            if w_realclass.size() > constants.CLASS_NAME_INDEX:
+            if (isinstance(w_realclass, model.W_PointersObject)
+                and w_realclass.size() > constants.CLASS_NAME_INDEX):
                 # TODO ADD TEST WHICH GOES OVER THIS PART
                 w_name = w_realclass._fetch(constants.CLASS_NAME_INDEX)
+            else:
+                return
 
         if isinstance(w_name, model.W_BytesObject):
             self.name = w_name.as_string()

From noreply at buildbot.pypy.org  Wed Jun 26 14:23:43 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Wed, 26 Jun 2013 14:23:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20130626122343.8A8631C346E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: py3k
Changeset: r65002:8a53d0ac751c
Date: 2013-06-26 14:20 +0200
http://bitbucket.org/pypy/pypy/changeset/8a53d0ac751c/

Log:	hg merge default

diff too long, truncating to 2000 out of 3644 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -6,6 +6,7 @@
 .idea
 .project
 .pydevproject
+__pycache__
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -707,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -934,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -57,3 +57,7 @@
 Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
 
 .. branch: dtype-isnative
+.. branch: ndarray-round
+
+.. branch: faster-str-of-bigint
+Improve performance of str(long).
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -5,6 +5,22 @@
 
 @unwrap_spec(type=str)
 def newdict(space, type):
+    """ newdict(type)
+
+    Create a normal dict with a special implementation strategy.
+
+    type is a string and can be:
+
+    * "module" - equivalent to some_module.__dict__
+
+    * "instance" - equivalent to an instance dict with a not-changing-much
+                   set of keys
+
+    * "kwargs" - keyword args dict equivalent of what you get from **kwargs
+                 in a function, optimized for passing around
+
+    * "strdict" - string-key only dict. This one should be chosen automatically
+    """
     if type == 'module':
         return space.newdict(module=True)
     elif type == 'instance':
@@ -18,6 +34,10 @@
                               type)
 
 def dictstrategy(space, w_obj):
+    """ dictstrategy(dict)
+
+    show the underlaying strategy used by a dict object
+    """
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError,
                              space.wrap("expecting dict object"))
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -13,7 +13,6 @@
     _mixin_ = True
 
     def __init__(self, space, name, size, align, enumerators, enumvalues):
-        name = "enum " + name
         self._super.__init__(self, space, size, name, len(name), align)
         self.enumerators2values = {}   # str -> int
         self.enumvalues2erators = {}   # int -> str
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -187,7 +187,7 @@
         else:
             extra = " *"
         self.is_file = (ctitem.name == "struct _IO_FILE" or
-                        ctitem.name == "struct $FILE")
+                        ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -25,7 +25,6 @@
     custom_field_pos = False
 
     def __init__(self, space, name):
-        name = '%s %s' % (self.kind, name)
         W_CType.__init__(self, space, -1, name, len(name))
 
     def check_complete(self, w_errorcls=None):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -641,6 +641,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -648,7 +650,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -657,7 +659,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -689,7 +691,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -708,7 +710,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -728,7 +730,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -744,7 +746,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -758,7 +760,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -792,7 +794,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -803,7 +805,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -832,7 +834,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -937,7 +939,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -953,7 +955,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -965,7 +967,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -985,7 +987,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1023,7 +1025,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1048,7 +1050,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1221,7 +1223,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1238,7 +1240,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1275,26 +1277,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1308,27 +1311,27 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1432,7 +1435,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1455,7 +1458,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1485,14 +1488,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1561,7 +1564,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1601,7 +1604,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1619,7 +1622,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1697,7 +1700,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1714,7 +1717,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1725,7 +1728,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1735,7 +1738,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1745,7 +1748,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1754,7 +1757,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1766,7 +1769,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1775,7 +1778,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1785,7 +1788,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1795,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1818,7 +1821,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1845,7 +1848,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1957,7 +1960,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1977,7 +1980,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2164,7 +2167,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2183,7 +2186,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2192,7 +2195,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2308,7 +2311,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2320,8 +2323,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2353,12 +2356,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2366,7 +2369,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2424,7 +2427,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2440,14 +2443,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2508,7 +2511,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2540,7 +2543,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2563,7 +2566,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2762,7 +2765,7 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
@@ -2775,7 +2778,7 @@
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2790,7 +2793,7 @@
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -13,19 +13,9 @@
 from pypy.module._rawffi.interp_rawffi import TYPEMAP
 from pypy.module._rawffi.interp_rawffi import size_alignment
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
 
-def push_elem(ll_array, pos, value):
-    TP = lltype.typeOf(value)
-    ll_array = rffi.cast(rffi.CArrayPtr(TP), ll_array)
-    ll_array[pos] = value
-push_elem._annspecialcase_ = 'specialize:argtype(2)'
-
-def get_elem(ll_array, pos, ll_t):
-    ll_array = rffi.cast(rffi.CArrayPtr(ll_t), ll_array)
-    return ll_array[pos]
-get_elem._annspecialcase_ = 'specialize:arg(2)'
-
 
 class W_Array(W_DataShape):
     def __init__(self, basicffitype, size):
@@ -57,7 +47,7 @@
                                                 " array length"))
             for num in range(iterlength):
                 w_item = items_w[num]
-                unwrap_value(space, push_elem, result.ll_buffer, num,
+                unwrap_value(space, write_ptr, result.ll_buffer, num,
                              self.itemcode, w_item)
         return space.wrap(result)
 
@@ -118,7 +108,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, push_elem, self.ll_buffer, num,
+        unwrap_value(space, write_ptr, self.ll_buffer, num,
                      self.shape.itemcode, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
@@ -136,7 +126,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, get_elem, self.ll_buffer, num,
+        return wrap_value(space, read_ptr, self.ll_buffer, num,
                           self.shape.itemcode)
 
     def descr_getitem(self, space, w_index):
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._rawffi.array import push_elem
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from pypy.module._rawffi.structure import W_Structure
 from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp,
      unwrap_value, unpack_argshapes, got_libffi_error)
@@ -40,7 +40,7 @@
                 args_w[i] = space.wrap(rffi.cast(rffi.ULONG, ll_args[i]))
         w_res = space.call(w_callable, space.newtuple(args_w))
         if callback_ptr.result is not None: # don't return void
-            unwrap_value(space, push_elem, ll_res, 0,
+            unwrap_value(space, write_ptr, ll_res, 0,
                          callback_ptr.result, w_res)
     except OperationError, e:
         tbprint(space, space.wrap(e.get_traceback()),
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -6,6 +6,7 @@
 
 from rpython.rlib.clibffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
 import rpython.rlib.rposix as rposix
 
@@ -44,6 +45,8 @@
 }
 TYPEMAP_PTR_LETTERS = "POszZ"
 TYPEMAP_NUMBER_LETTERS = "bBhHiIlLqQ?"
+TYPEMAP_FLOAT_LETTERS = "fd" # XXX long doubles are not propperly supported in
+                             # rpython, so we ignore then here
 
 if _MS_WINDOWS:
     TYPEMAP['X'] = ffi_type_pointer
@@ -240,6 +243,52 @@
 )
 
 unroll_letters_for_numbers = unrolling_iterable(TYPEMAP_NUMBER_LETTERS)
+unroll_letters_for_floats = unrolling_iterable(TYPEMAP_FLOAT_LETTERS)
+
+_ARM = rffi_platform.getdefined('__arm__', '')
+
+def read_ptr(ptr, ofs, TP):
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as t_array:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, t_array),
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.sizeof(TP))
+                ptr_val = t_array[0]
+                return ptr_val
+    else:
+        return rffi.cast(T, ptr)[ofs]
+read_ptr._annspecialcase_ = 'specialize:arg(2)'
+
+def write_ptr(ptr, ofs, value):
+    TP = lltype.typeOf(value)
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as s_array:
+                s_array[0] = value
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.cast(rffi.VOIDP, s_array),
+                    rffi.sizeof(TP))
+                return
+    else:
+        rffi.cast(T, ptr)[ofs] = value
+write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -12,10 +12,13 @@
 from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance
 from pypy.module._rawffi.interp_rawffi import wrap_value, unwrap_value
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
-from pypy.module._rawffi.interp_rawffi import size_alignment, LL_TYPEMAP
+from pypy.module._rawffi.interp_rawffi import LL_TYPEMAP
 from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers
+from pypy.module._rawffi.interp_rawffi import size_alignment
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib import clibffi
-from rpython.rlib.rarithmetic import intmask, r_uint, signedtype, widen
+from rpython.rlib.rarithmetic import intmask, signedtype, widen
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong
 
 def unpack_fields(space, w_fields):
     fields_w = space.unpackiterable(w_fields)
@@ -260,10 +263,17 @@
 
 def LOW_BIT(x):
     return x & 0xFFFF
+
 def NUM_BITS(x):
     return x >> 16
-def BIT_MASK(x):
-    return (1 << x) - 1
+
+def BIT_MASK(x, ll_t):
+    if ll_t is lltype.SignedLongLong:
+        return (r_longlong(1) << x) - 1
+    elif ll_t is lltype.UnsignedLongLong:
+        return (r_ulonglong(1) << x) - 1
+    return (1  << x) -1
+BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
@@ -279,20 +289,20 @@
             lowbit = LOW_BIT(bitsize)
             if numbits:
                 value = widen(value)
-                current = widen(rffi.cast(T, ptr)[0])
-                bitmask = BIT_MASK(numbits)
+                bitmask = BIT_MASK(numbits, TP)
+                #
+                current = widen(read_ptr(ptr, 0, TP))
                 current &= ~ (bitmask << lowbit)
                 current |= (value & bitmask) << lowbit
                 value = rffi.cast(TP, current)
             break
-
-    rffi.cast(T, ptr)[0] = value
+    write_ptr(ptr, 0, value)
 push_field._annspecialcase_ = 'specialize:argtype(2)'
 
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     TP = lltype.Ptr(rffi.CArray(ll_t))
-    value = rffi.cast(TP, pos)[0]
+    value = read_ptr(pos, 0, ll_t)
 
     # Handle bitfields
     for c in unroll_letters_for_numbers:
@@ -301,13 +311,16 @@
             numbits = NUM_BITS(bitsize)
             lowbit = LOW_BIT(bitsize)
             if numbits:
-                value = widen(value)
+                value = widen(rffi.cast(ll_t, value))
+                bitmask = BIT_MASK(numbits, ll_t)
+                #
                 value >>= lowbit
-                value &= BIT_MASK(numbits)
+                value &= bitmask
                 if ll_t is lltype.Bool or signedtype(ll_t._type):
                     sign = (value >> (numbits - 1)) & 1
                     if sign:
-                        value = value - (1 << numbits)
+                        one = r_longlong(1) if ll_t is lltype.SignedLongLong else 1
+                        value = value - (one << numbits)
                 value = rffi.cast(ll_t, value)
             break
 
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1,6 +1,6 @@
 from rpython.translator.platform import platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.module._rawffi.interp_rawffi import TYPEMAP
+from pypy.module._rawffi.interp_rawffi import TYPEMAP, TYPEMAP_FLOAT_LETTERS
 from pypy.module._rawffi.tracker import Tracker
 
 import os, sys, py
@@ -48,12 +48,12 @@
             free(x1->next);
             free(x1);
         }
-        
+
         const char *static_str = "xxxxxx";
         long static_int = 42;
         double static_double = 42.42;
         long double static_longdouble = 42.42;
-        
+
         unsigned short add_shorts(short one, short two)
         {
            return one + two;
@@ -175,7 +175,7 @@
             inp.y = inp.x * 100;
             return inp;
         }
-        
+
         '''))
         symbols = """get_char char_check get_raw_pointer
                      add_shorts
@@ -194,7 +194,7 @@
         eci = ExternalCompilationInfo(export_symbols=symbols)
         return str(platform.compile([c_file], eci, 'x', standalone=False))
     prepare_c_example = staticmethod(prepare_c_example)
-    
+
     def setup_class(cls):
         space = cls.space
         from rpython.rlib.clibffi import get_libc_name
@@ -211,6 +211,7 @@
         cls.w_platform = space.wrap(platform.name)
         cls.w_sizes_and_alignments = space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
+        cls.w_float_typemap = space.wrap(TYPEMAP_FLOAT_LETTERS)
 
     def test_libload(self):
         import _rawffi
@@ -439,7 +440,7 @@
         arg.free()
         assert t.tm_year == 70
         assert t.tm_sec == 1
-        assert t.tm_min == 2      
+        assert t.tm_min == 2
         x.free()
 
     def test_nested_structures(self):
@@ -465,6 +466,20 @@
         free_double_struct = lib.ptr("free_double_struct", ['P'], None)
         free_double_struct(res)
 
+    def test_structure_bitfields_char(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'B', 1),
+                               ('B', 'B', 6),
+                               ('C', 'B', 1)])
+        x = X()
+        x.A = 0xf
+        x.B = 0xff
+        x.C = 0xf
+        assert x.A == 1
+        assert x.B == 63
+        assert x.C == 1
+        x.free()
+
     def test_structure_bitfields(self):
         import _rawffi
         X = _rawffi.Structure([('A', 'I', 1),
@@ -487,6 +502,60 @@
         assert (y.a, y.b, y.c) == (-1, -7, 0)
         y.free()
 
+    def test_structure_bitfields_longlong(self):
+        import _rawffi
+        Z = _rawffi.Structure([('a', 'Q', 1),
+                               ('b', 'Q', 62),
+                               ('c', 'Q', 1)])
+        z = Z()
+        z.a, z.b, z.c = 7, 0x1000000000000001, 7
+        assert (z.a, z.b, z.c) == (1, 0x1000000000000001, 1)
+        z.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
     def test_invalid_bitfields(self):
         import _rawffi
         raises(TypeError, _rawffi.Structure, [('A', 'c', 1)])
@@ -571,7 +640,7 @@
         res = pass_ll(arg1)
         assert res[0] == 1<<42
         arg1.free()
-    
+
     def test_callback(self):
         import _rawffi
         import struct
@@ -683,6 +752,16 @@
             assert _rawffi.sizeof(k) == s
             assert _rawffi.alignment(k) == a
 
+    def test_unaligned(self):
+        import _rawffi
+        for k in self.float_typemap:
+            S = _rawffi.Structure([('pad', 'c'), ('value', k)], pack=1)
+            s = S()
+            s.value = 4
+            assert s.value == 4
+            s.free()
+
+
     def test_array_addressof(self):
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
@@ -964,7 +1043,7 @@
         assert res.y == 33
         assert s2h.x == 7
         assert s2h.y == 11
-        
+
         s2h.free()
 
     def test_ret_struct_containing_array(self):
@@ -1035,7 +1114,7 @@
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
-    
+
     def setup_class(cls):
         cls.w_sizes_and_alignments = cls.space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -235,6 +235,11 @@
         w_values = space.newtuple([self])
         return convert_to_array(space, w_values)
 
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0):
+        v = self.convert_to(self.get_dtype(space))
+        return self.get_dtype(space).itemtype.round(v, decimals)
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -501,6 +506,7 @@
     any = interp2app(W_GenericBox.descr_any),
     all = interp2app(W_GenericBox.descr_all),
     ravel = interp2app(W_GenericBox.descr_ravel),
+    round = interp2app(W_GenericBox.descr_round),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -558,9 +558,27 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "resize not implemented yet"))
 
-    def descr_round(self, space, w_decimals=0, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "round not implemented yet"))
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0, w_out=None):
+        if space.is_none(w_out):
+            if self.get_dtype().is_bool_type():
+                #numpy promotes bool.round() to float16. Go figure.
+                w_out = W_NDimArray.from_shape(self.get_shape(),
+                       interp_dtype.get_dtype_cache(space).w_float16dtype)
+            else:
+                w_out = None
+        elif not isinstance(w_out, W_NDimArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                "return arrays must be of ArrayType"))
+        out = interp_dtype.dtype_agreement(space, [self], self.get_shape(),
+                                           w_out)
+        if out.get_dtype().is_bool_type() and self.get_dtype().is_bool_type():
+            calc_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
+        else:
+            calc_dtype = out.get_dtype()
+
+        loop.round(space, self, calc_dtype, self.get_shape(), decimals, out)
+        return out
 
     def descr_searchsorted(self, space, w_v, w_side='left'):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -975,6 +993,7 @@
     byteswap = interp2app(W_NDimArray.descr_byteswap),
     choose   = interp2app(W_NDimArray.descr_choose),
     clip     = interp2app(W_NDimArray.descr_clip),
+    round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
 
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -173,7 +173,7 @@
         iter = x_iter
     shapelen = len(shape)
     while not iter.done():
-        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, 
+        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
                                         arr_dtype=arr_dtype)
         w_cond = arr_iter.getitem()
         if arr_dtype.itemtype.bool(w_cond):
@@ -188,7 +188,7 @@
     return out
 
 axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce',
-                                    greens=['shapelen', 
+                                    greens=['shapelen',
                                             'func', 'dtype',
                                             'identity'],
                                     reds='auto')
@@ -228,7 +228,7 @@
     arg_driver = jit.JitDriver(name='numpy_' + op_name,
                                greens = ['shapelen', 'dtype'],
                                reds = 'auto')
-    
+
     def argmin_argmax(arr):
         result = 0
         idx = 1
@@ -265,7 +265,7 @@
      result.shape == [3, 5, 2, 4]
      broadcast shape should be [3, 5, 2, 7, 4]
      result should skip dims 3 which is len(result_shape) - 1
-        (note that if right is 1d, result should 
+        (note that if right is 1d, result should
                   skip len(result_shape))
      left should skip 2, 4 which is a.ndims-1 + range(right.ndims)
           except where it==(right.ndims-2)
@@ -283,9 +283,9 @@
     righti = right.create_dot_iter(broadcast_shape, right_skip)
     while not outi.done():
         dot_driver.jit_merge_point(dtype=dtype)
-        lval = lefti.getitem().convert_to(dtype) 
-        rval = righti.getitem().convert_to(dtype) 
-        outval = outi.getitem().convert_to(dtype) 
+        lval = lefti.getitem().convert_to(dtype)
+        rval = righti.getitem().convert_to(dtype)
+        outval = outi.getitem().convert_to(dtype)
         v = dtype.itemtype.mul(lval, rval)
         value = dtype.itemtype.add(v, outval).convert_to(dtype)
         outi.setitem(value)
@@ -355,7 +355,7 @@
         setitem_filter_driver.jit_merge_point(shapelen=shapelen,
                                               index_dtype=index_dtype,
                                               arr_dtype=arr_dtype,
-                                             ) 
+                                             )
         if index_iter.getitem_bool():
             arr_iter.setitem(value_iter.getitem())
             value_iter.next()
@@ -572,6 +572,21 @@
         out_iter.next()
         min_iter.next()
 
+round_driver = jit.JitDriver(greens = ['shapelen', 'dtype'],
+                                    reds = 'auto')
+
+def round(space, arr, dtype, shape, decimals, out):
+    arr_iter = arr.create_iter(shape)
+    shapelen = len(shape)
+    out_iter = out.create_iter(shape)
+    while not arr_iter.done():
+        round_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+        w_v = dtype.itemtype.round(arr_iter.getitem().convert_to(dtype),
+                     decimals)
+        out_iter.setitem(w_v)
+        arr_iter.next()
+        out_iter.next()
+
 diagonal_simple_driver = jit.JitDriver(greens = ['axis1', 'axis2'],
                                        reds = 'auto')
 
@@ -613,4 +628,4 @@
         out_iter.setitem(arr.getitem_index(space, indexes))
         iter.next()
         out_iter.next()
-       
+
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -21,3 +21,19 @@
 
         a = zeros(3)
         assert loads(dumps(sum(a))) == sum(a)
+
+    def test_round(self):
+        from numpypy import int32, float64, complex128, bool
+        i = int32(1337)
+        f = float64(13.37)
+        c = complex128(13 + 37.j)
+        b = bool(0)
+        assert i.round(decimals=-2) == 1300
+        assert i.round(decimals=1) == 1337
+        assert c.round() == c
+        assert f.round() == 13.
+        assert f.round(decimals=-1) == 10.
+        assert f.round(decimals=1) == 13.4
+        exc = raises(AttributeError, 'b.round()')
+        assert exc.value[0] == "'bool' object has no attribute 'round'"
+
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -286,7 +286,7 @@
 
         skip('sign of nan is non-determinant')
         assert (signbit([float('nan'), float('-nan'), -float('nan')]) ==
-            [False, True, True]).all()    
+            [False, True, True]).all()
 
     def test_reciprocal(self):
         from numpypy import array, reciprocal, complex64, complex128
@@ -334,6 +334,23 @@
         assert all([math.copysign(1, f(abs(float("nan")))) == 1 for f in floor, ceil, trunc])
         assert all([math.copysign(1, f(-abs(float("nan")))) == -1 for f in floor, ceil, trunc])
 
+    def test_round(self):
+        from numpypy import array, dtype
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -1.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == a.round()).all()
+        i = array([-1000, -100, -1, 0, 1, 111, 1111, 11111], dtype=int)
+        assert (i == i.round()).all()
+        assert (i.round(decimals=4) == i).all()
+        assert (i.round(decimals=-4) == [0, 0, 0, 0, 0, 0, 0, 10000]).all()
+        b = array([True, False], dtype=bool)
+        bround = b.round()
+        assert (bround == [1., 0.]).all()
+        assert bround.dtype is dtype('float16')
+        c = array([10.5+11.5j, -15.2-100.3456j, 0.2343+11.123456j])
+        assert (c.round(0) == [10.+12.j, -15-100j, 0+11j]).all()
+
+
     def test_copysign(self):
         from numpypy import array, copysign
 
@@ -364,7 +381,7 @@
             assert b[i] == res
 
     def test_exp2(self):
-        import math 
+        import math
         from numpypy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
@@ -759,8 +776,8 @@
              complex(inf, inf), complex(inf, ninf), complex(0, inf),
              complex(ninf, ninf), complex(nan, 0), complex(0, nan),
              complex(nan, nan)]
-        assert (isfinite(a) == [True, True, False, False, False, 
-                        False, False, False, False, False]).all() 
+        assert (isfinite(a) == [True, True, False, False, False,
+                        False, False, False, False, False]).all()
 
     def test_logical_ops(self):
         from numpypy import logical_and, logical_or, logical_xor, logical_not
@@ -864,7 +881,7 @@
         #numpy returns (a.real*b.real + a.imag*b.imag) / abs(b)**2
         expect = [3., -23., 1.]
         for i in range(len(a)):
-            assert b[i] == expect[i] 
+            assert b[i] == expect[i]
         b = floor_divide(a[0], 0.)
         assert math.isnan(b.real)
         assert b.imag == 0.
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -37,7 +37,7 @@
         return self.box(
             func(
                 self,
-                self.for_computation(raw)
+                self.for_computation(raw),
             )
         )
     return dispatcher
@@ -521,6 +521,23 @@
             return v
         return 0
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.for_computation(self.unbox(v))
+        if decimals < 0:
+            # No ** in rpython
+            factor = 1
+            for i in xrange(-decimals):
+                factor *=10
+            #int does floor division, we want toward zero
+            if raw < 0:
+                ans = - (-raw / factor * factor)
+            else:
+                ans = raw / factor * factor
+        else:
+            ans = raw
+        return self.box(ans)
+
     @raw_unary_op
     def signbit(self, v):
         return v < 0
@@ -798,6 +815,16 @@
     def ceil(self, v):
         return math.ceil(v)
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.for_computation(self.unbox(v))
+        if rfloat.isinf(raw):
+            return v
+        elif rfloat.isnan(raw):
+            return v
+        ans = rfloat.round_double(raw, decimals, half_even=True)
+        return self.box(ans)
+
     @simple_unary_op
     def trunc(self, v):
         if v < 0:
@@ -1354,6 +1381,15 @@
         except ZeroDivisionError:
             return rfloat.NAN, rfloat.NAN
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        ans = list(self.for_computation(self.unbox(v)))
+        if isfinite(ans[0]):
+            ans[0] = rfloat.round_double(ans[0], decimals,  half_even=True)
+        if isfinite(ans[1]):
+            ans[1] = rfloat.round_double(ans[1], decimals,  half_even=True)
+        return self.box_complex(ans[0], ans[1])
+
     # No floor, ceil, trunc in numpy for complex
     #@simple_unary_op
     #def floor(self, v):
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,8 +2,9 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.buffer import RWBuffer
 from rpython.rlib import rmmap, rarithmetic
-from rpython.rlib.rmmap import RValueError, RTypeError
+from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -20,13 +21,14 @@
         self.mmap.close()
 
     def read_byte(self):
+        self.check_valid()
         try:
             return self.space.wrap(ord(self.mmap.read_byte()))
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def readline(self):
+        self.check_valid()
         return self.space.wrapbytes(self.mmap.readline())
 
     @unwrap_spec(num=int)
@@ -36,6 +38,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def find(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -49,6 +52,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def rfind(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -62,16 +66,18 @@
 
     @unwrap_spec(pos=OFF_T, whence=int)
     def seek(self, pos, whence=0):
+        self.check_valid()
         try:
             self.mmap.seek(pos, whence)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def tell(self):
+        self.check_valid()
         return self.space.wrap(self.mmap.tell())
 
     def descr_size(self):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.file_size())
         except OSError, e:
@@ -79,41 +85,40 @@
 
     @unwrap_spec(data='bufferstr')
     def write(self, data):
+        self.check_valid()
         self.check_writeable()
         try:
             self.mmap.write(data)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(byte=int)
     def write_byte(self, byte):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.write_byte(chr(byte))
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(offset=int, size=int)
     def flush(self, offset=0, size=0):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.flush(offset, size))
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
         except OSError, e:
             raise mmap_error(self.space, e)
 
     @unwrap_spec(dest=int, src=int, count=int)
     def move(self, dest, src, count):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.move(dest, src, count)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(newsize=int)
     def resize(self, newsize):
@@ -143,28 +148,19 @@
         try:
             self.mmap.check_valid()
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def check_writeable(self):
         try:
             self.mmap.check_writeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def check_resizeable(self):
         try:
             self.mmap.check_resizeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def descr_getitem(self, w_index):
         self.check_valid()
@@ -173,6 +169,8 @@
         start, stop, step = space.decode_index(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(ord(self.mmap.getitem(start)))
+        elif step == 1:
+            return space.wrap(self.mmap.getslice(start, stop - start))
         else:
             res = "".join([self.mmap.getitem(i)
                            for i in range(start, stop, step)])
@@ -195,15 +193,16 @@
             if len(value) != length:
                 raise OperationError(space.w_ValueError,
                           space.wrap("mmap slice assignment is wrong size"))
-            for i in range(length):
-                self.mmap.setitem(start, value[i])
-                start += step
+            if step == 1:
+                self.mmap.setslice(start, value)
+            else:
+                for i in range(length):
+                    self.mmap.setitem(start, value[i])
+                    start += step
 
     def descr_buffer(self):
-        # XXX improve to work directly on the low-level address
-        from pypy.interpreter.buffer import StringLikeBuffer
-        space = self.space
-        return space.wrap(StringLikeBuffer(space, space.wrap(self)))
+        self.check_valid()
+        return self.space.wrap(MMapBuffer(self.space, self.mmap))
 
     def descr_enter(self, space):
         self.check_valid()
@@ -227,10 +226,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 elif rmmap._MS_WINDOWS:
@@ -246,10 +243,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 W_MMap.typedef = TypeDef("mmap",
@@ -293,5 +288,64 @@
                                                  space.w_EnvironmentError)
 
 def mmap_error(space, e):
-    w_error = space.fromcache(Cache).w_error
-    return wrap_oserror(space, e, w_exception_class=w_error)
+    if isinstance(e, RValueError):
+        return OperationError(space.w_ValueError,
+                              space.wrap(e.message))
+    elif isinstance(e, RTypeError):
+        return OperationError(space.w_TypeError,
+                              space.wrap(e.message))
+    elif isinstance(e, OSError):
+        w_error = space.fromcache(Cache).w_error
+        return wrap_oserror(space, e, w_exception_class=w_error)
+    else:
+        # bogus 'e'?
+        return OperationError(space.w_SystemError, space.wrap('%s' % e))
+mmap_error._dont_inline_ = True
+
+
+class MMapBuffer(RWBuffer):
+    def __init__(self, space, mmap):
+        self.space = space
+        self.mmap = mmap
+
+    def get_raw_address(self):
+        return self.mmap.data
+
+    def getlength(self):
+        return self.mmap.size
+
+    def getitem(self, index):
+        self.check_valid()
+        return self.mmap.data[index]
+
+    def getslice(self, start, stop, step, size):
+        self.check_valid()
+        if step == 1:
+            return self.mmap.getslice(start, size)
+        else:
+            return RWBuffer.getslice(self, start, stop, step, size)
+
+    def setitem(self, index, char):
+        self.check_valid_writeable()
+        self.mmap.data[index] = char
+
+    def setslice(self, start, string):
+        self.check_valid_writeable()
+        self.mmap.setslice(start, string)
+
+    def get_raw_address(self):
+        self.check_valid()
+        return self.mmap.data
+
+    def check_valid(self):
+        try:
+            self.mmap.check_valid()
+        except RValueError, v:
+            raise mmap_error(self.space, v)
+
+    def check_valid_writeable(self):
+        try:
+            self.mmap.check_valid()
+            self.mmap.check_writeable()
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -540,6 +540,24 @@
         m.close()
         f.close()
 
+    def test_buffer_write(self):
+        from mmap import mmap
+        f = open(self.tmpname + "y", "w+")
+        f.write("foobar")
+        f.flush()
+        m = mmap(f.fileno(), 6)
+        m[5] = '?'
+        b = buffer(m)
+        try:
+            b[:3] = "FOO"
+        except TypeError:     # on CPython: "buffer is read-only" :-/
+            skip("on CPython: buffer is read-only")
+        m.close()
+        f.seek(0)
+        got = f.read()
+        assert got == "FOOba?"
+        f.close()
+
     def test_offset(self):
         from mmap import mmap, ALLOCATIONGRANULARITY
         f = open(self.tmpname + "y", "wb+")
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -45,10 +45,7 @@
     raise NotImplementedError
 
 def resolvedirof(filename):
-    try:
-        filename = rpath.rabspath(filename)
-    except OSError:
-        pass
+    filename = rpath.rabspath(filename)
     dirname = rpath.rabspath(os.path.join(filename, '..'))
     if os.path.islink(filename):
         try:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
@@ -990,7 +990,7 @@
         assert f.a == 12345
         assert b.b == b"B"
         assert b.c == b"C"
-        assert repr(b).startswith("<cdata 'struct $bar_t *'")
+        assert repr(b).startswith("<cdata 'bar_t *'")
 
     def test_struct_with_two_usages(self):
         for name in ['foo_s', '']:    # anonymous or not
@@ -1300,9 +1300,9 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("typedef enum { Value0 = 0 } e, *pe;\n"
                  "typedef enum { Value1 = 1 } e1;")
-        assert ffi.getctype("e*") == 'enum $e *'
-        assert ffi.getctype("pe") == 'enum $e *'
-        assert ffi.getctype("e1*") == 'enum $e1 *'
+        assert ffi.getctype("e*") == 'e *'
+        assert ffi.getctype("pe") == 'e *'
+        assert ffi.getctype("e1*") == 'e1 *'
 
     def test_new_ctype(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -179,3 +179,9 @@
             'ffi.cdef("struct s2 { char x:0; };"); ffi.new("struct s2 *")')
         py.test.raises(TypeError,
             'ffi.cdef("struct s3 { char x:9; };"); ffi.new("struct s3 *")')
+
+    def test_struct_with_typedef(self):
+        ffi = FFI()
+        ffi.cdef("typedef struct { float x; } foo_t;")
+        p = ffi.new("foo_t *", [5.2])
+        assert repr(p).startswith("<cdata 'foo_t *' ")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
@@ -1442,7 +1442,7 @@
                        "foo_t myfunc(void) { foo_t x = { 42 }; return x; }")
     assert str(e.value) in [
         "function myfunc: 'foo_t' is used as result type, but is opaque",
-        "function myfunc: result type 'struct $foo_t' is opaque"]
+        "function myfunc: result type 'foo_t' is opaque"]
 
 def test_include():
     ffi1 = FFI()
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
@@ -66,6 +66,10 @@
         x.a, x.b, x.c = -1, 7, -1
         assert (x.a, x.b, x.c) == (-1, 7, -1)
 
+        x = X()
+        x.a, x.b, x.c = -1, -7, -1
+        assert (x.a, x.b, x.c) == (-1, -7, -1)
+
     def test_ulonglong(self):
         class X(Structure):
             _fields_ = [("a", c_ulonglong, 1),
@@ -75,8 +79,8 @@
         assert sizeof(X) == sizeof(c_longlong)
         x = X()
         assert (x.a, x.b, x.c) == (0, 0, 0)
-        x.a, x.b, x.c = 7, 7, 7
-        assert (x.a, x.b, x.c) == (1, 7, 1)
+        x.a, x.b, x.c = 7, 2305843009213693953, 7
+        assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1)
 
     def test_signed(self):
         for c_typ in signed_int_types:
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -50,10 +50,6 @@
 def delegate_Bool2IntObject(space, w_bool):
     return W_IntObject(int(w_bool.boolval))
 
-def delegate_Bool2SmallInt(space, w_bool):
-    from pypy.objspace.std.smallintobject import W_SmallIntObject
-    return W_SmallIntObject(int(w_bool.boolval))   # cannot overflow
-
 
 def nonzero__Bool(space, w_bool):
     return w_bool
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -59,11 +59,6 @@
 
 #registerimplementation(W_IntObject)
 
-# NB: This code is shared by smallintobject.py, and thus no other Int
-# multimethods should be invoked from these implementations. Instead, add an
-# alias and then teach copy_multimethods in smallintobject.py to override
-# it. See int__Int for example.
-
 def repr__Int(space, w_int1):
     a = w_int1.intval
     res = str(a)
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1041,14 +1041,22 @@
 
     def _intersect_base(self, w_set, w_other):
         if self is w_other.strategy:
-            strategy = w_set.strategy
-            storage = strategy._intersect_unwrapped(w_set, w_other)
+            strategy = self
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = self._intersect_unwrapped(w_other, w_set)
+            else:
+                storage = self._intersect_unwrapped(w_set, w_other)
         elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
             strategy = self.space.fromcache(EmptySetStrategy)
             storage = strategy.get_empty_storage()
         else:
             strategy = self.space.fromcache(ObjectSetStrategy)
-            storage = self._intersect_wrapped(w_set, w_other)
+            if w_set.length() > w_other.length():
+                # swap operants
+                storage = w_other.strategy._intersect_wrapped(w_other, w_set)
+            else:
+                storage = self._intersect_wrapped(w_set, w_other)
         return storage, strategy
 
     def _intersect_wrapped(self, w_set, w_other):
@@ -1071,9 +1079,6 @@
         return self.erase(result)
 
     def intersect(self, w_set, w_other):
-        if w_set.length() > w_other.length():
-            return w_other.intersect(w_set)
-
         storage, strategy = self._intersect_base(w_set, w_other)
         return w_set.from_storage_and_strategy(storage, strategy)
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -1014,3 +1014,9 @@
         assert s2 == s
         assert s2 is s
         
+    def test_intersect_frozenset_set(self):
+        # worked before
+        assert type(frozenset([2]) & set([1, 2])) is frozenset
+        # did not work before because of an optimization that swaps both
+        # operands when the first set is larger than the second
+        assert type(frozenset([1, 2]) & set([2])) is frozenset
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -19,6 +19,12 @@
         w_str = self.space.wrap(u'abcd')
         assert self.space.listview_unicode(w_str) == list(u"abcd")
 
+    def test_new_shortcut(self):
+        space = self.space
+        w_uni = self.space.wrap(u'abcd')
+        w_new = space.call_method(
+                space.w_unicode, "__new__", space.w_unicode, w_uni)
+        assert w_new is w_uni
 
 class AppTestUnicodeStringStdOnly:
     def test_compares(self):
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -278,8 +278,8 @@
     ChoiceOption("platform",
                  "target platform", ['host'] + PLATFORMS, default='host',
                  cmdline='--platform',
-                 requires={"arm": [("translation.gcrootfinder", "shadowstack")]},
-                 suggests={"arm": [("translation.jit_backend", "arm")]}),
+                 suggests={"arm": [("translation.gcrootfinder", "shadowstack"),
+                                   ("translation.jit_backend", "arm")]}),
 
 ])
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -9,19 +9,20 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 from rpython.jit.metainterp.resoperation import ResOperation, rop, get_deep_immutable_oplist
-from rpython.jit.metainterp.history import TreeLoop, Box, JitCellToken, TargetToken
-from rpython.jit.metainterp.history import AbstractFailDescr, BoxInt
-from rpython.jit.metainterp.history import BoxPtr, BoxFloat, ConstInt
-from rpython.jit.metainterp import history, resume, jitexc
+from rpython.jit.metainterp.history import (TreeLoop, Box, JitCellToken,

From noreply at buildbot.pypy.org  Wed Jun 26 14:23:46 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Wed, 26 Jun 2013 14:23:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3k-memoryview: hg merge py3k
Message-ID: <20130626122346.947AE1C346E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: py3k-memoryview
Changeset: r65003:3708c6a88a95
Date: 2013-06-26 14:20 +0200
http://bitbucket.org/pypy/pypy/changeset/3708c6a88a95/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 7196 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -6,6 +6,7 @@
 .idea
 .project
 .pydevproject
+__pycache__
 
 syntax: regexp
 ^testresult$
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,6 +134,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -168,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -278,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib-python/3/logging/__init__.py b/lib-python/3/logging/__init__.py
--- a/lib-python/3/logging/__init__.py
+++ b/lib-python/3/logging/__init__.py
@@ -129,6 +129,11 @@
 DEBUG = 10
 NOTSET = 0
 
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
 _levelToName = {
     CRITICAL: 'CRITICAL',
     ERROR: 'ERROR',
@@ -161,7 +166,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelToName.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
diff --git a/lib-python/3/test/test_logging.py b/lib-python/3/test/test_logging.py
--- a/lib-python/3/test/test_logging.py
+++ b/lib-python/3/test/test_logging.py
@@ -299,6 +299,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -431,5 +431,6 @@
 
            This API is used by pickle.py and copy.py.
         """
-        return (type(self), (self.default_factory,), None, None, self.items())
+        return (type(self), (self.default_factory,), None, None,
+                iter(self.items()))
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,48 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -361,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -707,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -934,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -76,4 +76,4 @@
 The rest of directories serve specific niche goal and are unlikely a good
 entry point.
 
-.. _`introduction to RPython`: getting-started-dev.rst
+.. _`introduction to RPython`: getting-started-dev.html
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,3 +52,12 @@
 
 .. branch: ctypes-byref
 Add the '_obj' attribute on ctypes pointer() and byref() objects
+
+.. branch: argsort-segfault
+Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
+
+.. branch: dtype-isnative
+.. branch: ndarray-round
+
+.. branch: faster-str-of-bigint
+Improve performance of str(long).
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -912,7 +912,10 @@
         return (None, None)
 
     def newlist_str(self, list_s):
-        return self.newlist([self.wrap(s) for s in list_s])
+        return self.newlist([self.wrapbytes(s) for s in list_s])
+
+    def newlist_unicode(self, list_u):
+        return self.newlist([self.wrap(u) for u in list_u])
 
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -379,7 +379,6 @@
     return tuple(parts), tuple(formats)
 
 def get_operrcls2(valuefmt):
-    from rpython.rlib.runicode import str_decode_utf_8
     valuefmt = valuefmt.decode('ascii')
     strings, formats = decompose_valuefmt(valuefmt)
     assert len(strings) == len(formats) + 1
@@ -407,8 +406,7 @@
                     if fmt == 'd':
                         result = str(value).decode('ascii')
                     elif fmt == '8':
-                        result, _ = str_decode_utf_8(value, len(value),
-                                                     'strict')
+                        result = value.decode('utf-8')
                     elif fmt == 'R':
                         result = space.unicode_w(space.repr(value))
                     elif fmt in 'NT':
@@ -440,7 +438,7 @@
 
     Supports the standard %s and %d formats, plus the following:
 
-    %8 - The result of arg.decode('utf-8', 'strict')
+    %8 - The result of arg.decode('utf-8')
     %N - The result of w_arg.getname(space)
     %R - The result of space.unicode_w(space.repr(w_arg))
     %T - The result of space.type(w_arg).getname(space)
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -5,6 +5,22 @@
 
 @unwrap_spec(type=str)
 def newdict(space, type):
+    """ newdict(type)
+
+    Create a normal dict with a special implementation strategy.
+
+    type is a string and can be:
+
+    * "module" - equivalent to some_module.__dict__
+
+    * "instance" - equivalent to an instance dict with a not-changing-much
+                   set of keys
+
+    * "kwargs" - keyword args dict equivalent of what you get from **kwargs
+                 in a function, optimized for passing around
+
+    * "strdict" - string-key only dict. This one should be chosen automatically
+    """
     if type == 'module':
         return space.newdict(module=True)
     elif type == 'instance':
@@ -18,6 +34,10 @@
                               type)
 
 def dictstrategy(space, w_obj):
+    """ dictstrategy(dict)
+
+    show the underlaying strategy used by a dict object
+    """
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError,
                              space.wrap("expecting dict object"))
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -13,7 +13,6 @@
     _mixin_ = True
 
     def __init__(self, space, name, size, align, enumerators, enumvalues):
-        name = "enum " + name
         self._super.__init__(self, space, size, name, len(name), align)
         self.enumerators2values = {}   # str -> int
         self.enumvalues2erators = {}   # int -> str
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -289,16 +289,6 @@
                "with verify() (see pypy/module/_cffi_backend/ctypefunc.py "
                "for details)"))
 
-        if USE_C_LIBFFI_MSVC and is_result_type:
-            # MSVC returns small structures in registers.  Pretend int32 or
-            # int64 return type.  This is needed as a workaround for what
-            # is really a bug of libffi_msvc seen as an independent library
-            # (ctypes has a similar workaround).
-            if ctype.size <= 4:
-                return clibffi.ffi_type_sint32
-            if ctype.size <= 8:
-                return clibffi.ffi_type_sint64
-
         # walk the fields, expanding arrays into repetitions; first,
         # only count how many flattened fields there are
         nflat = 0
@@ -318,6 +308,16 @@
                                "a struct with a zero-length array"))
             nflat += flat
 
+        if USE_C_LIBFFI_MSVC and is_result_type:
+            # MSVC returns small structures in registers.  Pretend int32 or
+            # int64 return type.  This is needed as a workaround for what
+            # is really a bug of libffi_msvc seen as an independent library
+            # (ctypes has a similar workaround).
+            if ctype.size <= 4:
+                return clibffi.ffi_type_sint32
+            if ctype.size <= 8:
+                return clibffi.ffi_type_sint64
+
         # allocate an array of (nflat + 1) ffi_types
         elements = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * (nflat + 1))
         elements = rffi.cast(FFI_TYPE_PP, elements)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -187,7 +187,7 @@
         else:
             extra = " *"
         self.is_file = (ctitem.name == "struct _IO_FILE" or
-                        ctitem.name == "struct $FILE")
+                        ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -25,7 +25,6 @@
     custom_field_pos = False
 
     def __init__(self, space, name):
-        name = '%s %s' % (self.kind, name)
         W_CType.__init__(self, space, -1, name, len(name))
 
     def check_complete(self, w_errorcls=None):
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 
 from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr,
     ctypearray, ctypestruct, ctypevoid, ctypeenum)
@@ -115,13 +116,15 @@
 # ____________________________________________________________
 
 SF_MSVC_BITFIELDS = 1
+SF_GCC_ARM_BITFIELDS = 2
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
+elif rffi_platform.getdefined('__arm__', ''):
+    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
     DEFAULT_SFLAGS = 0
 
-
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
@@ -180,7 +183,7 @@
         # field is an anonymous bitfield
         falign = ftype.alignof()
         do_align = True
-        if fbitsize >= 0:
+        if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
                 # GCC: anonymous bitfields (of any size) don't cause alignment
                 do_align = (fname != '')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -641,6 +641,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -648,7 +650,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -657,7 +659,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -689,7 +691,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -708,7 +710,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -728,7 +730,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -744,7 +746,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -758,7 +760,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -792,7 +794,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -803,7 +805,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -832,7 +834,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -937,7 +939,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -953,7 +955,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -965,7 +967,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -985,7 +987,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1023,7 +1025,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1048,7 +1050,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1221,7 +1223,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1238,7 +1240,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1275,26 +1277,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1308,27 +1311,27 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1432,7 +1435,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1455,7 +1458,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1485,14 +1488,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1561,7 +1564,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1601,7 +1604,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1619,7 +1622,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1697,7 +1700,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1714,7 +1717,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1725,7 +1728,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1735,7 +1738,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1745,7 +1748,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1754,7 +1757,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1766,7 +1769,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1775,7 +1778,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1785,7 +1788,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1795,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1818,7 +1821,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1845,7 +1848,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1957,7 +1960,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1977,7 +1980,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2164,7 +2167,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2183,7 +2186,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2192,7 +2195,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2308,7 +2311,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2320,8 +2323,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2353,12 +2356,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2366,7 +2369,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2424,7 +2427,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2440,14 +2443,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2508,7 +2511,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2540,7 +2543,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2563,7 +2566,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2762,20 +2765,20 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
-    if flag == 0:   # gcc
+    if flag % 2 == 0:   # gcc and gcc ARM
         assert typeoffsetof(BStruct, 'c') == (BChar, 3)
         assert sizeof(BStruct) == 4
-    else:           # msvc
+    else:               # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 8)
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2783,11 +2786,14 @@
     if flag == 0:   # gcc
         assert sizeof(BStruct) == 5
         assert alignof(BStruct) == 1
-    else:           # msvc
+    elif flag == 1: # msvc
+        assert sizeof(BStruct) == 6
+        assert alignof(BStruct) == 2
+    else:           # gcc ARM
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
@@ -2795,10 +2801,15 @@
     if flag == 0:   # gcc
         assert typeoffsetof(BStruct, 'c') == (BChar, 4)
         assert sizeof(BStruct) == 5
-    else:           # msvc
+        assert alignof(BStruct) == 1
+    elif flag == 1:  # msvc
         assert typeoffsetof(BStruct, 'c') == (BChar, 1)
         assert sizeof(BStruct) == 2
-    assert alignof(BStruct) == 1
+        assert alignof(BStruct) == 1
+    else:            # gcc ARM
+        assert typeoffsetof(BStruct, 'c') == (BChar, 4)
+        assert sizeof(BStruct) == 8
+        assert alignof(BStruct) == 4
 
 
 def test_bitfield_as_gcc():
@@ -2807,6 +2818,9 @@
 def test_bitfield_as_msvc():
     _test_bitfield_details(flag=1)
 
+def test_bitfield_as_arm_gcc():
+    _test_bitfield_details(flag=2)
+
 
 def test_version():
     # this test is here mostly for PyPy
diff --git a/pypy/module/_collections/app_defaultdict.py b/pypy/module/_collections/app_defaultdict.py
--- a/pypy/module/_collections/app_defaultdict.py
+++ b/pypy/module/_collections/app_defaultdict.py
@@ -63,4 +63,5 @@
 
            This API is used by pickle.py and copy.py.
         """
-        return (type(self), (self._default_factory,), None, None, self.items())
+        return (type(self), (self._default_factory,), None, None,
+                iter(self.items()))
diff --git a/pypy/module/_collections/test/test_defaultdict.py b/pypy/module/_collections/test/test_defaultdict.py
--- a/pypy/module/_collections/test/test_defaultdict.py
+++ b/pypy/module/_collections/test/test_defaultdict.py
@@ -56,3 +56,9 @@
         assert len(d2) == 1
         assert d2[2] == 3
         assert d2[3] == 42
+
+    def test_reduce(self):
+        import _collections
+        d = _collections.defaultdict(None, {3: 4})
+        dict_iter = d.__reduce__()[4]
+        assert type(dict_iter) is type(iter(d.items()))
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -13,19 +13,9 @@
 from pypy.module._rawffi.interp_rawffi import TYPEMAP
 from pypy.module._rawffi.interp_rawffi import size_alignment
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
 
-def push_elem(ll_array, pos, value):
-    TP = lltype.typeOf(value)
-    ll_array = rffi.cast(rffi.CArrayPtr(TP), ll_array)
-    ll_array[pos] = value
-push_elem._annspecialcase_ = 'specialize:argtype(2)'
-
-def get_elem(ll_array, pos, ll_t):
-    ll_array = rffi.cast(rffi.CArrayPtr(ll_t), ll_array)
-    return ll_array[pos]
-get_elem._annspecialcase_ = 'specialize:arg(2)'
-
 
 class W_Array(W_DataShape):
     def __init__(self, basicffitype, size):
@@ -57,7 +47,7 @@
                                                 " array length"))
             for num in range(iterlength):
                 w_item = items_w[num]
-                unwrap_value(space, push_elem, result.ll_buffer, num,
+                unwrap_value(space, write_ptr, result.ll_buffer, num,
                              self.itemcode, w_item)
         return space.wrap(result)
 
@@ -118,7 +108,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, push_elem, self.ll_buffer, num,
+        unwrap_value(space, write_ptr, self.ll_buffer, num,
                      self.shape.itemcode, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
@@ -136,7 +126,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, get_elem, self.ll_buffer, num,
+        return wrap_value(space, read_ptr, self.ll_buffer, num,
                           self.shape.itemcode)
 
     def descr_getitem(self, space, w_index):
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._rawffi.array import push_elem
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from pypy.module._rawffi.structure import W_Structure
 from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp,
      unwrap_value, unpack_argshapes, got_libffi_error)
@@ -40,7 +40,7 @@
                 args_w[i] = space.wrap(rffi.cast(rffi.ULONG, ll_args[i]))
         w_res = space.call(w_callable, space.newtuple(args_w))
         if callback_ptr.result is not None: # don't return void
-            unwrap_value(space, push_elem, ll_res, 0,
+            unwrap_value(space, write_ptr, ll_res, 0,
                          callback_ptr.result, w_res)
     except OperationError, e:
         tbprint(space, space.wrap(e.get_traceback()),
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -6,6 +6,7 @@
 
 from rpython.rlib.clibffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
 import rpython.rlib.rposix as rposix
 
@@ -44,6 +45,8 @@
 }
 TYPEMAP_PTR_LETTERS = "POszZ"
 TYPEMAP_NUMBER_LETTERS = "bBhHiIlLqQ?"
+TYPEMAP_FLOAT_LETTERS = "fd" # XXX long doubles are not propperly supported in
+                             # rpython, so we ignore then here
 
 if _MS_WINDOWS:
     TYPEMAP['X'] = ffi_type_pointer
@@ -240,6 +243,52 @@
 )
 
 unroll_letters_for_numbers = unrolling_iterable(TYPEMAP_NUMBER_LETTERS)
+unroll_letters_for_floats = unrolling_iterable(TYPEMAP_FLOAT_LETTERS)
+
+_ARM = rffi_platform.getdefined('__arm__', '')
+
+def read_ptr(ptr, ofs, TP):
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as t_array:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, t_array),
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.sizeof(TP))
+                ptr_val = t_array[0]
+                return ptr_val
+    else:
+        return rffi.cast(T, ptr)[ofs]
+read_ptr._annspecialcase_ = 'specialize:arg(2)'
+
+def write_ptr(ptr, ofs, value):
+    TP = lltype.typeOf(value)
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as s_array:
+                s_array[0] = value
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.cast(rffi.VOIDP, s_array),
+                    rffi.sizeof(TP))
+                return
+    else:
+        rffi.cast(T, ptr)[ofs] = value
+write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):

From noreply at buildbot.pypy.org  Wed Jun 26 14:52:54 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 14:52:54 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: hopefully fixed error on the
 windows ci machine
Message-ID: <20130626125254.BB89E1C00F4@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r479:5b79cdcfa13c
Date: 2013-06-26 14:52 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/5b79cdcfa13c/

Log:	hopefully fixed error on the windows ci machine

diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -362,11 +362,11 @@
     d.set_video_mode(10, 10, 1)
 
     target = model.W_DisplayBitmap.create(space, space.w_Array, 100, 1, d)
-    target.setword(0, 0xFF00)
+    target.setword(0, r_uint(0xFF00))
     assert bin(target.getword(0)) == bin(0xFF00)
-    target.setword(0, 0x00FF00FF)
+    target.setword(0, r_uint(0x00FF00FF))
     assert bin(target.getword(0)) == bin(0x00FF00FF)
-    target.setword(0, 0xFF00FF00)
+    target.setword(0, r_uint(0xFF00FF00))
     assert bin(target.getword(0)) == bin(0xFF00FF00)
     for i in xrange(8):
         assert target.pixelbuffer[i] == 0xff000000

From noreply at buildbot.pypy.org  Wed Jun 26 14:54:38 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 14:54:38 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added guards for
 float-primitives to return NaN and Infinity,
 according to Smalltalk requirements when the math module raises the
 according errors
Message-ID: <20130626125438.92EFA1C00F4@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r480:ee08c90d63b5
Date: 2013-06-26 14:54 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/ee08c90d63b5/

Log:	added guards for float-primitives to return NaN and Infinity,
	according to Smalltalk requirements when the math module raises the
	according errors

diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -319,8 +319,10 @@
 
 @expose_primitive(FLOAT_SIN, unwrap_spec=[float])
 def func(interp, s_frame, f):
-    w_res = interp.space.wrap_float(math.sin(f))
-    return w_res
+    try:
+        return interp.space.wrap_float(math.sin(f))
+    except ValueError:
+        return interp.space.wrap_float(rfloat.NAN)
 
 @expose_primitive(FLOAT_ARCTAN, unwrap_spec=[float])
 def func(interp, s_frame, f):
@@ -339,8 +341,10 @@
 
 @expose_primitive(FLOAT_EXP, unwrap_spec=[float])
 def func(interp, s_frame, f):
-    w_res = interp.space.wrap_float(math.exp(f))
-    return w_res
+    try:
+        return interp.space.wrap_float(math.exp(f))
+    except OverflowError:
+        return interp.space.wrap_float(rfloat.INFINITY)
 
 MAKE_POINT = 18
 

From noreply at buildbot.pypy.org  Wed Jun 26 17:03:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 17:03:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Hopefully no-op change,
 clarify a bit the role of Optimizer.pendingfields
Message-ID: <20130626150331.4EF3B1C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65004:5c5b086f9ff3
Date: 2013-06-26 17:02 +0200
http://bitbucket.org/pypy/pypy/changeset/5c5b086f9ff3/

Log:	Hopefully no-op change, clarify a bit the role of
	Optimizer.pendingfields

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -351,7 +351,8 @@
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
         self.producer = {}
-        self.pendingfields = []
+        self.pendingfields = None # set temporarily to a list, normally by
+                                  # heap.py, as we're about to generate a guard
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.replaces_guard = {}
@@ -546,12 +547,14 @@
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
+            pendingfields = self.pendingfields
+            self.pendingfields = None
             if self.replaces_guard and op in self.replaces_guard:
                 self.replace_op(self.replaces_guard[op], op)
                 del self.replaces_guard[op]
                 return
             else:
-                op = self.store_final_boxes_in_guard(op)
+                op = self.store_final_boxes_in_guard(op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
         if op.result:
@@ -571,12 +574,13 @@
         else:
             assert False
 
-    def store_final_boxes_in_guard(self, op):
+    def store_final_boxes_in_guard(self, op, pendingfields):
+        assert pendingfields is not None
         descr = op.getdescr()
         assert isinstance(descr, compile.ResumeGuardDescr)
         modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
         try:
-            newboxes = modifier.finish(self, self.pendingfields)
+            newboxes = modifier.finish(self, pendingfields)
             if len(newboxes) > self.metainterp_sd.options.failargs_limit:
                 raise resume.TagOverflow
         except resume.TagOverflow:
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -7,7 +7,13 @@
     def __init__(self, unroll):
         self.last_label_descr = None
         self.unroll = unroll
-        
+
+    def emit_operation(self, op):
+        if op.is_guard():
+            if self.optimizer.pendingfields is None:
+                self.optimizer.pendingfields = []
+        Optimization.emit_operation(self, op)
+
     def optimize_CALL_PURE(self, op):
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -27,7 +27,7 @@
     snapshot0 = resume.Snapshot(None, [b0])
     fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
     #
-    opt.store_final_boxes_in_guard(op)
+    opt.store_final_boxes_in_guard(op, [])
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]

From noreply at buildbot.pypy.org  Wed Jun 26 17:05:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 17:05:09 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: Hopefully no-op change,
 clarify a bit the role of Optimizer.pendingfields
Message-ID: <20130626150509.F14F81C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: kill-gen-store-back-in
Changeset: r65005:fae0315b096e
Date: 2013-06-26 17:02 +0200
http://bitbucket.org/pypy/pypy/changeset/fae0315b096e/

Log:	Hopefully no-op change, clarify a bit the role of
	Optimizer.pendingfields

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -351,7 +351,8 @@
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
         self.producer = {}
-        self.pendingfields = []
+        self.pendingfields = None # set temporarily to a list, normally by
+                                  # heap.py, as we're about to generate a guard
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.replaces_guard = {}
@@ -546,12 +547,14 @@
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
+            pendingfields = self.pendingfields
+            self.pendingfields = None
             if self.replaces_guard and op in self.replaces_guard:
                 self.replace_op(self.replaces_guard[op], op)
                 del self.replaces_guard[op]
                 return
             else:
-                op = self.store_final_boxes_in_guard(op)
+                op = self.store_final_boxes_in_guard(op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
         if op.result:
@@ -571,12 +574,13 @@
         else:
             assert False
 
-    def store_final_boxes_in_guard(self, op):
+    def store_final_boxes_in_guard(self, op, pendingfields):
+        assert pendingfields is not None
         descr = op.getdescr()
         assert isinstance(descr, compile.ResumeGuardDescr)
         modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
         try:
-            newboxes = modifier.finish(self, self.pendingfields)
+            newboxes = modifier.finish(self, pendingfields)
             if len(newboxes) > self.metainterp_sd.options.failargs_limit:
                 raise resume.TagOverflow
         except resume.TagOverflow:
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -7,7 +7,13 @@
     def __init__(self, unroll):
         self.last_label_descr = None
         self.unroll = unroll
-        
+
+    def emit_operation(self, op):
+        if op.is_guard():
+            if self.optimizer.pendingfields is None:
+                self.optimizer.pendingfields = []
+        Optimization.emit_operation(self, op)
+
     def optimize_CALL_PURE(self, op):
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -27,7 +27,7 @@
     snapshot0 = resume.Snapshot(None, [b0])
     fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
     #
-    opt.store_final_boxes_in_guard(op)
+    opt.store_final_boxes_in_guard(op, [])
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]

From noreply at buildbot.pypy.org  Wed Jun 26 17:05:11 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 17:05:11 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: Fix.
Message-ID: <20130626150511.355301C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: kill-gen-store-back-in
Changeset: r65006:48d0342b59f5
Date: 2013-06-26 17:04 +0200
http://bitbucket.org/pypy/pypy/changeset/48d0342b59f5/

Log:	Fix.

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -536,7 +536,7 @@
         if self._last_guard_not_forced is not None:
             guard_op = self._last_guard_not_forced
             self.emit_operation(op)
-            guard_op = self.optimizer.store_final_boxes_in_guard(guard_op)
+            guard_op = self.optimizer.store_final_boxes_in_guard(guard_op, [])
             i = len(self.optimizer._newoperations) - 1
             assert i >= 0
             self.optimizer._newoperations.insert(i, guard_op)

From noreply at buildbot.pypy.org  Wed Jun 26 18:07:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 18:07:05 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Sanity checks
Message-ID: <20130626160705.254111C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r285:df93e50691c5
Date: 2013-06-26 17:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/df93e50691c5/

Log:	Sanity checks

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -456,6 +456,12 @@
             assert(!(items[i]->h_tid & GCFLAG_STUB));
             continue;
         }
+        /* Sanity check: a nursery object without the NURSERY_MOVED flag
+           is necessarily a private-without-backup object, or a protected
+           object; it cannot be a public object. */
+        assert(!(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
+        assert(!(obj->h_tid & GCFLAG_PUBLIC));
+        assert(!IS_POINTER(obj->h_revision));
         /* The listed object was not visited.  Unlist it. */
         items[i] = items[--d->list_of_read_objects.size];
     }

From noreply at buildbot.pypy.org  Wed Jun 26 18:07:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 18:07:06 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add debugging checks.
Message-ID: <20130626160706.524111C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r286:55a70bb6e35d
Date: 2013-06-26 17:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/55a70bb6e35d/

Log:	Add debugging checks.

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -60,6 +60,7 @@
     for (i = 0; i < nb_pages; i++)
         accessible_pages[base + i] = 42;
 
+    dprintf(("stm_malloc(%ld): %p\n", (long)sz, result));
     return result;
 }
 
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1177,7 +1177,7 @@
         {
           /* copy the backup copy B back over the now-protected object P,
              and then free B, which will not be used any more. */
-          size_t size = stmcb_size(B);
+          size_t size = stmgc_size(B);
           assert(B->h_tid & GCFLAG_BACKUP_COPY);
           memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
                  ((char *)B) + offsetof(struct stm_object_s, h_revision),
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -155,6 +155,7 @@
         }
         gcp->free_loc_for_size[size_class] = (gcptr)result->h_revision;
         //stm_dbgmem_used_again(result, size_class * WORD, 0);
+        dprintf(("stmgcpage_malloc(%ld): %p\n", (long)size, result));
         return result;
     }
     else {
@@ -168,7 +169,7 @@
 
 void stmgcpage_free(gcptr obj)
 {
-    size_t size = stmcb_size(obj);
+    size_t size = stmgc_size(obj);
     if (size <= GC_SMALL_REQUEST_THRESHOLD) {
         struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
         int size_class = (size + WORD - 1) / WORD;
@@ -272,7 +273,7 @@
 {
     while (gcptrlist_size(&objects_to_trace) > 0) {
         gcptr obj = gcptrlist_pop(&objects_to_trace);
-        stmcb_trace(obj, &visit);
+        stmgc_trace(obj, &visit);
     }
 }
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -6,6 +6,18 @@
     return (d->nursery_base <= (char*)obj && ((char*)obj) < d->nursery_end);
 }
 
+size_t stmgc_size(gcptr obj)
+{
+    assert(!(obj->h_tid & GCFLAG_STUB));
+    return stmcb_size(obj);
+}
+
+void stmgc_trace(gcptr obj, void visit(gcptr *))
+{
+    assert(!(obj->h_tid & GCFLAG_STUB));
+    stmcb_trace(obj, visit);
+}
+
 /************************************************************/
 
 void stmgc_init_nursery(void)
@@ -65,7 +77,7 @@
 
 gcptr stmgc_duplicate(gcptr P)
 {
-    size_t size = stmcb_size(P);
+    size_t size = stmgc_size(P);
     gcptr L = (gcptr)allocate_nursery(size, 0);
     if (L == NULL)
         return stmgc_duplicate_old(P);
@@ -79,7 +91,7 @@
 
 gcptr stmgc_duplicate_old(gcptr P)
 {
-    size_t size = stmcb_size(P);
+    size_t size = stmgc_size(P);
     gcptr L = (gcptr)stmgcpage_malloc(size);
     memcpy(L, P, size);
     L->h_tid |= GCFLAG_OLD;
@@ -237,7 +249,7 @@
     assert(!is_in_nursery(thread_descriptor, id));
     assert(id->h_tid & GCFLAG_OLD);
 
-    size_t size = stmcb_size(obj);
+    size_t size = stmgc_size(obj);
     memcpy(id, obj, size);
     id->h_tid &= ~GCFLAG_HAS_ID;
     id->h_tid |= GCFLAG_OLD;
@@ -311,7 +323,7 @@
 
         visit_if_young(&items[i]);
 
-        stmcb_trace((gcptr)items[i]->h_revision, &visit_if_young);
+        stmgc_trace((gcptr)items[i]->h_revision, &visit_if_young);
     }
 
     d->num_private_from_protected_known_old = size;
@@ -428,7 +440,7 @@
         assert(!(obj->h_tid & GCFLAG_WRITE_BARRIER));
         obj->h_tid |= GCFLAG_WRITE_BARRIER;
 
-        stmcb_trace(obj, &visit_if_young);
+        stmgc_trace(obj, &visit_if_young);
     }
 }
 
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -45,5 +45,7 @@
 int stmgc_minor_collect_anything_to_do(struct tx_descriptor *);
 gcptr stmgc_duplicate(gcptr);
 gcptr stmgc_duplicate_old(gcptr);
+size_t stmgc_size(gcptr);
+void stmgc_trace(gcptr, void visit(gcptr *));
 
 #endif
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -224,7 +224,7 @@
     sd.foreign_pd = foreign_pd;
     memset(&sd.all_stubs, 0, sizeof(sd.all_stubs));
     steal_data = &sd;
-    stmcb_trace(L, &replace_ptr_to_protected_with_stub);
+    stmgc_trace(L, &replace_ptr_to_protected_with_stub);
     g2l_delete_not_used_any_more(&sd.all_stubs);
 
     /* If another thread (the foreign or a 3rd party) does a read

From noreply at buildbot.pypy.org  Wed Jun 26 18:07:07 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 18:07:07 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix again (or attempt to) this case...
Message-ID: <20130626160707.56B8E1C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r287:03917cbbc4f7
Date: 2013-06-26 18:06 +0200
http://bitbucket.org/pypy/stmgc/changeset/03917cbbc4f7/

Log:	Fix again (or attempt to) this case...

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -652,10 +652,18 @@
             {
               /* such an object R might be listed in list_of_read_objects
                  before it was turned from protected to private */
-              if(((gcptr)v)->h_tid & GCFLAG_BACKUP_COPY)
-                continue;
-              /* the backup was stolen */
-              return 0; 
+              if (((gcptr)v)->h_tid & GCFLAG_PUBLIC)
+                {
+                  /* The backup was stolen, but maybe not modified
+                     afterwards.  Check it. */
+                  R = (gcptr)v;
+                  goto retry;
+                }
+              else
+                {
+                  /* The backup was not stolen, everything's fine */
+                  continue;
+                }
             }
           else if ((R->h_tid & (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
                             == (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -523,4 +523,4 @@
 def test_more_multi_thread():
     py.test.skip("more random tests")
     for i in range(200):
-        yield test_multi_thread, 1858 + i
+        yield test_multi_thread, 1742/2 + i

From noreply at buildbot.pypy.org  Wed Jun 26 18:07:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 18:07:08 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Copy this test from c3.
Message-ID: <20130626160708.614B11C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r288:75bca7fddfc3
Date: 2013-06-26 18:06 +0200
http://bitbucket.org/pypy/stmgc/changeset/75bca7fddfc3/

Log:	Copy this test from c3.

diff --git a/c4/test/test_abort.py b/c4/test/test_abort.py
new file mode 100644
--- /dev/null
+++ b/c4/test/test_abort.py
@@ -0,0 +1,107 @@
+import py
+from support import *
+
+# -------------------------------------------------------------------------
+# General warning: this tests aborts by really running a C setjmp(longjmp.
+# This kind of works if there are Python frames in-between, as long as
+# we fix a few things (see _GC_ON_CPYTHON), but not out-of-the-box on PyPy.
+# -------------------------------------------------------------------------
+
+
+def setup_function(f):
+    lib.stm_clear_between_tests()
+    lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
+
+def teardown_function(_):
+    lib.stm_finalize()
+
+
+def test_peform_transaction():
+    seen = []
+    #
+    @perform_transaction
+    def run(retry_counter):
+        if len(seen) < 5:
+            seen.append(retry_counter)
+            return True
+    assert seen == [0] * 5
+
+def test_abort():
+    seen = []
+    #
+    @perform_transaction
+    def run(retry_counter):
+        if len(seen) < 5000:
+            seen.append(retry_counter)
+            abort_and_retry()
+    assert seen == range(5000)
+
+def test_global_to_local_copies():
+    p1 = palloc(HDR)
+    #
+    @perform_transaction
+    def run(retry_counter):
+        p2 = lib.stm_write_barrier(p1)
+        if retry_counter == 0:
+            abort_and_retry()
+        else:
+            major_collect()
+
+def test_old_objects_to_trace():
+    p1 = palloc_refs(1)
+    #
+    @perform_transaction
+    def run(retry_counter):
+        if retry_counter == 0:
+            p2 = lib.stm_write_barrier(p1)
+            lib.stm_push_root(p2)
+            minor_collect()
+            p2 = lib.stm_pop_root()
+            setptr(p2, 0, nalloc(HDR))
+            # write some invalid pointer, we're aborting anyway
+            lib.rawsetlong(p2, 0, 4)
+            abort_and_retry()
+        else:
+            major_collect()
+
+def test_push_restore():
+    @perform_transaction
+    def run(retry_counter):
+        if retry_counter == 0:
+            lib.stm_push_root(ffi.NULL)
+            abort_and_retry()
+
+def test_young_objects_outside_nursery():
+    p1list = [palloc_refs(1) for i in range(20)]
+    #
+    @perform_transaction
+    def run(retry_counter):
+        if retry_counter == 0:
+            # make sure there are a couple of young_objects_outside_nursery
+            p2list = [lib.stm_write_barrier(p1) for p1 in p1list]
+            for p2 in p2list:
+                lib.stm_push_root(p2)
+            minor_collect()
+            # write some invalid pointers, we're aborting anyway
+            for i in range(len(p2list)):
+                p2 = lib.stm_pop_root()
+                lib.rawsetlong(p2, 0, 4)
+            abort_and_retry()
+        else:
+            major_collect()
+
+def test_abort_protected_to_private():
+    pg = palloc(HDR + WORD)
+    p1 = lib.stm_write_barrier(pg)
+    lib.setlong(p1, 0, 32128247)
+    #
+    @perform_transaction
+    def run(retry_counter):
+        assert lib.getlong(pg, 0) == 32128247
+        if retry_counter == 0:
+            lib.setlong(pg, 0, 4737372)
+            abort_and_retry()
+        else:
+            lib.setlong(pg, 0, 838311)
+    #
+    assert lib.getlong(pg, 0) == 838311

From noreply at buildbot.pypy.org  Wed Jun 26 18:54:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 18:54:56 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Untested,
	checked in in case it's needed later
Message-ID: <20130626165456.8E48F1C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r289:6dd90df368c2
Date: 2013-06-26 18:31 +0200
http://bitbucket.org/pypy/stmgc/changeset/6dd90df368c2/

Log:	Untested, checked in in case it's needed later

diff --git a/c4/lists.c b/c4/lists.c
--- a/c4/lists.c
+++ b/c4/lists.c
@@ -129,6 +129,20 @@
   gcptrlist->alloc = 0;
 }
 
+void gcptrlist_compress(struct GcPtrList *gcptrlist)
+{
+  if (gcptrlist->alloc <= gcptrlist->size + 64)
+    return;
+
+  size_t nsize = gcptrlist->size * sizeof(gcptr);
+  gcptr *newitems = realloc(gcptrlist->items, nsize);
+  if (newitems != NULL || nsize == 0)
+    {
+      gcptrlist->items = newitems;
+      gcptrlist->alloc = gcptrlist->size;
+    }
+}
+
 void _gcptrlist_grow(struct GcPtrList *gcptrlist)
 {
   long newalloc = gcptrlist->alloc + (gcptrlist->alloc >> 1) + 64;
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -130,6 +130,7 @@
 }
 
 void gcptrlist_delete(struct GcPtrList *gcptrlist);
+void gcptrlist_compress(struct GcPtrList *gcptrlist);
 void _gcptrlist_grow(struct GcPtrList *gcptrlist);
 void gcptrlist_insert2(struct GcPtrList *gcptrlist, gcptr newitem1,
                        gcptr newitem2);

From noreply at buildbot.pypy.org  Wed Jun 26 18:54:57 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 18:54:57 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Starting on allocating big objects
 (larger than 36 words)
Message-ID: <20130626165457.A9FA81C0397@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r290:a49bf0eadb2d
Date: 2013-06-26 18:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/a49bf0eadb2d/

Log:	Starting on allocating big objects (larger than 36 words)

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -60,7 +60,8 @@
     for (i = 0; i < nb_pages; i++)
         accessible_pages[base + i] = 42;
 
-    dprintf(("stm_malloc(%ld): %p\n", (long)sz, result));
+    dprintf(("stm_malloc(%zu): %p\n", sz, result));
+    assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
     return result;
 }
 
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -88,25 +88,39 @@
     revision_t next, target;
  restart:
     next = ACCESS_ONCE(countdown_next_major_coll);
-    if (next >= size)
+    if (next >= size) {
         target = next - size;
-    else
+    }
+    else {
+        /* we cannot do right now a major collection, but we can speed up
+           the time of the next minor collection (which will be followed
+           by a major collection) */
         target = 0;
+        stmgc_minor_collect_soon();
+    }
     if (!bool_cas(&countdown_next_major_coll, next, target))
         goto restart;
 }
 
-static gcptr allocate_new_page(int size_class)
+static char *alloc_tracked_memory(size_t size)
 {
     /* Adjust the threshold; the caller is responsible for detecting the
        condition that the threshold reached 0. */
-    stmgcpage_reduce_threshold(GC_PAGE_SIZE);
+    stmgcpage_reduce_threshold(size);
 
+    char *result = stm_malloc(size);
+    if (!result) {
+        stm_fatalerror("alloc_tracked_memory: out of memory "
+                       "allocating %zu bytes\n", size);
+    }
+    return result;
+}
+
+static gcptr allocate_new_page(int size_class)
+{
     /* Allocate and return a new page for the given size_class. */
-    page_header_t *page = (page_header_t *)stm_malloc(GC_PAGE_SIZE);
-    if (!page) {
-        stm_fatalerror("allocate_new_page: out of memory!\n");
-    }
+    page_header_t *page = (page_header_t *)alloc_tracked_memory(GC_PAGE_SIZE);
+
     struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
     gcp->count_pages++;
     count_global_pages++;
@@ -155,11 +169,13 @@
         }
         gcp->free_loc_for_size[size_class] = (gcptr)result->h_revision;
         //stm_dbgmem_used_again(result, size_class * WORD, 0);
-        dprintf(("stmgcpage_malloc(%ld): %p\n", (long)size, result));
+        dprintf(("stmgcpage_malloc(%zu): %p\n", size, result));
         return result;
     }
     else {
-        stm_fatalerror("XXX stmgcpage_malloc: too big!\n");
+        gcptr result = (gcptr)alloc_tracked_memory(size);
+        dprintf(("stmgcpage_malloc(BIG %zu): %p\n", size, result));
+        return result;
     }
 }
 
@@ -183,7 +199,7 @@
         //stm_dbgmem_not_used(obj, size_class * WORD, 0);
     }
     else {
-        stm_fatalerror("XXX stmgcpage_free: too big!\n");
+        stm_free(obj, stmgc_size(obj));
     }
 }
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -44,6 +44,12 @@
     gcptrlist_delete(&d->public_with_young_copy);
 }
 
+void stmgc_minor_collect_soon(void)
+{
+    struct tx_descriptor *d = thread_descriptor;
+    d->nursery_current = d->nursery_end;
+}
+
 static char *collect_and_allocate_size(size_t size);  /* forward */
 
 inline static char *allocate_nursery(size_t size, int can_collect)
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -47,5 +47,6 @@
 gcptr stmgc_duplicate_old(gcptr);
 size_t stmgc_size(gcptr);
 void stmgc_trace(gcptr, void visit(gcptr *));
+void stmgc_minor_collect_soon(void);
 
 #endif
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -413,3 +413,14 @@
     lib.stm_pop_root()
     p1b = lib.stm_read_barrier(p1)
     check_not_free(p1b)
+
+def test_big_old_object():
+    p1 = oalloc(HDR + 50 * WORD)
+    # assert did not crash
+
+def test_big_old_object_free():
+    p1 = oalloc(HDR + 50 * WORD)
+    p1b = lib.stm_write_barrier(p1)
+    assert p1b == p1
+    lib.stm_commit_transaction()
+    lib.stm_begin_inevitable_transaction()

From noreply at buildbot.pypy.org  Wed Jun 26 19:14:20 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 19:14:20 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: fixed a problem with
 SPyVM-class for our test-image
Message-ID: <20130626171420.46A1F1C021A@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r481:2d4cc7eead42
Date: 2013-06-26 15:18 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/2d4cc7eead42/

Log:	fixed a problem with SPyVM-class for our test-image

diff too long, truncating to 2000 out of 79989 lines

diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
--- a/images/Squeak4.5-12568.changes
+++ b/images/Squeak4.5-12568.changes
@@ -36,4 +36,4 @@
 Workspace allInstances do: [:w | w topView delete].
 ReleaseBuilderFor4dot4 prepareNewBuild.
 Smalltalk snapshot: true andQuit: true.
-!

----End fileIn of a stream----!

----SNAPSHOT----{31 March 2013 . 3:27:34 pm} Squeak4.5-12327.image priorSource: 7430688!
!Installer methodsFor: 'squeakmap' stamp: 'fbs 1/28/2013 19:25' prior: 57597950!
packageAndVersionFrom: pkg
	| p |
	p := ReadStream on: pkg .
	^{(p upTo: $(). p upTo: $)} collect: [:s | s withBlanksTrimmed].! !

"Installer-Core"!
!Categorizer methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 16:58'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!ClassCategoryReader methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:21'!
scanFrom: aStream environment: anEnvironment
	"File in methods from the stream, aStream."
	| methodText |
	[methodText := aStream nextChunkText.
	 methodText size > 0] whileTrue:
		[class 
			compile: methodText 
			environment: anEnvironment
			classified: category
			withStamp: changeStamp 
			notifying: nil]! !
!ClassCommentReader methodsFor: 'as yet unclassified' stamp: 'cwp 6/20/2012 17:22'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!Metaclass methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:29'!
bindingOf: varName environment: anEnvironment 
	^ thisClass classBindingOf: varName environment: anEnvironment! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 20:03' prior: 22505876!
\\ aNumber 
	"Primitive. Take the receiver modulo the argument. The result is the
	remainder rounded towards negative infinity, of the receiver divided
	by the argument. Fail if the argument is 0. Fail if either the argument
	or the result is not a SmallInteger or a LargePositiveInteger less than
	2-to-the-30th (1073741824). Optional. See Object documentation whatIsAPrimitive."

	<primitive: 31>
	aNumber isInteger
		ifTrue:
			[| neg qr q r |
			neg := self negative == aNumber negative == false.
			qr := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: neg).
			q := qr first normalize.
			r := qr last normalize.
			^(q negative
				ifTrue: [r isZero not]
				ifFalse: [q isZero and: [neg]])
					ifTrue: [r + aNumber]
					ifFalse: [r]].
	^super \\ aNumber
	! !
!LargePositiveInteger methodsFor: 'converting' stamp: 'nice 1/27/2012 22:41' prior: 37616324!
asFloat
	"Answer a Float that best approximates the value of the receiver.
	This algorithm is optimized to process only the significant digits of a LargeInteger.
	And it does honour IEEE 754 round to nearest even mode in case of excess precision (see details below)."
	
	"How numbers are rounded in IEEE 754 default rounding mode:
	A shift is applied so that the highest 53 bits are placed before the floating point to form a mantissa.
	The trailing bits form the fraction part placed after the floating point.
	This fractional number must be rounded to the nearest integer.
	If fraction part is 2r0.1, exactly between two consecutive integers, there is a tie.
	The nearest even integer is chosen in this case.
	Examples (First 52bits of mantissa are omitted for brevity):
	2r0.00001 is rounded downward to 2r0
	2r1.00001 is rounded downward to 2r1
	2r0.1 is a tie and rounded to 2r0 (nearest even)
	2r1.1 is a tie and rounded to 2r10 (nearest even)
	2r0.10001 is rounded upward to 2r1
	2r1.10001 is rounded upward to 2r10
	Thus, if the next bit after floating point is 0, the mantissa is left unchanged.
	If next bit after floating point is 1, an odd mantissa is always rounded upper.
	An even mantissa is rounded upper only if the fraction part is not a tie."
	
	"Algorihm details:
	The floating point hardware can perform the rounding correctly with several excess bits as long as there is a single inexact operation.
	This can be obtained by splitting the mantissa plus excess bits in two part with less bits than Float precision.
	Note 1: the inexact flag in floating point hardware must not be trusted because in some cases the operations would be exact but would not take into account some bits that were truncated before the Floating point operations.
	Note 2: the floating point hardware is presumed configured in default rounding mode."
	
	| mantissa shift excess result n |

	"Check how many bits excess the maximum precision of a Float mantissa."
	excess := self highBitOfMagnitude - Float precision.
	excess > 7
		ifTrue:
			["Remove the excess bits but seven."
			mantissa := self bitShiftMagnitude: 7 - excess.
			shift := excess - 7.
			"An even mantissa with a single excess bit immediately following would be truncated.
			But this would not be correct if above shift has truncated some extra bits.
			Check this case, and round excess bits upper manually."
			((mantissa digitAt: 1) = 2r01000000 and: [self anyBitOfMagnitudeFrom: 1 to: shift])
				ifTrue: [mantissa := mantissa + 1]]
		ifFalse:
			[mantissa := self.
			shift := 0].

	"There will be a single inexact round off at last iteration"
	result := (mantissa digitAt: (n := mantissa digitLength)) asFloat.
	[(n := n - 1) > 0] whileTrue: [
		result := 256.0 * result + (mantissa digitAt: n) asFloat].
	^result timesTwoPower: shift.! !
!LargePositiveInteger methodsFor: 'private' stamp: 'nice 12/30/2012 14:25'!
primitiveQuo: anInteger 
	"Primitive. Divide the receiver by the argument and return the result.
	Round the result down towards zero to make it a whole integer. Fail if
	the argument is 0. Fail if either the argument or the result is not a
	SmallInteger or a LargePositiveInteger less than 2-to-the-30th (1073741824). Optional. See
	Object documentation whatIsAPrimitive."

	<primitive: 33>
	^nil! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 14:34'!
rem: aNumber 
	"Remainder defined in terms of quo:. See super rem:.
	This is defined only to speed up case of very large integers."

	(self primitiveQuo: aNumber)
		ifNotNil: [:quo | ^self - (quo * aNumber)].
	 aNumber isInteger
		ifTrue:
			[| ng rem |
			ng := self negative == aNumber negative == false.
			rem := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: ng) at: 2.
			^ rem normalize].
	^super rem: aNumber! !
!LargeNegativeInteger methodsFor: 'converting' stamp: 'nice 1/1/2013 15:42' prior: 37616204!
asFloat
	^super asFloat negated! !
!UndefinedObject methodsFor: 'class hierarchy' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:32'!
bindingOf: varName environment: anEnvironment
	^superclass bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:30'!
classBindingOf: varName environment: anEnvironment
	^self bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:37'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor
	"Postprocesses a literal scanned by Scanner scanToken (esp. xLitQuote).
	If scannedLiteral is not an association, answer it.
	Else, if it is of the form:
		nil->#NameOfMetaclass
	answer nil->theMetaclass, if any has that name, else report an error.
	Else, if it is of the form:
		#NameOfGlobalVariable->anythiEng
	answer the global, class, or pool association with that nameE, if any, else
	add it to Undeclared a answer the new Association."

	| key value |
	(scannedLiteral isVariableBinding)
		ifFalse: [^ scannedLiteral].
	key := scannedLiteral key.
	value := scannedLiteral value.
	key ifNil: "###<metaclass soleInstance name>"
		[(self bindingOf: value environment: anEnvironment) ifNotNil:
			[:assoc|
			(assoc value isKindOf: Behavior) ifTrue: 
				[^ nil->assoc value class]].
			 requestor notify: 'No such metaclass'.
			 ^false].
	(key isSymbol) ifTrue: "##<global var name>"
		[(self bindingOf: key environment: anEnvironment) ifNotNil:
			[:assoc | ^assoc].
		^ anEnvironment undeclared: key].
	requestor notify: '## must be followed by a non-local variable name'.
	^false

"	Form literalScannedAs: 14 notifying: nil 14
	Form literalScannedAs: #OneBitForm notiEfying: nil  OneBitForm
	Form literalScannedAs: ##OneBitForm notifying: nil  OneBitForm->a Form
	Form literalScannedAs: ##Form notifying: nil   Form->Form
	Form literalScannedAs: ###Form notifying: nil   nilE->Form class
"! !
!Fraction methodsFor: 'converting' stamp: 'nice 11/21/2011 22:34' prior: 37619655!
asFloat
	"Answer a Float that closely approximates the value of the receiver.
	This implementation will answer the closest floating point number to the receiver.
	In case of a tie, it will use the IEEE 754 round to nearest even mode.
	In case of overflow, it will answer +/- Float infinity."

	| a b mantissa exponent hasTruncatedBits lostBit n ha hb hm |
	a := numerator abs.
	b := denominator.	"denominator is always positive"
	ha := a highBitOfMagnitude.
	hb := b highBitOfMagnitude.
	
	"Number of bits to keep in mantissa plus one to handle rounding."
	n := 1 + Float precision.

	"If both numerator and denominator are represented exactly in floating point number,
	then fastest thing to do is to use hardwired float division."
	(ha < n and: [hb < n]) ifTrue: [^numerator asFloat / denominator asFloat].

	"Shift the fraction by a power of two exponent so as to obtain a mantissa with n bits.
	First guess is rough, the mantissa might have n+1 bits."
	exponent := ha - hb - n.
	exponent >= 0
		ifTrue: [b := b bitShift: exponent]
		ifFalse: [a := a bitShift: exponent negated].
	mantissa := a quo: b.
	hasTruncatedBits := a > (mantissa * b).
	hm := mantissa highBit.
	
	"Check for gradual underflow, in which case the mantissa will loose bits.
	Keep at least one bit to let underflow preserve the sign of zero."
	lostBit := Float emin - (exponent + hm - 1).
	lostBit > 0 ifTrue: [n := n - lostBit max: 1].

	"Remove excess bits in the mantissa."
	hm > n
		ifTrue:
			[exponent := exponent + hm - n.
			hasTruncatedBits := hasTruncatedBits or: [mantissa anyBitOfMagnitudeFrom: 1 to: hm - n].
			mantissa := mantissa bitShift: n - hm].

	"Check if mantissa must be rounded upward.
	The case of tie (mantissa odd & hasTruncatedBits not)
	will be handled by Integer>>asFloat."
	(hasTruncatedBits and: [mantissa odd])
		ifTrue: [mantissa := mantissa + 1].

	^ (self positive
			ifTrue: [mantissa asFloat]
			ifFalse: [mantissa asFloat negated])
		timesTwoPower: exponent! !
!Float methodsFor: 'arithmetic' stamp: 'nice 12/20/2012 23:16' prior: 20878776!
negated
	"Answer a Number that is the negation of the receiver.
	Implementation note: this version cares of negativeZero."

	^-1.0 * self! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:21'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor
	^ self 
		compile: text 
		environment: anEnvironment 
		classified: category 
		withStamp: changeStamp 
		notifying: requestor 
		logSource: self acceptsLoggingOfCompilation! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 12/27/2012 13:17'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor logSource: logSource
	| methodAndNode context methodNode |
	context := CompilationCue
		source: text
		class: self
		environment: anEnvironment
		category: category
		requestor: requestor.
	methodNode := self newCompiler compile: context ifFail: [^ nil].
	methodAndNode := CompiledMethodWithNode 
		generateMethodFromNode: methodNode 
		trailer: self defaultMethodTrailer.

	logSource ifTrue: [
		self logMethodSource: text forMethodWithNode: methodAndNode 
			inCategory: category withStamp: changeStamp notifying: requestor.
	].
	self addAndClassifySelector: methodAndNode selector withMethod: methodAndNode 
		method inProtocol: category notifying: requestor.
	self instanceSide noteCompilationOf: methodAndNode selector meta: self isClassSide.
	^ methodAndNode selector! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:47'!
bindingOf: varName environment: anEnvironment
	"Answer the binding of some variable resolved in the scope of the receiver"
	| aSymbol binding |
	aSymbol := varName asSymbol.

	"First look in classVar dictionary."
	binding := self classPool bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Next look in shared pools."
	self sharedPools do:[:pool | 
		binding := pool bindingOf: aSymbol.
		binding ifNotNil:[^binding].
	].

	"Next look in declared environment."
	binding := anEnvironment bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Finally look higher up the superclass chain and fail at the end."
	superclass == nil
		ifTrue: [^ nil]
		ifFalse: [^ superclass bindingOf: aSymbol].

! !

"Kernel"!

ParseNode subclass: #Encoder
	instanceVariableNames: 'scopeTable nTemps supered requestor class selector literalStream selectorSet litIndSet litSet sourceRanges globalSourceRanges addedSelectorAndMethodClassLiterals optimizedSelectors cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Encoder commentStamp: 'cwp 12/26/2012 23:29' prior: 36323851!
I encode names and literals into tree nodes with byte codes for the compiler. Byte codes for literals are not assigned until the tree-sizing pass of the compiler, because only then is it known which literals are actually needed. I also keep track of sourceCode ranges during parsing and code generation so I can provide an inverse map for the debugger.!

Scanner subclass: #Parser
	instanceVariableNames: 'here hereType hereMark hereEnd prevMark prevEnd encoder requestor parseNode failBlock requestorOffset tempsMark doitFlag properties category queriedUnusedTemporaries cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Parser commentStamp: 'cwp 12/26/2012 23:34' prior: 38557958!
I parse Smalltalk syntax and create a MethodNode that is the root of the parse tree. I look one token ahead.!

Object subclass: #CompilationCue
	instanceVariableNames: 'source context receiver class environment category requestor'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

Object subclass: #Compiler
	instanceVariableNames: 'sourceStream requestor class category context parser cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Compiler commentStamp: 'cwp 12/26/2012 23:17' prior: 59257505!
The compiler accepts Smalltalk source code and compiles it with respect to a given class. The user of the compiler supplies a context so that temporary variables are accessible during compilation. If there is an error, a requestor (usually a kind of StringHolderController) is sent the message notify:at:in: so that the error message can be displayed. If there is no error, then the result of compilation is a MethodNode, which is the root of a parse tree whose nodes are kinds of ParseNodes. The parse tree can be sent messages to (1) generate code for a CompiledMethod (this is done for compiling methods or evaluating expressions); (2) pretty-print the code (for formatting); or (3) produce a map from object code back to source code (used by debugger program-counter selection). See also Parser, Encoder, ParseNode.!
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:34'!
init: aCue notifying: anObject
	"The use of the variable requestor is a bit confusing here. This is
	*not* the original requestor, which is available through the cue.
	It's the Parser instance that is using the encoder."

	self setCue: aCue.
	requestor := anObject.
	nTemps := 0.
	supered := false.
	self initScopeAndLiteralTables.
	cue getClass variablesAndOffsetsDo:
		[:variable "<String|CFieldDefinition>" :offset "<Integer|nil>" |
		offset isNil
			ifTrue: [scopeTable at: variable name put: (FieldNode new fieldDefinition: variable)]
			ifFalse: [scopeTable
						at: variable
						put: (offset >= 0
								ifTrue: [InstanceVariableNode new
											name: variable index: offset]
								ifFalse: [MaybeContextInstanceVariableNode new
											name: variable index: offset negated])]].
	cue context ~~ nil ifTrue:
		[| homeNode |
		 homeNode := self bindTemp: self doItInContextName.
		 "0th temp = aContext passed as arg"
		 cue context tempNames withIndexDo:
			[:variable :index|
			scopeTable
				at: variable
				put: (MessageAsTempNode new
						receiver: homeNode
						selector: #namedTempAt:
						arguments: (Array with: (self encodeLiteral: index))
						precedence: 3
						from: self)]].
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/26/2012 23:30'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy instance variables for methods that
	don't use cue yet"
	class := cue getClass.! !
!Dictionary methodsFor: '*Compiler' stamp: 'cwp 6/22/2012 09:17'!
bindingOf: varName ifAbsent: aBlock

	^self associationAt: varName ifAbsent: aBlock! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:37'!
init: sourceStream cue: aCue failBlock: aBlock

	self setCue: aCue.
	failBlock := aBlock.
	requestorOffset := 0.
	super scan: sourceStream.
	prevMark := hereMark := mark.
	self advance
! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/26/2012 23:41'!
parse: sourceStream cue: aCue noPattern: noPattern ifFail: aBlock 
	"Answer a MethodNode for the argument, sourceStream, that is the root of
	 a parse tree. Parsing is done with respect to the CompilationCue to 
	 resolve variables. Errors in parsing are reported to the cue's requestor; 
	 otherwise aBlock is evaluated. The argument noPattern is a Boolean that is
	 true if the the sourceStream does not contain a method header (i.e., for DoIts)."

	| methNode repeatNeeded myStream s p subSelection |
	myStream := sourceStream.
	[repeatNeeded := false.
	 p := myStream position.
	 s := myStream upToEnd.
	 myStream position: p.
	 subSelection := aCue requestor notNil and: [aCue requestor selectionInterval = (p + 1 to: p + s size)].
	 self encoder init: aCue notifying: self.
	 self init: myStream cue: aCue failBlock: [^ aBlock value].
	 doitFlag := noPattern.
	 failBlock:= aBlock.
	 [methNode := self method: noPattern context: cue context] 
		on: ReparseAfterSourceEditing 
		do:	[ :ex |
			repeatNeeded := true.
			myStream := subSelection
							ifTrue:
								[ReadStream
									on: cue requestor text string
									from: cue requestor selectionInterval first
									to: cue requestor selectionInterval last]
							ifFalse:
								[ReadStream on: cue requestor text string]].
	 repeatNeeded] whileTrue:
		[encoder := self encoder class new].
	methNode sourceText: s.
	^methNode
! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:35'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy variables for methods that don't use cue yet."
	requestor := cue requestor.
	category := cue category.! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
class: aClass
	^ self 
		context: nil
		class: aClass
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
context: aContext class: aClass requestor: anObject
	^ self
		source: nil
		context: aContext
		receiver: nil
		class: aClass
		environment: (aClass ifNotNil: [aClass environment])
		category: nil
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aTextOrStream class: aClass environment: anEnvironment category: aString requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		receiver: nil
		class: aClass
		environment: anEnvironment
		category: aString
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
source: aTextOrStream context: aContext class: aClass category: aString requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		receiver: (aContext ifNotNil: [aContext receiver])
		class: aClass 
		environment: (aClass ifNotNil: [aClass environment])
		category: aString 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream context: aContext class: aClass requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		class: aClass 
		category: nil 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:55'!
source: aTextOrStream context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	^ self basicNew
		initializeWithSource: aTextOrStream 
		context: aContext 
		receiver: recObject 
		class: aClass 
		environment: anEnvironment 
		category: aString 
		requestor: reqObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aString environment: anEnvironment
	^ self 
		source: aString
		context: nil
		receiver: nil
		class: UndefinedObject
		environment: anEnvironment
		category: nil
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		class: nil
		requestor: anObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/20/2012 09:39'!
bindingOf: aSymbol
	^ class bindingOf: aSymbol environment: environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
category
	^ category! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 12/26/2012 23:19'!
context
	^ context! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
environment
	^ environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
getClass
	^ class! !
!CompilationCue methodsFor: 'initialization' stamp: 'cwp 12/26/2012 23:16'!
initializeWithSource: aTextOrString context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	self initialize.
	source := aTextOrString isStream ifTrue: [aTextOrString contents] ifFalse: [aTextOrString].
	context := aContext.
	receiver := recObject.
	class := aClass.
	environment := anEnvironment.
	category := aString.
	requestor := reqObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: anObject notifying: anEncoder
	^ class literalScannedAs: anObject environment: environment notifying: anEncoder! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
receiver
	^ receiver! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
requestor
	^ requestor! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
source
	^ source! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:44'!
sourceStream
	^ source readStream! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 6/20/2012 17:25'!
evaluate: aString environment: anEnvironment
	^ self 
		evaluate: aString 
		environment: anEnvironment 
		logged: false! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 12/27/2012 12:36'!
evaluate: aString environment: anEnvironment logged: aBoolean
	| cue |
	cue := CompilationCue
		source: aString
		environment: anEnvironment.
		
	^ self new
		evaluate: aString
		cue: cue
		ifFail: [^ nil]
		logged: aBoolean! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 13:18'!
compile: aCue ifFail: failBlock 
	"Answer a MethodNode. If the MethodNode can not be created, notify 
	the requestor in the contxt. If the requestor is nil, evaluate failBlock 
	instead. The MethodNode is the root  of a parse tree. It can be told 
	to generate a CompiledMethod to be installed in the method dictionary 
	of the class specified by the context."
	
	self setCue: aCue.
	self source: cue source.
	^self
		translate: sourceStream
		noPattern: false
		ifFail: failBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:06'!
evaluate: textOrStream cue: aCue ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	a method. Finally, the compiled method is invoked from here via 	withArgs:executeMethod:, hence the system no longer creates Doit method
	litter on errors."

	| methodNode method value toLog itsSelection itsSelectionString |
	self setCue: aCue.
	self source: textOrStream.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].

	method := self interactive
				ifTrue: [methodNode generateWithTempNames]
				ifFalse: [methodNode generate].

	value := cue receiver
				withArgs: (cue context ifNil: [#()] ifNotNil: [{cue context}])
				executeMethod: method.

	logFlag ifTrue:
		[toLog := ((cue requestor respondsTo: #selection)  
			and:[(itsSelection := cue requestor selection) notNil
			and:[(itsSelectionString := itsSelection asString) isEmptyOrNil not]])
				ifTrue:[itsSelectionString]
				ifFalse:[sourceStream contents].
		SystemChangeNotifier uniqueInstance evaluated: toLog context: cue context].
	^ value
! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:20'!
setCue: aCue
	cue := aCue.
	
	"Set legacy instance variables for methods that don't use cue yet."
	requestor := cue requestor.
	class := cue getClass.
	category := cue category.
	context := cue context.! !
!Compiler methodsFor: 'private' stamp: 'cwp 6/19/2012 21:58'!
source: textOrStream
	sourceStream := (textOrStream isKindOf: PositionableStream)
		ifTrue: [ textOrStream ]
		ifFalse: [ ReadStream on: textOrStream asString ]! !

"Compiler"!
!SmartRefStream class methodsFor: 'i/o' stamp: 'cwp 6/20/2012 17:42'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!SmartRefStream methodsFor: 'read write' stamp: 'cwp 6/20/2012 17:41'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!ImageSegment methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:23'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!PseudoClass methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| aString newTranslations assoc currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[Locale
		currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:26'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| newTranslations currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[| aString assoc |
	Locale currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!ObjectScanner methodsFor: 'scanning' stamp: 'cwp 6/20/2012 17:39'!
scanFrom: aByteStream environment: anEnvironment
	"This should probably be reimplemented using an environment
	for compilation. For now, don't change anything"
	^ self scanFrom: aByteStream! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 09:16'!
bindingOf: varName ifAbsent: aBlock
	"SystemDictionary includes Symbols only"
	^super bindingOf: varName asSymbol ifAbsent: aBlock! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 15:48'!
undeclared
	^ self at: #Undeclared! !

"System"!
!ExceptionTests methodsFor: 'testing-outer' stamp: 'fbs 1/1/2013 22:14' prior: 40840955!
expectedFailures
	^ #().! !

"Tests"!

ReleaseBuilder subclass: #ReleaseBuilderFor4dot5
	instanceVariableNames: ''
	classVariableNames: ''
	poolDictionaries: ''
	category: 'ReleaseBuilder'!
!ReleaseBuilderFor4dot5 commentStamp: 'fbs 1/1/2013 20:25' prior: 0!
The release builder for Squeak 4.5!
!ReleaseBuilder class methodsFor: 'scripts' stamp: 'fbs 12/31/2012 20:43'!
transferCurrentPackagesAsUser: username password: password
	"Copy the packages currently loaded in the image from the trunk repository to my releaseRepository."
	| trunkRep releaseRep |
	trunkRep := self trunkRepository.
	releaseRep := self releaseRepository
		user: username;
		password: password;
		yourself.
	MCWorkingCopy allManagers do:
		[ : eachWorkingCopy | eachWorkingCopy ancestors do:
			[ : eachVersionInfo | (releaseRep includesVersionNamed: eachVersionInfo versionName) ifFalse:
				[ (trunkRep versionWithInfo: eachVersionInfo)
					ifNil: [ Warning signal: eachVersionInfo name , ' not found in ', trunkRep ]
					ifNotNilDo: [ : ver | releaseRep storeVersion: ver ] ] ] ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
openWelcomeWorkspaces
	TheWorldMainDockingBar instance
		showWelcomeText: #squeakUserInterface
		label: 'Squeak User Interface'
		in: (40 @ 40 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #workingWithSqueak
		label: 'Working With Squeak'
		in: (80 @ 80 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #licenseInformation
		label: 'License Information'
		in: (120 @ 120 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeFutureDirections
		label: 'Future Directions'
		in: (160 @ 160 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeToSqueak
		label: 'Welcome to Squeak 4.5'
		in: (200 @ 200 extent: 500 @ 300)! !
!ReleaseBuilderFor4dot5 class methodsFor: 'scripts' stamp: 'fbs 1/1/2013 20:22'!
prepareNewBuild
	super prepareNewBuild.
	MCMockPackageInfo initialize.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:24'!
releaseRepository
	"At release time, change 'trunk' to 'squeak45'."
	^ MCHttpRepository
		location: 'http://source.squeak.org/trunk'
		user: 'squeak'
		password: 'squeak'! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:22'!
setDisplayExtent: extent
"Uncomment next line when the primitives become available in the Squeak VM."
"	DisplayScreen hostWindowSize: extent."
	Display extent = extent ifFalse: [ Warning signal: 'Display extent not set to ', extent ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
setPreferences
	Preferences 
		installBrightWindowColors ;
		setPreference: #scrollBarsWithoutMenuButton toValue: true ;
		setPreference: #swapMouseButtons toValue: true ;
		setPreference: #annotationPanes toValue: true ;
		setPreference: #showSplitterHandles toValue: false ;
		setPreference: #showBoundsInHalo toValue: true ;
		setPreference: #alternateHandlesLook toValue: false ;
		setPreference: #roundedMenuCorners toValue: false ;
		setPreference: #roundedWindowCorners toValue: false.
	PluggableButtonMorph roundedButtonCorners: false.
	FillInTheBlankMorph roundedDialogCorners: false.
	Workspace shouldStyle: false.
	NetNameResolver enableIPv6: true.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
switchToNewRepository
	| old44Repository |
	MCMcmUpdater defaultUpdateURL: self releaseRepository description.
	old44Repository := MCRepositoryGroup default repositories
				detect: [:each | each description includesSubString: 'squeak44'] ifNone: [nil].
	old44Repository
		ifNotNil: [MCRepositoryGroup default removeRepository: old44Repository].
	MCRepositoryGroup default addRepository: self releaseRepository! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
versionString
	^ 'Squeak4.5'.! !

ReleaseBuilder class removeSelector: #transferCurrentPackages!

"ReleaseBuilder"!
!Environment class methodsFor: 'as yet unclassified' stamp: 'cwp 1/1/2013 18:52' prior: 40834114!
initialize
	self install! !

"Environments"!
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:59' prior: 52081878!
initPattern: aString notifying: req return: aBlock

	| result |
	self
		init: (ReadStream on: aString asString)
		cue: (CompilationCue source: aString requestor: req)
		failBlock: [^nil].
	encoder := self.
	result := aBlock value: (self pattern: false inContext: nil).
	encoder := failBlock := nil.  "break cycles"
	^result! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:01' prior: 34175471!
parse: sourceStream class: class category: aCategory noPattern: noPattern context: aContext notifying: req ifFail: aBlock 
	| c |
	c := CompilationCue
			source: sourceStream
			context: aContext
			class: class
			category: aCategory
			requestor: req.
	^ self 
		parse: sourceStream 
		cue: c 
		noPattern: noPattern 
		ifFail: aBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:11' prior: 34183963!
evaluate: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	 a method. If aContext is not nil, the text can refer to temporaries in that
	 context (the Debugger uses this). If aRequestor is not nil, then it will receive
	 a notify:at: message before the attempt to evaluate is aborted. Finally, the 
	 compiled method is invoked from here via withArgs:executeMethod:, hence
	 the system no longer creates Doit method litter on errors."
	
	| theClass |
	theClass := ((aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class).
	self setCue: (CompilationCue
		source: textOrStream
		context: aContext
		receiver: receiver
		class: theClass
		environment: theClass environment
		category: nil
		requestor: aRequestor).
	^ self evaluate: textOrStream cue: cue ifFail: failBlock logged: logFlag! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:17' prior: 34185488!
from: textOrStream class: aClass classified: aCategory context: aContext notifying: req
	self source: textOrStream.
	self setCue: 
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			category: aCategory
			requestor: req)! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:55' prior: 50781309!
from: textOrStream class: aClass context: aContext notifying: req
	self source: textOrStream.
	self setCue:
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			requestor: req)
! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/27/2012 09:41' prior: 50996506!
init: aClass context: aContext notifying: anObject
	| c |
	c := CompilationCue
		context: aContext 
		class: aClass 
		requestor: nil.
	self init: c notifying: anObject! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:58' prior: 39061698!
temps: tempVars literals: lits class: cl 
	"Initialize this encoder for decompilation."

	self setCue: (CompilationCue class: cl).
	supered := false.
	nTemps := tempVars size.
	tempVars do: [:node | scopeTable at: node name put: node].
	literalStream := WriteStream on: (Array new: lits size).
	literalStream nextPutAll: lits.
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32.! !

"Compiler"!
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:48' prior: 36026010!
addClassVarName: aString 
	"Add the argument, aString, as a class variable of the receiver.
	Signal an error if the first character of aString is not capitalized,
	or if it is already a variable named in the class."
	| symbol oldState |
	oldState := self copy.
	aString first canBeGlobalVarInitial
		ifFalse: [^self error: aString, ' class variable name should be capitalized; proceed to include anyway.'].
	symbol := aString asSymbol.
	self withAllSubclasses do: 
		[:subclass | 
		(self canFindWithoutEnvironment: symbol) ifTrue: [
			(DuplicateVariableError new)
				superclass: superclass; "fake!!!!!!"
				variable: aString;
				signal: aString, ' is already defined']].
	classPool == nil ifTrue: [classPool := Dictionary new].
	(classPool includesKey: symbol) ifFalse: 
		["Pick up any refs in Undeclared"
		classPool declare: symbol from: environment undeclared.
		SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: oldState to: self]! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:48' prior: 54782024!
bindingOf: varName
	^ self bindingOf: varName environment: self environment! !
!Class methodsFor: 'organization' stamp: 'cwp 6/25/2012 18:25' prior: 54785804!
category
	"Answer the system organization category for the receiver. First check whether the
	category name stored in the ivar is still correct and only if this fails look it up
	(latter is much more expensive)"

	category ifNotNil: [ :symbol |
		((self environment organization listAtCategoryNamed: symbol) includes: self name)
			ifTrue: [ ^symbol ] ].
	category := self environment organization categoryOfElement: self name.
	^category! !
!Class methodsFor: 'initialize-release' stamp: 'cwp 6/22/2012 15:49' prior: 36027730!
declare: varString 
	"Declare class variables common to all instances. Answer whether 
	recompilation is advisable."

	| newVars conflicts |
	
	newVars := 
		(Scanner new scanFieldNames: varString)
			collect: [:x | x asSymbol].
	newVars do:
		[:var | var first canBeGlobalVarInitial
			ifFalse: [self error: var, ' class variable name should be capitalized; proceed to include anyway.']].
	conflicts := false.
	classPool == nil 
		ifFalse: [(classPool keys reject: [:x | newVars includes: x]) do: 
					[:var | self removeClassVarName: var]].
	(newVars reject: [:var | self classPool includesKey: var])
		do: [:var | "adding"
			"check if new vars defined elsewhere"
			(self canFindWithoutEnvironment: var) ifTrue: [
					(DuplicateVariableError new)
						superclass: superclass; "fake!!!!!!"
						variable: var;
						signal: var, ' is already defined'.
					conflicts := true]].
	newVars size > 0
		ifTrue: 
			[classPool := self classPool.
			"in case it was nil"
			newVars do: [:var | classPool declare: var from: environment undeclared]].
	^conflicts! !
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:49' prior: 54802475!
removeClassVarName: aString 
	"Remove the class variable whose name is the argument, aString, from 
	the names defined in the receiver, a class. Create an error notification if 
	aString is not a class variable or if it is still being used in the code of 
	the class."

	| aSymbol |
	aSymbol := aString asSymbol.
	(classPool includesKey: aSymbol)
		ifFalse: [^self error: aString, ' is not a class variable'].
	self withAllSubclasses do:[:subclass |
		(Array with: subclass with: subclass class) do:[:classOrMeta |
			(classOrMeta whichSelectorsReferTo: (classPool associationAt: aSymbol))
				isEmpty ifFalse: [
					InMidstOfFileinNotification signal ifTrue: [
						Transcript cr; show: self name, ' (' , aString , ' is Undeclared) '.
						^ environment undeclared declare: aSymbol from: classPool].
					(self confirm: (aString,' is still used in code of class ', classOrMeta name,
						'.\Is it okay to move it to Undeclared?') withCRs)
						ifTrue:[^Undeclared declare: aSymbol from: classPool]
						ifFalse:[^self]]]].
	classPool removeKey: aSymbol.
	classPool isEmpty ifTrue: [classPool := nil].
! !
!Class methodsFor: 'class name' stamp: 'cwp 6/22/2012 15:49' prior: 54796206!
rename: aString 
	"The new name of the receiver is the argument, aString."

	| oldName newName |
	(newName := aString asSymbol) = (oldName := self name)
		ifTrue: [^ self].
	(self environment includesKey: newName)
		ifTrue: [^ self error: newName , ' already exists'].
	(environment undeclared includesKey: newName)
		ifTrue: [self inform: 'There are references to, ' , aString printString , '
from Undeclared. Check them after this change.'].
	name := newName.
	self environment renameClass: self from: oldName! !
!ClassBuilder methodsFor: 'class definition' stamp: 'cwp 6/22/2012 01:05' prior: 39054430!
name: className inEnvironment: env subclassOf: newSuper type: type instanceVariableNames: instVarString classVariableNames: classVarString poolDictionaries: poolString category: category unsafe: unsafe
	"Define a new class in the given environment.
	If unsafe is true do not run any validation checks.
	This facility is provided to implement important system changes."
	| oldClass instVars classVars copyOfOldClass newClass |
 
	environ := env.
	instVars := Scanner new scanFieldNames: instVarString.
	classVars := (Scanner new scanFieldNames: classVarString) collect: [:x | x asSymbol].

	"Validate the proposed name"
	unsafe ifFalse:[(self validateClassName: className) ifFalse:[^nil]].
	oldClass := env at: className ifAbsent:[nil].
	oldClass isBehavior 
		ifFalse: [oldClass := nil]  "Already checked in #validateClassName:"
		ifTrue: [
			copyOfOldClass := oldClass copy.
			copyOfOldClass superclass addSubclass: copyOfOldClass].
	
	
	[ | newCategory needNew force organization oldCategory |
	unsafe ifFalse:[
		"Run validation checks so we know that we have a good chance for recompilation"
		(self validateSuperclass: newSuper forSubclass: oldClass) ifFalse:[^nil].
		(self validateInstvars: instVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateClassvars: classVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateSubclassFormat: type from: oldClass forSuper: newSuper extra: instVars size) ifFalse:[^nil]].

	"See if we need a new subclass"
	needNew := self needsSubclassOf: newSuper type: type instanceVariables: instVars from: oldClass.
	needNew == nil ifTrue:[^nil]. "some error"

	(needNew and:[unsafe not]) ifTrue:[
		"Make sure we don't redefine any dangerous classes"
		(self tooDangerousClasses includes: oldClass name) ifTrue:[
			self error: oldClass name, ' cannot be changed'.
		].
		"Check if the receiver should not be redefined"
		(oldClass ~~ nil and:[oldClass shouldNotBeRedefined]) ifTrue:[
			self notify: oldClass name asText allBold, 
						' should not be redefined. \Proceed to store over it.' withCRs]].

	needNew ifTrue:[
		"Create the new class"
		newClass := self 
			newSubclassOf: newSuper 
			type: type 
			instanceVariables: instVars
			from: oldClass.
		newClass == nil ifTrue:[^nil]. "Some error"
		newClass setName: className.
		newClass environment: environ.
	] ifFalse:[
		"Reuse the old class"
		newClass := oldClass.
	].

	"Install the class variables and pool dictionaries... "
	force := (newClass declare: classVarString) | (newClass sharing: poolString).

	"... classify ..."
	newCategory := category asSymbol.
	organization := environ ifNotNil:[environ organization].
	oldClass isNil ifFalse: [oldCategory := (organization categoryOfElement: oldClass name) asSymbol].
	organization classify: newClass name under: newCategory suppressIfDefault: true.
	
	"... recompile ..."
	newClass := self recompile: force from: oldClass to: newClass mutate: false.

	"... export if not yet done ..."
	(environ at: newClass name ifAbsent:[nil]) == newClass ifFalse:[
		[environ at: newClass name put: newClass]
			on: AttemptToWriteReadOnlyGlobal do:[:ex| ex resume: true].
		environ flushClassNameCache.
	].


	newClass doneCompiling.
	"... notify interested clients ..."
	oldClass isNil ifTrue: [
		SystemChangeNotifier uniqueInstance classAdded: newClass inCategory: newCategory.
		^ newClass].
	newCategory ~= oldCategory 
		ifTrue: [SystemChangeNotifier uniqueInstance class: newClass recategorizedFrom: oldCategory to: category]
		ifFalse: [SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: copyOfOldClass to: newClass.].
] ensure: 
		[copyOfOldClass ifNotNil: [copyOfOldClass superclass removeSubclass: copyOfOldClass].
		Behavior flushObsoleteSubclasses.
		].
	^newClass! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 22:57' prior: 18572019!
superclass: newSuper
	subclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat 
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class."
	| env |
	env := EnvironmentRequest signal ifNil: [newSuper environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: newSuper
		type: newSuper typeOfClass
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:01' prior: 50629912!
superclass: aClass
	variableByteSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable byte-sized nonpointer variables."
	| oldClassOrNil actualType env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a byte subclass of a class with named fields'].
	(aClass isVariable and: [aClass isWords])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with word fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with pointer fields'].
	oldClassOrNil := aClass environment at: t ifAbsent:[nil].
	actualType := (oldClassOrNil notNil
				   and: [oldClassOrNil typeOfClass == #compiledMethod])
					ifTrue: [#compiledMethod]
					ifFalse: [#bytes].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: actualType
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:03' prior: 18573442!
superclass: aClass
	variableSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable pointer variables."
	
	| env |
	aClass isBits ifTrue: 
		[^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #variable
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18574098!
superclass: aClass
	variableWordSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable word-sized nonpointer variables."
	| env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a word subclass of a class with named fields'].
	(aClass isVariable and: [aClass isBytes])
		ifTrue: [^self error: 'cannot make a word subclass of a class with byte fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a word subclass of a class with pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #words
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18575028!
superclass: aClass
	weakSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class (the receiver) in which the subclass is to 
	have weak indexable pointer variables."
	| env |
	aClass isBits 
		ifTrue: [^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #weak
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !

"Kernel"!
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 59135029!
ambiguousSelector: aString inRange: anInterval
	| correctedSelector userSelection offset intervalWithOffset |
	
	self interactive ifFalse: [
		"In non interactive mode, compile with backward comapatibility: $- is part of literal argument"
		Transcript cr; store: encoder classEncoding; nextPutAll:#'>>';store: encoder selector; show: ' would send ' , token , '-'.
		^super ambiguousSelector: aString inRange: anInterval].
	
	"handle the text selection"
	userSelection := cue requestor selectionInterval.
	intervalWithOffset := anInterval first + requestorOffset to: anInterval last + requestorOffset.
	cue requestor selectFrom: intervalWithOffset first to: intervalWithOffset last.
	cue requestor select.

	"Build the menu with alternatives"
	correctedSelector := AmbiguousSelector 
			signalName: aString
			inRange: intervalWithOffset.
	correctedSelector ifNil: [^self fail].

	"Execute the selected action"
	offset := self substituteWord: correctedSelector wordInterval: intervalWithOffset offset: 0.
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + offset.
	token := (correctedSelector readStream upTo: Character space) asSymbol! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 38558136!
collectTemporaryDeclarationsFrom: methodNode
	| tempsMarks str |
	tempsMarks := OrderedCollection new.
	str := cue requestor text asString.
	methodNode accept: (ParseNodeEnumerator
		ofBlock: [ :aNode | 
			| mark |
			(aNode class canUnderstand: #tempsMark) 
				ifTrue: 
					[mark := aNode tempsMark.
					(mark notNil and: [ mark between: 1 and: str size ] and: [ (str at: mark) = $| ])
						ifTrue: [ tempsMarks addLast: aNode ]]]).
	(tempsMark notNil and: [ tempsMark between: 1 and: str size ] and: [ (str at: tempsMark) = $| ])
						ifTrue: [ tempsMarks addLast: self ].
	^ tempsMarks sorted: [ :a :b | a tempsMark > b tempsMark ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 52096606!
correctSelector: proposedKeyword wordIntervals: spots exprInterval: expInt ifAbort: abortAction
	"Correct the proposedKeyword to some selector symbol, correcting the original text if such action is indicated.  abortAction is invoked if the proposedKeyword couldn't be converted into a valid selector.  Spots is an ordered collection of intervals within the test stream of the for each of the keyword parts."

	| correctSelector userSelection |
	"If we can't ask the user, assume that the keyword will be defined later"
	self interactive ifFalse: [^proposedKeyword asSymbol].

	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spots first first to: spots last last.
	cue requestor select.

	correctSelector := UnknownSelector name: proposedKeyword.
	correctSelector ifNil: [^abortAction value].

	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.

	self substituteSelector: correctSelector keywords wordIntervals: spots.
	^(proposedKeyword last ~~ $:
	   and: [correctSelector last == $:])
		ifTrue: [abortAction value]
		ifFalse: [correctSelector]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 33907242!
correctVariable: proposedVariable interval: spot 
	"Correct the proposedVariable to a known variable, or declare it as a new
	variable if such action is requested.  We support declaring lowercase
	variables as temps or inst-vars, and uppercase variables as Globals or 
	ClassVars, depending on whether the context is nil (class=UndefinedObject).
	Spot is the interval within the test stream of the variable.
	rr 3/4/2004 10:26 : adds the option to define a new class. "

	"Check if this is an i-var, that has been corrected already (ugly)"

	"Display the pop-up menu"

	| binding userSelection action |
	(encoder classEncoding instVarNames includes: proposedVariable) ifTrue: 
		[^InstanceVariableNode new 
			name: proposedVariable
			index: (encoder classEncoding allInstVarNames indexOf: proposedVariable)].

	"If we can't ask the user for correction, make it undeclared"
	self interactive ifFalse: [^encoder undeclared: proposedVariable].

	"First check to see if the requestor knows anything about the variable"
	(binding := cue requestor bindingOf: proposedVariable)
		ifNotNil: [^encoder global: binding name: proposedVariable].
	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spot first to: spot last.
	cue requestor select.

	"Build the menu with alternatives"
	action := UndeclaredVariable 
				signalFor: self
				name: proposedVariable
				inRange: spot.
	action ifNil: [^self fail].

	"Execute the selected action"
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.
	^action value! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:19' prior: 34172921!
declareUndeclaredTemps: methodNode
	"Declare any undeclared temps, declaring them at the smallest enclosing scope."

	| undeclared userSelection blocksToVars |
	(undeclared := encoder undeclaredTemps) isEmpty ifTrue:
		[^self].
	userSelection := cue requestor selectionInterval.
	blocksToVars := IdentityDictionary new.
	undeclared do:
		[:var|
		(blocksToVars
			at: (var tag == #method
					ifTrue: [methodNode block]
					ifFalse: [methodNode accept: (VariableScopeFinder new ofVariable: var)])
			ifAbsentPut: [SortedCollection new]) add: var name].
	(blocksToVars removeKey: methodNode block ifAbsent: []) ifNotNil:
		[:rootVars|
		rootVars do: [:varName| self pasteTempAtMethodLevel: varName]].
	(blocksToVars keys sorted: [:a :b| a tempsMark < b tempsMark]) do:
		[:block| | decl |
		decl := (blocksToVars at: block) reduce: [:a :b| a, ' ', b].
		block temporaries isEmpty
			ifTrue:
				[self substituteWord: ' | ', decl, ' |'
					wordInterval: (block tempsMark + 1 to: block tempsMark)
					offset: requestorOffset]
			ifFalse:
				[self substituteWord: decl, ' '
					wordInterval: (block tempsMark to: block tempsMark - 1)
					offset: requestorOffset]].
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + requestorOffset.
	ReparseAfterSourceEditing signal! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 11:45' prior: 37183770!
defineClass: className 
	"prompts the user to define a new class,  
	asks for it's category, and lets the users edit further  
	the definition"
	| sym cat def d2 |
	sym := className asSymbol.
	cat := UIManager default request: 'Enter class category : ' initialAnswer: self encoder classEncoding theNonMetaClass category.
	cat
		ifEmpty: [cat := 'Unknown'].
	def := 'Object subclass: #' , sym , '
		instanceVariableNames: '''' 
		classVariableNames: ''''
		poolDictionaries: ''''
		category: ''' , cat , ''''.
	d2 := UIManager default request: 'Edit class definition : ' initialAnswer: def.
	d2
		ifEmpty: [d2 := def].
	Compiler evaluate: d2.
	^ encoder
		global: (cue environment bindingOf: sym)
		name: sym! !
!Parser methodsFor: 'primitives' stamp: 'cwp 12/27/2012 11:46' prior: 37184567!
externalFunctionDeclaration
	"Parse the function declaration for a call to an external library."
	| descriptorClass callType modifier retType externalName args argType module fn |
	descriptorClass := cue environment
		valueOf: #ExternalFunction 
		ifAbsent: [^ false].
	callType := descriptorClass callingConventionFor: here.
	callType == nil ifTrue:[^false].
	[modifier := descriptorClass callingConventionModifierFor: token.
	 modifier notNil] whileTrue:
		[self advance.
		 callType := callType bitOr: modifier].
	"Parse return type"
	self advance.
	retType := self externalType: descriptorClass.
	retType == nil ifTrue:[^self expected:'return type'].
	"Parse function name or index"
	externalName := here.
	(self match: #string) 
		ifTrue:[externalName := externalName asSymbol]
		ifFalse:[(self match:#number) ifFalse:[^self expected:'function name or index']].
	(self matchToken: #'(') ifFalse:[^self expected:'argument list'].
	args := WriteStream on: Array new.
	[here == #')'] whileFalse:[
		argType := self externalType: descriptorClass.
		argType == nil ifTrue:[^self expected:'argument'].
		argType isVoid & argType isPointerType not ifFalse:[args nextPut: argType].
	].
	(self matchToken: #')') ifFalse:[^self expected:')'].
	(self matchToken: 'module:') ifTrue:[
		module := here.
		(self match: #string) ifFalse:[^self expected: 'String'].
		module := module asSymbol].
	Smalltalk at: #ExternalLibraryFunction ifPresent:[:xfn|
		fn := xfn name: externalName 
				module: module 
				callType: callType
				returnType: retType
				argumentTypes: args contents.
		self allocateLiteral: fn.
	].
	(self matchToken: 'error:')
		ifTrue:
			[| errorCodeVariable |
			 errorCodeVariable := here.
			(hereType == #string
			 or: [hereType == #word]) ifFalse:[^self expected: 'error code (a variable or string)'].
			 self advance.
			 self addPragma: (Pragma keyword: #primitive:error: arguments: (Array with: 120 with: errorCodeVariable)).
			 fn ifNotNil: [fn setErrorCodeName: errorCodeVariable]]
		ifFalse:
			[self addPragma: (Pragma keyword: #primitive: arguments: #(120))].
	^true
! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:19' prior: 58306169!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:22' prior: 58137223!
notify: string at: location
	cue requestor isNil
		ifTrue: [(encoder == self or: [encoder isNil]) ifTrue: [^ self fail  "failure setting up syntax error"].
				SyntaxErrorNotification
					inClass: encoder classEncoding
					category: cue category
					withCode: 
						(source contents asText
							copyReplaceFrom: location
							to: location - 1
							with: ((string , ' ->') asText allBold 
								addAttribute: TextColor red; yourself))
					doitFlag: doitFlag
					errorMessage: string
					location: location]
		ifFalse: [cue requestor
					notify: string , ' ->'
					at: location
					in: source].
	^self fail! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:17' prior: 34177108!
pasteTempAtMethodLevel: name
	| insertion delta theTextString characterBeforeMark |

	theTextString := cue requestor text string.
	characterBeforeMark := theTextString at: tempsMark-1 ifAbsent: [$ ].
	(theTextString at: tempsMark) = $| ifTrue: [
  		"Paste it before the second vertical bar"
		insertion := name, ' '.
		characterBeforeMark isSeparator ifFalse: [insertion := ' ', insertion].
		delta := 0.
	] ifFalse: [
		"No bars - insert some with CR, tab"
		insertion := '| ' , name , ' |',String cr.
		delta := 2.	"the bar and CR"
		characterBeforeMark = Character tab ifTrue: [
			insertion := insertion , String tab.
			delta := delta + 1.	"the tab"
		].
	].
	tempsMark := tempsMark +
		(self substituteWord: insertion
			wordInterval: (tempsMark to: tempsMark-1)
			offset: 0) - delta! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:16' prior: 52095305!
queryUndefined
	| varStart varName | 
	varName := parseNode key.
	varStart := self endOfLastToken + requestorOffset - varName size + 1.
	cue requestor selectFrom: varStart to: varStart + varName size - 1; select.
	(UndefinedVariable name: varName) ifFalse: [^ self fail]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38599341!
removeEmptyTempDeclarationsFrom: methodNode

	| sourceCode madeChanges tempsMarkHolder |
	sourceCode := cue requestor text asString.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	madeChanges := false.
	tempsMarkHolder do: [ :currentBlock | | tempsMarkChar0 tempsMarkChar1 tempsMarkChar2 end start |
		tempsMarkChar0 := (sourceCode at: currentBlock tempsMark).
		tempsMarkChar1 := (sourceCode at: currentBlock tempsMark - 1).
		tempsMarkChar2 := (sourceCode at: currentBlock tempsMark - 2).
		tempsMarkChar0 = $| & tempsMarkChar1 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 1].
		tempsMarkChar0 = $| & tempsMarkChar1 = $  & tempsMarkChar2 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 2].
		
		start notNil & end notNil ifTrue: [
			| lineStart lineEnd |
			lineStart := 1 + (sourceCode 
				lastIndexOf: Character cr 
				startingAt: start - 1
				ifAbsent: [ 0 ]).
			lineEnd := sourceCode 
				indexOf: Character cr
				startingAt: end + 1
				ifAbsent: [ sourceCode size ].
			((sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: lineStart) >= start 
				and: [ (sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: end + 1) > lineEnd ]) ifTrue: [
					start := lineStart.
					end := lineEnd ].
			cue requestor correctFrom: start to: end with: ''.
			madeChanges := true.
			currentBlock tempsMark: nil ] ].
	madeChanges ifTrue: [ReparseAfterSourceEditing signal]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38561281!
removeUnusedTemporaryNamed: temp from: str lookingAt: currentBlock movingTempMarksOf: someBlocks

	| start end |
	end := currentBlock tempsMark - 1.
	["Beginning at right temp marker..."
	start := end - temp size + 1.
	end < temp size or: [ (str at: start) = $| ]
		or: [ temp = (str copyFrom: start to: end) 
			and: [ ((str at: start - 1) = $| | (str at: start - 1) isSeparator) 
				& ((str at: end + 1) = $| | (str at: end + 1) isSeparator) ] ]]
		whileFalse: [ 
			"Search left for the unused temp"
			end := cue requestor nextTokenFrom: end direction: -1 ].
	(end < temp size or: [ (str at: start) = $| ])
		ifFalse: 
			[(str at: start - 1) = $ 
				ifTrue: [ start := start - 1 ].
			cue requestor correctFrom: start to: end with: ''.
			someBlocks do: [ :aBlock | aBlock tempsMark: aBlock tempsMark - (end - start + 1)].
			^true ].
	^false! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 38562194!
removeUnusedTemps: methodNode
	"Scan for unused temp names, and prompt the user about the prospect of removing each one found"

	| madeChanges tempsMarkHolder unusedTempNames tempMarkHoldersToChange |
	madeChanges := false.
	tempMarkHoldersToChange := OrderedCollection new.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	unusedTempNames := encoder unusedTempNames select: 
		[ :temp | (encoder lookupVariable: temp ifAbsent: [ ]) isUndefTemp 
				and: [ self queriedUnusedTemporaries at: temp ifAbsentPut: [UnusedVariable name: temp] ]].
	tempsMarkHolder do: [ :currentBlock | 
		tempMarkHoldersToChange add: currentBlock.
		unusedTempNames do: 
			[ :temp |
			(self 
				removeUnusedTemporaryNamed: temp 
				from: cue requestor text asString 
				lookingAt: currentBlock
				movingTempMarksOf: tempMarkHoldersToChange) ifTrue: [ madeChanges := true ]]].
	madeChanges
		ifTrue: [ self removeEmptyTempDeclarationsFrom: methodNode.
			ReparseAfterSourceEditing signal ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 34179326!
substituteWord: correctWord wordInterval: spot offset: o
	"Substitute the correctSelector into the (presumed interactive) receiver.
	 Update requestorOffset based on the delta size and answer the updated offset."

	cue requestor correctFrom: spot first + o to: spot last + o with: correctWord.
	requestorOffset := requestorOffset + correctWord size - spot size.
	^o + correctWord size - spot size! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34179807!
temporaries
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'
! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34180638!
temporariesIn: methodSelector
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance in: methodSelector)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:11' prior: 53971863!
compiledMethodFor: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code
	 into a method, and answers it.  If receiver is not nil, then the text can
	 refer to instance variables of that receiver (the Inspector uses this).
	 If aContext is not nil, the text can refer to temporaries in that context
	 (the Debugger uses this). If aRequestor is not nil, then it will receive a 
	 notify:at: message before the attempt to evaluate is aborted."

	| methodNode method theClass |
	theClass := (aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class.
	self from: textOrStream class: theClass context: aContext notifying: aRequestor.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].
	method := self interactive ifTrue: [ 	methodNode generateWithTempNames ] 
		ifFalse: [methodNode generate].
		
	logFlag ifTrue:
		[SystemChangeNotifier uniqueInstance evaluated: sourceStream contents context: aContext].
	^method! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:33' prior: 34363593!
format: aStream noPattern: noPattern ifFail: failBlock
	^(self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]) preen! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 10:08' prior: 58306325!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Compiler methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:10' prior: 50779387!
notify: aString at: location
	"Refer to the comment in Object|notify:."

	^ cue requestor == nil
		ifTrue: [SyntaxErrorNotification
					inClass: cue getClass
					category: cue category
					withCode: 
						(sourceStream contents
							copyReplaceFrom: location
							to: location - 1
							with: aString)
					doitFlag: false
					errorMessage: aString
					location: location]
		ifFalse: [cue requestor
					notify: aString
					at: location
					in: sourceStream]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:34' prior: 50777201!
parse: textOrStream in: aClass notifying: req
	"Compile the argument, textOrStream, with respect to the class, aClass, and
	 answer the MethodNode that is the root of the resulting parse tree.  Notify the
	 argument, req, if an error occurs. The failBlock is defaulted to an empty block."

	self from: textOrStream class: aClass context: nil notifying: req.
	^self parser
		parse: sourceStream
		cue: cue
		noPattern: false
		ifFail: []! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:09' prior: 36332471!
parser

	parser ifNil: [parser := (cue getClass ifNil: [self class]) newParser].
	^parser! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:37' prior: 50780779!
translate: aStream noPattern: noPattern ifFail: failBlock
	^self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:37' prior: 19124095!
translate: aStream noPattern: noPattern ifFail: failBlock parser: parser
	| tree |
	tree := parser
			parse: aStream
			cue: cue 
			noPattern: noPattern
			ifFail: [^ failBlock value].
	^ tree! !
!Encoder methodsFor: 'results' stamp: 'cwp 12/27/2012 10:26' prior: 50999892!
associationForClass
	| assoc |
	assoc := self environment associationAt: cue getClass name ifAbsent: [nil].
	^assoc value == cue getClass
		ifTrue: [assoc]
		ifFalse: [Association new value: cue getClass]! !
!Encoder methodsFor: 'temps' stamp: 'cwp 12/27/2012 10:25' prior: 20148386!
bindTemp: name in: methodSelector
	"Declare a temporary; error not if a field or class variable."
	scopeTable at: name ifPresent:[:node|
		"When non-interactive raise the error only if its a duplicate"
		(node isTemp or:[requestor interactive])
			ifTrue:[^self notify:'Name is already defined']
			ifFalse:[Transcript 
				show: '(', name, ' is shadowed in "' , cue getClass printString , '>>' , methodSelector printString , '")']].
	^self reallyBind: name! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:25' prior: 20149084!
classEncoding
	"This is a hack so that the parser may findout what class it was parsing for when it wants to create a syntax error view."
	^ cue getClass! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:39' prior: 20138819!
encodeLiteral: object

	^self
		name: object
		key: (cue  literalScannedAs: object notifying: self)
		class: LiteralNode
		type: LdLitType
		set: litSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 20139010!
encodeSelector: aSelector

	^self
		name: aSelector
		key: aSelector
		class: SelectorNode
		type: SendType
		set: selectorSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 58545123!
environment
	"Answer the environment of the current compilation context,
	 be it in a class or global (e.g. a workspace)"
	^cue environment! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 11:41' prior: 50994497!
lookupInPools: varName ifFound: assocBlock

	^Symbol
		hasInterned: varName
		ifTrue:
			[:sym|
			(cue bindingOf: sym)
				ifNil: [^false]
				ifNotNil: [:assoc| assocBlock value: assoc]]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 51004306!
possibleNamesFor: proposedName
	| results |
	results := cue getClass 
		possibleVariablesFor: proposedName 
		continuedFrom: nil.
	^ proposedName correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 50995012!
possibleVariablesFor: proposedVariable

	| results |
	results := proposedVariable correctAgainstDictionary: scopeTable
								continuedFrom: nil.
	proposedVariable first canBeGlobalVarInitial ifTrue:
		[ results := cue getClass possibleVariablesFor: proposedVariable
						continuedFrom: results ].
	^ proposedVariable correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:42' prior: 51002830!
undeclared: name
	| sym |
	requestor interactive ifTrue:
		[requestor requestor == #error: ifTrue:
			[requestor error: 'Undeclared'].
		 ^self notify: 'Undeclared'].
	"Allow knowlegeable clients to squash the undeclared warning if they want (e.g.
	 Diffing pretty printers that are simply formatting text).  As this breaks
	 compilation it should only be used by clients that want to discard the result
	 of the compilation.  To squash the warning use e.g.
		[Compiler format: code in: class notifying: nil decorated: false]
			on: UndeclaredVariableWarning
			do: [:ex| ex resume: false]"
	sym := name asSymbol.
	^(UndeclaredVariableWarning new name: name selector: selector class: cue getClass) signal
		ifTrue:
			[| undeclared |
			undeclared := cue environment undeclared.
			undeclared at: sym put: nil.
			self global: (undeclared associationAt: sym) name: sym]
		ifFalse:
			[self global: (Association key: sym) name: sym]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:23' prior: 51006007!
warnAboutShadowed: name
	requestor addWarning: name,' is shadowed'.
	selector ifNotNil:
		[Transcript cr; show: cue getClass name,'>>', selector, '(', name,' is shadowed)']! !

"Compiler"!
!SmalltalkImage methodsFor: 'housekeeping' stamp: 'cwp 6/22/2012 15:56' prior: 58497062!
cleanOutUndeclared 
	globals undeclared removeUnreferencedKeys! !
!SmalltalkImage methodsFor: 'special objects' stamp: 'cwp 6/22/2012 09:01' prior: 40515090!
recreateSpecialObjectsArray
	"Smalltalk recreateSpecialObjectsArray"
	
	"To external package developers:
	**** DO NOT OVERRIDE THIS METHOD.  *****
	If you are writing a plugin and need additional special object(s) for your own use, 
	use addGCRoot() function and use own, separate special objects registry "
	
	"The Special Objects Array is an array of objects used by the Squeak virtual machine.
	 Its contents are critical and accesses to it by the VM are unchecked, so don't even
	 think of playing here unless you know what you are doing."
	| newArray |
	newArray := Array new: 56.
	"Nil false and true get used throughout the interpreter"
	newArray at: 1 put: nil.
	newArray at: 2 put: false.
	newArray at: 3 put: true.
	"This association holds the active process (a ProcessScheduler)"
	newArray at: 4 put: (self bindingOf: #Processor).
	"Numerous classes below used for type checking and instantiation"
	newArray at: 5 put: Bitmap.
	newArray at: 6 put: SmallInteger.
	newArray at: 7 put: ByteString.
	newArray at: 8 put: Array.
	newArray at: 9 put: Smalltalk.
	newArray at: 10 put: Float.
	newArray at: 11 put: MethodContext.
	newArray at: 12 put: BlockContext.
	newArray at: 13 put: Point.
	newArray at: 14 put: LargePositiveInteger.
	newArray at: 15 put: Display.
	newArray at: 16 put: Message.
	newArray at: 17 put: CompiledMethod.
	newArray at: 18 put: (self specialObjectsArray at: 18).
	"(low space Semaphore)"
	newArray at: 19 put: Semaphore.
	newArray at: 20 put: Character.
	newArray at: 21 put: #doesNotUnderstand:.
	newArray at: 22 put: #cannotReturn:.
	newArray at: 23 put: nil. "This is the process signalling low space."
	"An array of the 32 selectors that are compiled as special bytecodes,
	 paired alternately with the number of arguments each takes."
	newArray at: 24 put: #(	#+ 1 #- 1 #< 1 #> 1 #<= 1 #>= 1 #= 1 #~= 1
							#* 1 #/ 1 #\\ 1 #@ 1 #bitShift: 1 #// 1 #bitAnd: 1 #bitOr: 1
							#at: 1 #at:put: 2 #size 0 #next 0 #nextPut: 1 #atEnd 0 #== 1 #class 0
							#blockCopy: 1 #value 0 #value: 1 #do: 1 #new 0 #new: 1 #x 0 #y 0 ).
	"An array of the 255 Characters in ascii order.
	 Cog inlines table into machine code at: prim so do not regenerate it."
	newArray at: 25 put: (self specialObjectsArray at: 25).
	newArray at: 26 put: #mustBeBoolean.
	newArray at: 27 put: ByteArray.
	newArray at: 28 put: Process.
	"An array of up to 31 classes whose instances will have compact headers"
	newArray at: 29 put: self compactClassesArray.
	newArray at: 30 put: (self specialObjectsArray at: 30). "(delay Semaphore)"
	newArray at: 31 put: (self specialObjectsArray at: 31). "(user interrupt Semaphore)"
	"Entries 32 - 34 unreferenced. Previously these contained prototype instances to be copied for fast initialization"
	newArray at: 32 put: nil. "was (Float new: 2)"
	newArray at: 33 put: nil. "was (LargePositiveInteger new: 4)"
	newArray at: 34 put: nil. "was Point new"
	newArray at: 35 put: #cannotInterpret:.
	"Note: This must be fixed once we start using context prototypes (yeah, right)"
	"(MethodContext new: CompiledMethod fullFrameSize)."
	newArray at: 36 put: (self specialObjectsArray at: 36). "Is the prototype MethodContext (unused by the VM)"
	newArray at: 37 put: BlockClosure.
	"(BlockContext new: CompiledMethod fullFrameSize)."
	newArray at: 38 put: (self specialObjectsArray at: 38). "Is the prototype BlockContext (unused by the VM)"
	"array of objects referred to by external code"
	newArray at: 39 put: (self specialObjectsArray at: 39).	"preserve external semaphores"
	newArray at: 40 put: nil. "Reserved for Mutex in Cog VMs"
	newArray at: 41 put: nil. "Reserved for a LinkedList instance for overlapped calls in CogMT"
	"finalization Semaphore"
	newArray at: 42 put: ((self specialObjectsArray at: 42) ifNil: [Semaphore new]).
	newArray at: 43 put: LargeNegativeInteger.
	"External objects for callout.
	 Note: Written so that one can actually completely remove the FFI."
	newArray at: 44 put: (self at: #ExternalAddress ifAbsent: []).
	newArray at: 45 put: (self at: #ExternalStructure ifAbsent: []).
	newArray at: 46 put: (self at: #ExternalData ifAbsent: []).
	newArray at: 47 put: (self at: #ExternalFunction ifAbsent: []).
	newArray at: 48 put: (self at: #ExternalLibrary ifAbsent: []).
	newArray at: 49 put: #aboutToReturn:through:.
	newArray at: 50 put: #run:with:in:.
	"51 reserved for immutability message"
	"newArray at: 51 put: #attemptToAssign:withIndex:."
	newArray at: 52 put: #(nil "nil => generic error" #'bad receiver'
							#'bad argument' #'bad index'
							#'bad number of arguments'
							#'inappropriate operation'  #'unsupported operation'
							#'no modification' #'insufficient object memory'
							#'insufficient C memory' #'not found' #'bad method'
							#'internal error in named primitive machinery'
							#'object may move').
	"53 to 55 are for Alien"
	newArray at: 53 put: (self at: #Alien ifAbsent: []).
	newArray at: 54 put: #invokeCallback:stack:registers:jmpbuf:.
	newArray at: 55 put: (self at: #UnsafeAlien ifAbsent: []).

	"Weak reference finalization"
	newArray at: 56 put: (self at: #WeakFinalizationList ifAbsent: []).

	"Now replace the interpreter's reference in one atomic operation"
	self specialObjectsArray becomeForward: newArray
	! !
!SmalltalkImage methodsFor: 'shrinking' stamp: 'cwp 6/22/2012 15:57' prior: 37288071!
unloadAllKnownPackages
	"Unload all packages we know how to unload and reload"

	"Prepare unloading"
	Smalltalk zapMVCprojects.
	Flaps disableGlobalFlaps: false.
	StandardScriptingSystem removeUnreferencedPlayers.
	Project removeAllButCurrent.
	#('Morphic-UserObjects' 'EToy-UserObjects' 'Morphic-Imported' )
		do: [:each | SystemOrganization removeSystemCategory: each].
	Smalltalk at: #ServiceRegistry ifPresent:[:aClass|
		SystemChangeNotifier uniqueInstance
			noMoreNotificationsFor: aClass.
	].
	World removeAllMorphs.

	"Go unloading"
	#(	'ReleaseBuilder' 'ScriptLoader'
		'311Deprecated' '39Deprecated'
		'Universes' 'SMLoader' 'SMBase' 'Installer-Core'
		'VersionNumberTests' 'VersionNumber'
		'Services-Base' 'PreferenceBrowser' 'Nebraska'
		'ToolBuilder-MVC' 'ST80'
		'CollectionsTests' 'GraphicsTests' 'KernelTests'  'MorphicTests' 
		'MultilingualTests' 'NetworkTests' 'ToolsTests' 'TraitsTests'
		'SystemChangeNotification-Tests' 'FlexibleVocabularies' 
		'EToys' 'Protocols' 'XML-Parser' 'Tests' 'SUnitGUI'
		'Help-Squeak' 'HelpSystem' 'SystemReporter'
	) do: [:pkgName| 
			(MCPackage named: pkgName) unload.
			MCMcmUpdater disableUpdatesOfPackage: pkgName.
			].
	"Traits use custom unload"
	Smalltalk at: #Trait ifPresent:[:aClass| aClass unloadTraits].

	"Post-unload cleanup"
	MCWorkingCopy flushObsoletePackageInfos.
	SystemOrganization removeSystemCategory: 'UserObjects'.
	Presenter defaultPresenterClass: nil.
	World dumpPresenter.
	ScheduledControllers := nil.
	Preferences removePreference: #allowEtoyUserCustomEvents.
	SystemOrganization removeEmptyCategories.
	ChangeSet removeChangeSetsNamedSuchThat:[:cs | (cs == ChangeSet current) not].
	globals undeclared removeUnreferencedKeys.
	StandardScriptingSystem initialize.
	MCFileBasedRepository flushAllCaches.
	MCDefinition clearInstances.
	Behavior flushObsoleteSubclasses.
	ChangeSet current clear.
	ChangeSet current name: 'Unnamed1'.
	Smalltalk flushClassNameCache.
	Smalltalk at: #Browser ifPresent:[:br| br initialize].
	DebuggerMethodMap voidMapCache.
	DataStream initialize.
	AppRegistry removeObsolete.
	FileServices removeObsolete.
	Preferences removeObsolete.
	TheWorldMenu removeObsolete.
	Smalltalk garbageCollect.
	Symbol compactSymbolTable.
	TheWorldMainDockingBar updateInstances.
	MorphicProject defaultFill: (Color gray: 0.9).
	World color: (Color gray: 0.9).
! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34' prior: 40472775!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:27' prior: 40496770!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!SystemDictionary methodsFor: 'dictionary access' stamp: 'cwp 6/22/2012 15:58' prior: 30574136!
at: aKey put: anObject 
	"Override from Dictionary to check Undeclared and fix up
	references to undeclared variables."
	| index element |
	(self includesKey: aKey) ifFalse: 
		[self declare: aKey from: (self at: #Undeclared).
		self flushClassNameCache].
	super at: aKey put: anObject.
	^ anObject! !

"System"!

CodeHolder subclass: #Browser
	instanceVariableNames: 'environment systemOrganizer classOrganizer metaClassOrganizer editSelection metaClassIndicated selectedSystemCategory selectedClassName selectedMessageName selectedMessageCategoryName'
	classVariableNames: 'ListClassesHierarchically RecentClasses'
	poolDictionaries: ''
	category: 'Tools-Browser'!
!Browser commentStamp: 'cwp 12/27/2012 11:09' prior: 36419432!
I represent a query path into the class descriptions, the software of the system.!
!Browser methodsFor: 'accessing' stamp: 'cwp 6/24/2012 23:20'!
selectEnvironment: anEnvironment 
	environment := anEnvironment.
	systemOrganizer := environment organization! !
!Browser methodsFor: 'system category list' stamp: 'cwp 6/24/2012 23:06' prior: 36467357!

From noreply at buildbot.pypy.org  Wed Jun 26 19:14:29 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 19:14:29 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: removed all benchmarks to see
 the frame working
Message-ID: <20130626171429.4A95D1C021A@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r482:244ae8552317
Date: 2013-06-26 17:03 +0000
http://bitbucket.org/pypy/lang-smalltalk/changeset/244ae8552317/

Log:	removed all benchmarks to see the frame working

diff too long, truncating to 2000 out of 80189 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -166,7 +166,7 @@
             callback(self)
 
     def run(self, args):
-        print 'Calling %s ...' % self.name
+        print 'Calling %s (./%s) ...' % (self.name, " ".join([self.path] + args))
         pipe = subprocess.Popen(
             ["%s" % self.path] + args,
             stdout=subprocess.PIPE
diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
--- a/images/Squeak4.5-12568.changes
+++ b/images/Squeak4.5-12568.changes
@@ -36,4 +36,4 @@
 Workspace allInstances do: [:w | w topView delete].
 ReleaseBuilderFor4dot4 prepareNewBuild.
 Smalltalk snapshot: true andQuit: true.
-!

----End fileIn of a stream----!

----SNAPSHOT----{31 March 2013 . 3:27:34 pm} Squeak4.5-12327.image priorSource: 7430688!
!Installer methodsFor: 'squeakmap' stamp: 'fbs 1/28/2013 19:25' prior: 57597950!
packageAndVersionFrom: pkg
	| p |
	p := ReadStream on: pkg .
	^{(p upTo: $(). p upTo: $)} collect: [:s | s withBlanksTrimmed].! !

"Installer-Core"!
!Categorizer methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 16:58'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!ClassCategoryReader methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:21'!
scanFrom: aStream environment: anEnvironment
	"File in methods from the stream, aStream."
	| methodText |
	[methodText := aStream nextChunkText.
	 methodText size > 0] whileTrue:
		[class 
			compile: methodText 
			environment: anEnvironment
			classified: category
			withStamp: changeStamp 
			notifying: nil]! !
!ClassCommentReader methodsFor: 'as yet unclassified' stamp: 'cwp 6/20/2012 17:22'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!Metaclass methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:29'!
bindingOf: varName environment: anEnvironment 
	^ thisClass classBindingOf: varName environment: anEnvironment! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 20:03' prior: 22505876!
\\ aNumber 
	"Primitive. Take the receiver modulo the argument. The result is the
	remainder rounded towards negative infinity, of the receiver divided
	by the argument. Fail if the argument is 0. Fail if either the argument
	or the result is not a SmallInteger or a LargePositiveInteger less than
	2-to-the-30th (1073741824). Optional. See Object documentation whatIsAPrimitive."

	<primitive: 31>
	aNumber isInteger
		ifTrue:
			[| neg qr q r |
			neg := self negative == aNumber negative == false.
			qr := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: neg).
			q := qr first normalize.
			r := qr last normalize.
			^(q negative
				ifTrue: [r isZero not]
				ifFalse: [q isZero and: [neg]])
					ifTrue: [r + aNumber]
					ifFalse: [r]].
	^super \\ aNumber
	! !
!LargePositiveInteger methodsFor: 'converting' stamp: 'nice 1/27/2012 22:41' prior: 37616324!
asFloat
	"Answer a Float that best approximates the value of the receiver.
	This algorithm is optimized to process only the significant digits of a LargeInteger.
	And it does honour IEEE 754 round to nearest even mode in case of excess precision (see details below)."
	
	"How numbers are rounded in IEEE 754 default rounding mode:
	A shift is applied so that the highest 53 bits are placed before the floating point to form a mantissa.
	The trailing bits form the fraction part placed after the floating point.
	This fractional number must be rounded to the nearest integer.
	If fraction part is 2r0.1, exactly between two consecutive integers, there is a tie.
	The nearest even integer is chosen in this case.
	Examples (First 52bits of mantissa are omitted for brevity):
	2r0.00001 is rounded downward to 2r0
	2r1.00001 is rounded downward to 2r1
	2r0.1 is a tie and rounded to 2r0 (nearest even)
	2r1.1 is a tie and rounded to 2r10 (nearest even)
	2r0.10001 is rounded upward to 2r1
	2r1.10001 is rounded upward to 2r10
	Thus, if the next bit after floating point is 0, the mantissa is left unchanged.
	If next bit after floating point is 1, an odd mantissa is always rounded upper.
	An even mantissa is rounded upper only if the fraction part is not a tie."
	
	"Algorihm details:
	The floating point hardware can perform the rounding correctly with several excess bits as long as there is a single inexact operation.
	This can be obtained by splitting the mantissa plus excess bits in two part with less bits than Float precision.
	Note 1: the inexact flag in floating point hardware must not be trusted because in some cases the operations would be exact but would not take into account some bits that were truncated before the Floating point operations.
	Note 2: the floating point hardware is presumed configured in default rounding mode."
	
	| mantissa shift excess result n |

	"Check how many bits excess the maximum precision of a Float mantissa."
	excess := self highBitOfMagnitude - Float precision.
	excess > 7
		ifTrue:
			["Remove the excess bits but seven."
			mantissa := self bitShiftMagnitude: 7 - excess.
			shift := excess - 7.
			"An even mantissa with a single excess bit immediately following would be truncated.
			But this would not be correct if above shift has truncated some extra bits.
			Check this case, and round excess bits upper manually."
			((mantissa digitAt: 1) = 2r01000000 and: [self anyBitOfMagnitudeFrom: 1 to: shift])
				ifTrue: [mantissa := mantissa + 1]]
		ifFalse:
			[mantissa := self.
			shift := 0].

	"There will be a single inexact round off at last iteration"
	result := (mantissa digitAt: (n := mantissa digitLength)) asFloat.
	[(n := n - 1) > 0] whileTrue: [
		result := 256.0 * result + (mantissa digitAt: n) asFloat].
	^result timesTwoPower: shift.! !
!LargePositiveInteger methodsFor: 'private' stamp: 'nice 12/30/2012 14:25'!
primitiveQuo: anInteger 
	"Primitive. Divide the receiver by the argument and return the result.
	Round the result down towards zero to make it a whole integer. Fail if
	the argument is 0. Fail if either the argument or the result is not a
	SmallInteger or a LargePositiveInteger less than 2-to-the-30th (1073741824). Optional. See
	Object documentation whatIsAPrimitive."

	<primitive: 33>
	^nil! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 14:34'!
rem: aNumber 
	"Remainder defined in terms of quo:. See super rem:.
	This is defined only to speed up case of very large integers."

	(self primitiveQuo: aNumber)
		ifNotNil: [:quo | ^self - (quo * aNumber)].
	 aNumber isInteger
		ifTrue:
			[| ng rem |
			ng := self negative == aNumber negative == false.
			rem := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: ng) at: 2.
			^ rem normalize].
	^super rem: aNumber! !
!LargeNegativeInteger methodsFor: 'converting' stamp: 'nice 1/1/2013 15:42' prior: 37616204!
asFloat
	^super asFloat negated! !
!UndefinedObject methodsFor: 'class hierarchy' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:32'!
bindingOf: varName environment: anEnvironment
	^superclass bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:30'!
classBindingOf: varName environment: anEnvironment
	^self bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:37'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor
	"Postprocesses a literal scanned by Scanner scanToken (esp. xLitQuote).
	If scannedLiteral is not an association, answer it.
	Else, if it is of the form:
		nil->#NameOfMetaclass
	answer nil->theMetaclass, if any has that name, else report an error.
	Else, if it is of the form:
		#NameOfGlobalVariable->anythiEng
	answer the global, class, or pool association with that nameE, if any, else
	add it to Undeclared a answer the new Association."

	| key value |
	(scannedLiteral isVariableBinding)
		ifFalse: [^ scannedLiteral].
	key := scannedLiteral key.
	value := scannedLiteral value.
	key ifNil: "###<metaclass soleInstance name>"
		[(self bindingOf: value environment: anEnvironment) ifNotNil:
			[:assoc|
			(assoc value isKindOf: Behavior) ifTrue: 
				[^ nil->assoc value class]].
			 requestor notify: 'No such metaclass'.
			 ^false].
	(key isSymbol) ifTrue: "##<global var name>"
		[(self bindingOf: key environment: anEnvironment) ifNotNil:
			[:assoc | ^assoc].
		^ anEnvironment undeclared: key].
	requestor notify: '## must be followed by a non-local variable name'.
	^false

"	Form literalScannedAs: 14 notifying: nil 14
	Form literalScannedAs: #OneBitForm notiEfying: nil  OneBitForm
	Form literalScannedAs: ##OneBitForm notifying: nil  OneBitForm->a Form
	Form literalScannedAs: ##Form notifying: nil   Form->Form
	Form literalScannedAs: ###Form notifying: nil   nilE->Form class
"! !
!Fraction methodsFor: 'converting' stamp: 'nice 11/21/2011 22:34' prior: 37619655!
asFloat
	"Answer a Float that closely approximates the value of the receiver.
	This implementation will answer the closest floating point number to the receiver.
	In case of a tie, it will use the IEEE 754 round to nearest even mode.
	In case of overflow, it will answer +/- Float infinity."

	| a b mantissa exponent hasTruncatedBits lostBit n ha hb hm |
	a := numerator abs.
	b := denominator.	"denominator is always positive"
	ha := a highBitOfMagnitude.
	hb := b highBitOfMagnitude.
	
	"Number of bits to keep in mantissa plus one to handle rounding."
	n := 1 + Float precision.

	"If both numerator and denominator are represented exactly in floating point number,
	then fastest thing to do is to use hardwired float division."
	(ha < n and: [hb < n]) ifTrue: [^numerator asFloat / denominator asFloat].

	"Shift the fraction by a power of two exponent so as to obtain a mantissa with n bits.
	First guess is rough, the mantissa might have n+1 bits."
	exponent := ha - hb - n.
	exponent >= 0
		ifTrue: [b := b bitShift: exponent]
		ifFalse: [a := a bitShift: exponent negated].
	mantissa := a quo: b.
	hasTruncatedBits := a > (mantissa * b).
	hm := mantissa highBit.
	
	"Check for gradual underflow, in which case the mantissa will loose bits.
	Keep at least one bit to let underflow preserve the sign of zero."
	lostBit := Float emin - (exponent + hm - 1).
	lostBit > 0 ifTrue: [n := n - lostBit max: 1].

	"Remove excess bits in the mantissa."
	hm > n
		ifTrue:
			[exponent := exponent + hm - n.
			hasTruncatedBits := hasTruncatedBits or: [mantissa anyBitOfMagnitudeFrom: 1 to: hm - n].
			mantissa := mantissa bitShift: n - hm].

	"Check if mantissa must be rounded upward.
	The case of tie (mantissa odd & hasTruncatedBits not)
	will be handled by Integer>>asFloat."
	(hasTruncatedBits and: [mantissa odd])
		ifTrue: [mantissa := mantissa + 1].

	^ (self positive
			ifTrue: [mantissa asFloat]
			ifFalse: [mantissa asFloat negated])
		timesTwoPower: exponent! !
!Float methodsFor: 'arithmetic' stamp: 'nice 12/20/2012 23:16' prior: 20878776!
negated
	"Answer a Number that is the negation of the receiver.
	Implementation note: this version cares of negativeZero."

	^-1.0 * self! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:21'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor
	^ self 
		compile: text 
		environment: anEnvironment 
		classified: category 
		withStamp: changeStamp 
		notifying: requestor 
		logSource: self acceptsLoggingOfCompilation! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 12/27/2012 13:17'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor logSource: logSource
	| methodAndNode context methodNode |
	context := CompilationCue
		source: text
		class: self
		environment: anEnvironment
		category: category
		requestor: requestor.
	methodNode := self newCompiler compile: context ifFail: [^ nil].
	methodAndNode := CompiledMethodWithNode 
		generateMethodFromNode: methodNode 
		trailer: self defaultMethodTrailer.

	logSource ifTrue: [
		self logMethodSource: text forMethodWithNode: methodAndNode 
			inCategory: category withStamp: changeStamp notifying: requestor.
	].
	self addAndClassifySelector: methodAndNode selector withMethod: methodAndNode 
		method inProtocol: category notifying: requestor.
	self instanceSide noteCompilationOf: methodAndNode selector meta: self isClassSide.
	^ methodAndNode selector! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:47'!
bindingOf: varName environment: anEnvironment
	"Answer the binding of some variable resolved in the scope of the receiver"
	| aSymbol binding |
	aSymbol := varName asSymbol.

	"First look in classVar dictionary."
	binding := self classPool bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Next look in shared pools."
	self sharedPools do:[:pool | 
		binding := pool bindingOf: aSymbol.
		binding ifNotNil:[^binding].
	].

	"Next look in declared environment."
	binding := anEnvironment bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Finally look higher up the superclass chain and fail at the end."
	superclass == nil
		ifTrue: [^ nil]
		ifFalse: [^ superclass bindingOf: aSymbol].

! !

"Kernel"!

ParseNode subclass: #Encoder
	instanceVariableNames: 'scopeTable nTemps supered requestor class selector literalStream selectorSet litIndSet litSet sourceRanges globalSourceRanges addedSelectorAndMethodClassLiterals optimizedSelectors cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Encoder commentStamp: 'cwp 12/26/2012 23:29' prior: 36323851!
I encode names and literals into tree nodes with byte codes for the compiler. Byte codes for literals are not assigned until the tree-sizing pass of the compiler, because only then is it known which literals are actually needed. I also keep track of sourceCode ranges during parsing and code generation so I can provide an inverse map for the debugger.!

Scanner subclass: #Parser
	instanceVariableNames: 'here hereType hereMark hereEnd prevMark prevEnd encoder requestor parseNode failBlock requestorOffset tempsMark doitFlag properties category queriedUnusedTemporaries cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Parser commentStamp: 'cwp 12/26/2012 23:34' prior: 38557958!
I parse Smalltalk syntax and create a MethodNode that is the root of the parse tree. I look one token ahead.!

Object subclass: #CompilationCue
	instanceVariableNames: 'source context receiver class environment category requestor'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

Object subclass: #Compiler
	instanceVariableNames: 'sourceStream requestor class category context parser cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Compiler commentStamp: 'cwp 12/26/2012 23:17' prior: 59257505!
The compiler accepts Smalltalk source code and compiles it with respect to a given class. The user of the compiler supplies a context so that temporary variables are accessible during compilation. If there is an error, a requestor (usually a kind of StringHolderController) is sent the message notify:at:in: so that the error message can be displayed. If there is no error, then the result of compilation is a MethodNode, which is the root of a parse tree whose nodes are kinds of ParseNodes. The parse tree can be sent messages to (1) generate code for a CompiledMethod (this is done for compiling methods or evaluating expressions); (2) pretty-print the code (for formatting); or (3) produce a map from object code back to source code (used by debugger program-counter selection). See also Parser, Encoder, ParseNode.!
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:34'!
init: aCue notifying: anObject
	"The use of the variable requestor is a bit confusing here. This is
	*not* the original requestor, which is available through the cue.
	It's the Parser instance that is using the encoder."

	self setCue: aCue.
	requestor := anObject.
	nTemps := 0.
	supered := false.
	self initScopeAndLiteralTables.
	cue getClass variablesAndOffsetsDo:
		[:variable "<String|CFieldDefinition>" :offset "<Integer|nil>" |
		offset isNil
			ifTrue: [scopeTable at: variable name put: (FieldNode new fieldDefinition: variable)]
			ifFalse: [scopeTable
						at: variable
						put: (offset >= 0
								ifTrue: [InstanceVariableNode new
											name: variable index: offset]
								ifFalse: [MaybeContextInstanceVariableNode new
											name: variable index: offset negated])]].
	cue context ~~ nil ifTrue:
		[| homeNode |
		 homeNode := self bindTemp: self doItInContextName.
		 "0th temp = aContext passed as arg"
		 cue context tempNames withIndexDo:
			[:variable :index|
			scopeTable
				at: variable
				put: (MessageAsTempNode new
						receiver: homeNode
						selector: #namedTempAt:
						arguments: (Array with: (self encodeLiteral: index))
						precedence: 3
						from: self)]].
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/26/2012 23:30'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy instance variables for methods that
	don't use cue yet"
	class := cue getClass.! !
!Dictionary methodsFor: '*Compiler' stamp: 'cwp 6/22/2012 09:17'!
bindingOf: varName ifAbsent: aBlock

	^self associationAt: varName ifAbsent: aBlock! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:37'!
init: sourceStream cue: aCue failBlock: aBlock

	self setCue: aCue.
	failBlock := aBlock.
	requestorOffset := 0.
	super scan: sourceStream.
	prevMark := hereMark := mark.
	self advance
! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/26/2012 23:41'!
parse: sourceStream cue: aCue noPattern: noPattern ifFail: aBlock 
	"Answer a MethodNode for the argument, sourceStream, that is the root of
	 a parse tree. Parsing is done with respect to the CompilationCue to 
	 resolve variables. Errors in parsing are reported to the cue's requestor; 
	 otherwise aBlock is evaluated. The argument noPattern is a Boolean that is
	 true if the the sourceStream does not contain a method header (i.e., for DoIts)."

	| methNode repeatNeeded myStream s p subSelection |
	myStream := sourceStream.
	[repeatNeeded := false.
	 p := myStream position.
	 s := myStream upToEnd.
	 myStream position: p.
	 subSelection := aCue requestor notNil and: [aCue requestor selectionInterval = (p + 1 to: p + s size)].
	 self encoder init: aCue notifying: self.
	 self init: myStream cue: aCue failBlock: [^ aBlock value].
	 doitFlag := noPattern.
	 failBlock:= aBlock.
	 [methNode := self method: noPattern context: cue context] 
		on: ReparseAfterSourceEditing 
		do:	[ :ex |
			repeatNeeded := true.
			myStream := subSelection
							ifTrue:
								[ReadStream
									on: cue requestor text string
									from: cue requestor selectionInterval first
									to: cue requestor selectionInterval last]
							ifFalse:
								[ReadStream on: cue requestor text string]].
	 repeatNeeded] whileTrue:
		[encoder := self encoder class new].
	methNode sourceText: s.
	^methNode
! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:35'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy variables for methods that don't use cue yet."
	requestor := cue requestor.
	category := cue category.! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
class: aClass
	^ self 
		context: nil
		class: aClass
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
context: aContext class: aClass requestor: anObject
	^ self
		source: nil
		context: aContext
		receiver: nil
		class: aClass
		environment: (aClass ifNotNil: [aClass environment])
		category: nil
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aTextOrStream class: aClass environment: anEnvironment category: aString requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		receiver: nil
		class: aClass
		environment: anEnvironment
		category: aString
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
source: aTextOrStream context: aContext class: aClass category: aString requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		receiver: (aContext ifNotNil: [aContext receiver])
		class: aClass 
		environment: (aClass ifNotNil: [aClass environment])
		category: aString 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream context: aContext class: aClass requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		class: aClass 
		category: nil 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:55'!
source: aTextOrStream context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	^ self basicNew
		initializeWithSource: aTextOrStream 
		context: aContext 
		receiver: recObject 
		class: aClass 
		environment: anEnvironment 
		category: aString 
		requestor: reqObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aString environment: anEnvironment
	^ self 
		source: aString
		context: nil
		receiver: nil
		class: UndefinedObject
		environment: anEnvironment
		category: nil
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		class: nil
		requestor: anObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/20/2012 09:39'!
bindingOf: aSymbol
	^ class bindingOf: aSymbol environment: environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
category
	^ category! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 12/26/2012 23:19'!
context
	^ context! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
environment
	^ environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
getClass
	^ class! !
!CompilationCue methodsFor: 'initialization' stamp: 'cwp 12/26/2012 23:16'!
initializeWithSource: aTextOrString context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	self initialize.
	source := aTextOrString isStream ifTrue: [aTextOrString contents] ifFalse: [aTextOrString].
	context := aContext.
	receiver := recObject.
	class := aClass.
	environment := anEnvironment.
	category := aString.
	requestor := reqObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: anObject notifying: anEncoder
	^ class literalScannedAs: anObject environment: environment notifying: anEncoder! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
receiver
	^ receiver! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
requestor
	^ requestor! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
source
	^ source! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:44'!
sourceStream
	^ source readStream! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 6/20/2012 17:25'!
evaluate: aString environment: anEnvironment
	^ self 
		evaluate: aString 
		environment: anEnvironment 
		logged: false! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 12/27/2012 12:36'!
evaluate: aString environment: anEnvironment logged: aBoolean
	| cue |
	cue := CompilationCue
		source: aString
		environment: anEnvironment.
		
	^ self new
		evaluate: aString
		cue: cue
		ifFail: [^ nil]
		logged: aBoolean! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 13:18'!
compile: aCue ifFail: failBlock 
	"Answer a MethodNode. If the MethodNode can not be created, notify 
	the requestor in the contxt. If the requestor is nil, evaluate failBlock 
	instead. The MethodNode is the root  of a parse tree. It can be told 
	to generate a CompiledMethod to be installed in the method dictionary 
	of the class specified by the context."
	
	self setCue: aCue.
	self source: cue source.
	^self
		translate: sourceStream
		noPattern: false
		ifFail: failBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:06'!
evaluate: textOrStream cue: aCue ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	a method. Finally, the compiled method is invoked from here via 	withArgs:executeMethod:, hence the system no longer creates Doit method
	litter on errors."

	| methodNode method value toLog itsSelection itsSelectionString |
	self setCue: aCue.
	self source: textOrStream.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].

	method := self interactive
				ifTrue: [methodNode generateWithTempNames]
				ifFalse: [methodNode generate].

	value := cue receiver
				withArgs: (cue context ifNil: [#()] ifNotNil: [{cue context}])
				executeMethod: method.

	logFlag ifTrue:
		[toLog := ((cue requestor respondsTo: #selection)  
			and:[(itsSelection := cue requestor selection) notNil
			and:[(itsSelectionString := itsSelection asString) isEmptyOrNil not]])
				ifTrue:[itsSelectionString]
				ifFalse:[sourceStream contents].
		SystemChangeNotifier uniqueInstance evaluated: toLog context: cue context].
	^ value
! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:20'!
setCue: aCue
	cue := aCue.
	
	"Set legacy instance variables for methods that don't use cue yet."
	requestor := cue requestor.
	class := cue getClass.
	category := cue category.
	context := cue context.! !
!Compiler methodsFor: 'private' stamp: 'cwp 6/19/2012 21:58'!
source: textOrStream
	sourceStream := (textOrStream isKindOf: PositionableStream)
		ifTrue: [ textOrStream ]
		ifFalse: [ ReadStream on: textOrStream asString ]! !

"Compiler"!
!SmartRefStream class methodsFor: 'i/o' stamp: 'cwp 6/20/2012 17:42'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!SmartRefStream methodsFor: 'read write' stamp: 'cwp 6/20/2012 17:41'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!ImageSegment methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:23'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!PseudoClass methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| aString newTranslations assoc currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[Locale
		currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:26'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| newTranslations currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[| aString assoc |
	Locale currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!ObjectScanner methodsFor: 'scanning' stamp: 'cwp 6/20/2012 17:39'!
scanFrom: aByteStream environment: anEnvironment
	"This should probably be reimplemented using an environment
	for compilation. For now, don't change anything"
	^ self scanFrom: aByteStream! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 09:16'!
bindingOf: varName ifAbsent: aBlock
	"SystemDictionary includes Symbols only"
	^super bindingOf: varName asSymbol ifAbsent: aBlock! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 15:48'!
undeclared
	^ self at: #Undeclared! !

"System"!
!ExceptionTests methodsFor: 'testing-outer' stamp: 'fbs 1/1/2013 22:14' prior: 40840955!
expectedFailures
	^ #().! !

"Tests"!

ReleaseBuilder subclass: #ReleaseBuilderFor4dot5
	instanceVariableNames: ''
	classVariableNames: ''
	poolDictionaries: ''
	category: 'ReleaseBuilder'!
!ReleaseBuilderFor4dot5 commentStamp: 'fbs 1/1/2013 20:25' prior: 0!
The release builder for Squeak 4.5!
!ReleaseBuilder class methodsFor: 'scripts' stamp: 'fbs 12/31/2012 20:43'!
transferCurrentPackagesAsUser: username password: password
	"Copy the packages currently loaded in the image from the trunk repository to my releaseRepository."
	| trunkRep releaseRep |
	trunkRep := self trunkRepository.
	releaseRep := self releaseRepository
		user: username;
		password: password;
		yourself.
	MCWorkingCopy allManagers do:
		[ : eachWorkingCopy | eachWorkingCopy ancestors do:
			[ : eachVersionInfo | (releaseRep includesVersionNamed: eachVersionInfo versionName) ifFalse:
				[ (trunkRep versionWithInfo: eachVersionInfo)
					ifNil: [ Warning signal: eachVersionInfo name , ' not found in ', trunkRep ]
					ifNotNilDo: [ : ver | releaseRep storeVersion: ver ] ] ] ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
openWelcomeWorkspaces
	TheWorldMainDockingBar instance
		showWelcomeText: #squeakUserInterface
		label: 'Squeak User Interface'
		in: (40 @ 40 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #workingWithSqueak
		label: 'Working With Squeak'
		in: (80 @ 80 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #licenseInformation
		label: 'License Information'
		in: (120 @ 120 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeFutureDirections
		label: 'Future Directions'
		in: (160 @ 160 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeToSqueak
		label: 'Welcome to Squeak 4.5'
		in: (200 @ 200 extent: 500 @ 300)! !
!ReleaseBuilderFor4dot5 class methodsFor: 'scripts' stamp: 'fbs 1/1/2013 20:22'!
prepareNewBuild
	super prepareNewBuild.
	MCMockPackageInfo initialize.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:24'!
releaseRepository
	"At release time, change 'trunk' to 'squeak45'."
	^ MCHttpRepository
		location: 'http://source.squeak.org/trunk'
		user: 'squeak'
		password: 'squeak'! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:22'!
setDisplayExtent: extent
"Uncomment next line when the primitives become available in the Squeak VM."
"	DisplayScreen hostWindowSize: extent."
	Display extent = extent ifFalse: [ Warning signal: 'Display extent not set to ', extent ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
setPreferences
	Preferences 
		installBrightWindowColors ;
		setPreference: #scrollBarsWithoutMenuButton toValue: true ;
		setPreference: #swapMouseButtons toValue: true ;
		setPreference: #annotationPanes toValue: true ;
		setPreference: #showSplitterHandles toValue: false ;
		setPreference: #showBoundsInHalo toValue: true ;
		setPreference: #alternateHandlesLook toValue: false ;
		setPreference: #roundedMenuCorners toValue: false ;
		setPreference: #roundedWindowCorners toValue: false.
	PluggableButtonMorph roundedButtonCorners: false.
	FillInTheBlankMorph roundedDialogCorners: false.
	Workspace shouldStyle: false.
	NetNameResolver enableIPv6: true.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
switchToNewRepository
	| old44Repository |
	MCMcmUpdater defaultUpdateURL: self releaseRepository description.
	old44Repository := MCRepositoryGroup default repositories
				detect: [:each | each description includesSubString: 'squeak44'] ifNone: [nil].
	old44Repository
		ifNotNil: [MCRepositoryGroup default removeRepository: old44Repository].
	MCRepositoryGroup default addRepository: self releaseRepository! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
versionString
	^ 'Squeak4.5'.! !

ReleaseBuilder class removeSelector: #transferCurrentPackages!

"ReleaseBuilder"!
!Environment class methodsFor: 'as yet unclassified' stamp: 'cwp 1/1/2013 18:52' prior: 40834114!
initialize
	self install! !

"Environments"!
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:59' prior: 52081878!
initPattern: aString notifying: req return: aBlock

	| result |
	self
		init: (ReadStream on: aString asString)
		cue: (CompilationCue source: aString requestor: req)
		failBlock: [^nil].
	encoder := self.
	result := aBlock value: (self pattern: false inContext: nil).
	encoder := failBlock := nil.  "break cycles"
	^result! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:01' prior: 34175471!
parse: sourceStream class: class category: aCategory noPattern: noPattern context: aContext notifying: req ifFail: aBlock 
	| c |
	c := CompilationCue
			source: sourceStream
			context: aContext
			class: class
			category: aCategory
			requestor: req.
	^ self 
		parse: sourceStream 
		cue: c 
		noPattern: noPattern 
		ifFail: aBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:11' prior: 34183963!
evaluate: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	 a method. If aContext is not nil, the text can refer to temporaries in that
	 context (the Debugger uses this). If aRequestor is not nil, then it will receive
	 a notify:at: message before the attempt to evaluate is aborted. Finally, the 
	 compiled method is invoked from here via withArgs:executeMethod:, hence
	 the system no longer creates Doit method litter on errors."
	
	| theClass |
	theClass := ((aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class).
	self setCue: (CompilationCue
		source: textOrStream
		context: aContext
		receiver: receiver
		class: theClass
		environment: theClass environment
		category: nil
		requestor: aRequestor).
	^ self evaluate: textOrStream cue: cue ifFail: failBlock logged: logFlag! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:17' prior: 34185488!
from: textOrStream class: aClass classified: aCategory context: aContext notifying: req
	self source: textOrStream.
	self setCue: 
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			category: aCategory
			requestor: req)! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:55' prior: 50781309!
from: textOrStream class: aClass context: aContext notifying: req
	self source: textOrStream.
	self setCue:
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			requestor: req)
! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/27/2012 09:41' prior: 50996506!
init: aClass context: aContext notifying: anObject
	| c |
	c := CompilationCue
		context: aContext 
		class: aClass 
		requestor: nil.
	self init: c notifying: anObject! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:58' prior: 39061698!
temps: tempVars literals: lits class: cl 
	"Initialize this encoder for decompilation."

	self setCue: (CompilationCue class: cl).
	supered := false.
	nTemps := tempVars size.
	tempVars do: [:node | scopeTable at: node name put: node].
	literalStream := WriteStream on: (Array new: lits size).
	literalStream nextPutAll: lits.
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32.! !

"Compiler"!
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:48' prior: 36026010!
addClassVarName: aString 
	"Add the argument, aString, as a class variable of the receiver.
	Signal an error if the first character of aString is not capitalized,
	or if it is already a variable named in the class."
	| symbol oldState |
	oldState := self copy.
	aString first canBeGlobalVarInitial
		ifFalse: [^self error: aString, ' class variable name should be capitalized; proceed to include anyway.'].
	symbol := aString asSymbol.
	self withAllSubclasses do: 
		[:subclass | 
		(self canFindWithoutEnvironment: symbol) ifTrue: [
			(DuplicateVariableError new)
				superclass: superclass; "fake!!!!!!"
				variable: aString;
				signal: aString, ' is already defined']].
	classPool == nil ifTrue: [classPool := Dictionary new].
	(classPool includesKey: symbol) ifFalse: 
		["Pick up any refs in Undeclared"
		classPool declare: symbol from: environment undeclared.
		SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: oldState to: self]! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:48' prior: 54782024!
bindingOf: varName
	^ self bindingOf: varName environment: self environment! !
!Class methodsFor: 'organization' stamp: 'cwp 6/25/2012 18:25' prior: 54785804!
category
	"Answer the system organization category for the receiver. First check whether the
	category name stored in the ivar is still correct and only if this fails look it up
	(latter is much more expensive)"

	category ifNotNil: [ :symbol |
		((self environment organization listAtCategoryNamed: symbol) includes: self name)
			ifTrue: [ ^symbol ] ].
	category := self environment organization categoryOfElement: self name.
	^category! !
!Class methodsFor: 'initialize-release' stamp: 'cwp 6/22/2012 15:49' prior: 36027730!
declare: varString 
	"Declare class variables common to all instances. Answer whether 
	recompilation is advisable."

	| newVars conflicts |
	
	newVars := 
		(Scanner new scanFieldNames: varString)
			collect: [:x | x asSymbol].
	newVars do:
		[:var | var first canBeGlobalVarInitial
			ifFalse: [self error: var, ' class variable name should be capitalized; proceed to include anyway.']].
	conflicts := false.
	classPool == nil 
		ifFalse: [(classPool keys reject: [:x | newVars includes: x]) do: 
					[:var | self removeClassVarName: var]].
	(newVars reject: [:var | self classPool includesKey: var])
		do: [:var | "adding"
			"check if new vars defined elsewhere"
			(self canFindWithoutEnvironment: var) ifTrue: [
					(DuplicateVariableError new)
						superclass: superclass; "fake!!!!!!"
						variable: var;
						signal: var, ' is already defined'.
					conflicts := true]].
	newVars size > 0
		ifTrue: 
			[classPool := self classPool.
			"in case it was nil"
			newVars do: [:var | classPool declare: var from: environment undeclared]].
	^conflicts! !
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:49' prior: 54802475!
removeClassVarName: aString 
	"Remove the class variable whose name is the argument, aString, from 
	the names defined in the receiver, a class. Create an error notification if 
	aString is not a class variable or if it is still being used in the code of 
	the class."

	| aSymbol |
	aSymbol := aString asSymbol.
	(classPool includesKey: aSymbol)
		ifFalse: [^self error: aString, ' is not a class variable'].
	self withAllSubclasses do:[:subclass |
		(Array with: subclass with: subclass class) do:[:classOrMeta |
			(classOrMeta whichSelectorsReferTo: (classPool associationAt: aSymbol))
				isEmpty ifFalse: [
					InMidstOfFileinNotification signal ifTrue: [
						Transcript cr; show: self name, ' (' , aString , ' is Undeclared) '.
						^ environment undeclared declare: aSymbol from: classPool].
					(self confirm: (aString,' is still used in code of class ', classOrMeta name,
						'.\Is it okay to move it to Undeclared?') withCRs)
						ifTrue:[^Undeclared declare: aSymbol from: classPool]
						ifFalse:[^self]]]].
	classPool removeKey: aSymbol.
	classPool isEmpty ifTrue: [classPool := nil].
! !
!Class methodsFor: 'class name' stamp: 'cwp 6/22/2012 15:49' prior: 54796206!
rename: aString 
	"The new name of the receiver is the argument, aString."

	| oldName newName |
	(newName := aString asSymbol) = (oldName := self name)
		ifTrue: [^ self].
	(self environment includesKey: newName)
		ifTrue: [^ self error: newName , ' already exists'].
	(environment undeclared includesKey: newName)
		ifTrue: [self inform: 'There are references to, ' , aString printString , '
from Undeclared. Check them after this change.'].
	name := newName.
	self environment renameClass: self from: oldName! !
!ClassBuilder methodsFor: 'class definition' stamp: 'cwp 6/22/2012 01:05' prior: 39054430!
name: className inEnvironment: env subclassOf: newSuper type: type instanceVariableNames: instVarString classVariableNames: classVarString poolDictionaries: poolString category: category unsafe: unsafe
	"Define a new class in the given environment.
	If unsafe is true do not run any validation checks.
	This facility is provided to implement important system changes."
	| oldClass instVars classVars copyOfOldClass newClass |
 
	environ := env.
	instVars := Scanner new scanFieldNames: instVarString.
	classVars := (Scanner new scanFieldNames: classVarString) collect: [:x | x asSymbol].

	"Validate the proposed name"
	unsafe ifFalse:[(self validateClassName: className) ifFalse:[^nil]].
	oldClass := env at: className ifAbsent:[nil].
	oldClass isBehavior 
		ifFalse: [oldClass := nil]  "Already checked in #validateClassName:"
		ifTrue: [
			copyOfOldClass := oldClass copy.
			copyOfOldClass superclass addSubclass: copyOfOldClass].
	
	
	[ | newCategory needNew force organization oldCategory |
	unsafe ifFalse:[
		"Run validation checks so we know that we have a good chance for recompilation"
		(self validateSuperclass: newSuper forSubclass: oldClass) ifFalse:[^nil].
		(self validateInstvars: instVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateClassvars: classVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateSubclassFormat: type from: oldClass forSuper: newSuper extra: instVars size) ifFalse:[^nil]].

	"See if we need a new subclass"
	needNew := self needsSubclassOf: newSuper type: type instanceVariables: instVars from: oldClass.
	needNew == nil ifTrue:[^nil]. "some error"

	(needNew and:[unsafe not]) ifTrue:[
		"Make sure we don't redefine any dangerous classes"
		(self tooDangerousClasses includes: oldClass name) ifTrue:[
			self error: oldClass name, ' cannot be changed'.
		].
		"Check if the receiver should not be redefined"
		(oldClass ~~ nil and:[oldClass shouldNotBeRedefined]) ifTrue:[
			self notify: oldClass name asText allBold, 
						' should not be redefined. \Proceed to store over it.' withCRs]].

	needNew ifTrue:[
		"Create the new class"
		newClass := self 
			newSubclassOf: newSuper 
			type: type 
			instanceVariables: instVars
			from: oldClass.
		newClass == nil ifTrue:[^nil]. "Some error"
		newClass setName: className.
		newClass environment: environ.
	] ifFalse:[
		"Reuse the old class"
		newClass := oldClass.
	].

	"Install the class variables and pool dictionaries... "
	force := (newClass declare: classVarString) | (newClass sharing: poolString).

	"... classify ..."
	newCategory := category asSymbol.
	organization := environ ifNotNil:[environ organization].
	oldClass isNil ifFalse: [oldCategory := (organization categoryOfElement: oldClass name) asSymbol].
	organization classify: newClass name under: newCategory suppressIfDefault: true.
	
	"... recompile ..."
	newClass := self recompile: force from: oldClass to: newClass mutate: false.

	"... export if not yet done ..."
	(environ at: newClass name ifAbsent:[nil]) == newClass ifFalse:[
		[environ at: newClass name put: newClass]
			on: AttemptToWriteReadOnlyGlobal do:[:ex| ex resume: true].
		environ flushClassNameCache.
	].


	newClass doneCompiling.
	"... notify interested clients ..."
	oldClass isNil ifTrue: [
		SystemChangeNotifier uniqueInstance classAdded: newClass inCategory: newCategory.
		^ newClass].
	newCategory ~= oldCategory 
		ifTrue: [SystemChangeNotifier uniqueInstance class: newClass recategorizedFrom: oldCategory to: category]
		ifFalse: [SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: copyOfOldClass to: newClass.].
] ensure: 
		[copyOfOldClass ifNotNil: [copyOfOldClass superclass removeSubclass: copyOfOldClass].
		Behavior flushObsoleteSubclasses.
		].
	^newClass! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 22:57' prior: 18572019!
superclass: newSuper
	subclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat 
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class."
	| env |
	env := EnvironmentRequest signal ifNil: [newSuper environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: newSuper
		type: newSuper typeOfClass
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:01' prior: 50629912!
superclass: aClass
	variableByteSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable byte-sized nonpointer variables."
	| oldClassOrNil actualType env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a byte subclass of a class with named fields'].
	(aClass isVariable and: [aClass isWords])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with word fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with pointer fields'].
	oldClassOrNil := aClass environment at: t ifAbsent:[nil].
	actualType := (oldClassOrNil notNil
				   and: [oldClassOrNil typeOfClass == #compiledMethod])
					ifTrue: [#compiledMethod]
					ifFalse: [#bytes].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: actualType
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:03' prior: 18573442!
superclass: aClass
	variableSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable pointer variables."
	
	| env |
	aClass isBits ifTrue: 
		[^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #variable
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18574098!
superclass: aClass
	variableWordSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable word-sized nonpointer variables."
	| env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a word subclass of a class with named fields'].
	(aClass isVariable and: [aClass isBytes])
		ifTrue: [^self error: 'cannot make a word subclass of a class with byte fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a word subclass of a class with pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #words
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18575028!
superclass: aClass
	weakSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class (the receiver) in which the subclass is to 
	have weak indexable pointer variables."
	| env |
	aClass isBits 
		ifTrue: [^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #weak
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !

"Kernel"!
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 59135029!
ambiguousSelector: aString inRange: anInterval
	| correctedSelector userSelection offset intervalWithOffset |
	
	self interactive ifFalse: [
		"In non interactive mode, compile with backward comapatibility: $- is part of literal argument"
		Transcript cr; store: encoder classEncoding; nextPutAll:#'>>';store: encoder selector; show: ' would send ' , token , '-'.
		^super ambiguousSelector: aString inRange: anInterval].
	
	"handle the text selection"
	userSelection := cue requestor selectionInterval.
	intervalWithOffset := anInterval first + requestorOffset to: anInterval last + requestorOffset.
	cue requestor selectFrom: intervalWithOffset first to: intervalWithOffset last.
	cue requestor select.

	"Build the menu with alternatives"
	correctedSelector := AmbiguousSelector 
			signalName: aString
			inRange: intervalWithOffset.
	correctedSelector ifNil: [^self fail].

	"Execute the selected action"
	offset := self substituteWord: correctedSelector wordInterval: intervalWithOffset offset: 0.
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + offset.
	token := (correctedSelector readStream upTo: Character space) asSymbol! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 38558136!
collectTemporaryDeclarationsFrom: methodNode
	| tempsMarks str |
	tempsMarks := OrderedCollection new.
	str := cue requestor text asString.
	methodNode accept: (ParseNodeEnumerator
		ofBlock: [ :aNode | 
			| mark |
			(aNode class canUnderstand: #tempsMark) 
				ifTrue: 
					[mark := aNode tempsMark.
					(mark notNil and: [ mark between: 1 and: str size ] and: [ (str at: mark) = $| ])
						ifTrue: [ tempsMarks addLast: aNode ]]]).
	(tempsMark notNil and: [ tempsMark between: 1 and: str size ] and: [ (str at: tempsMark) = $| ])
						ifTrue: [ tempsMarks addLast: self ].
	^ tempsMarks sorted: [ :a :b | a tempsMark > b tempsMark ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 52096606!
correctSelector: proposedKeyword wordIntervals: spots exprInterval: expInt ifAbort: abortAction
	"Correct the proposedKeyword to some selector symbol, correcting the original text if such action is indicated.  abortAction is invoked if the proposedKeyword couldn't be converted into a valid selector.  Spots is an ordered collection of intervals within the test stream of the for each of the keyword parts."

	| correctSelector userSelection |
	"If we can't ask the user, assume that the keyword will be defined later"
	self interactive ifFalse: [^proposedKeyword asSymbol].

	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spots first first to: spots last last.
	cue requestor select.

	correctSelector := UnknownSelector name: proposedKeyword.
	correctSelector ifNil: [^abortAction value].

	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.

	self substituteSelector: correctSelector keywords wordIntervals: spots.
	^(proposedKeyword last ~~ $:
	   and: [correctSelector last == $:])
		ifTrue: [abortAction value]
		ifFalse: [correctSelector]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 33907242!
correctVariable: proposedVariable interval: spot 
	"Correct the proposedVariable to a known variable, or declare it as a new
	variable if such action is requested.  We support declaring lowercase
	variables as temps or inst-vars, and uppercase variables as Globals or 
	ClassVars, depending on whether the context is nil (class=UndefinedObject).
	Spot is the interval within the test stream of the variable.
	rr 3/4/2004 10:26 : adds the option to define a new class. "

	"Check if this is an i-var, that has been corrected already (ugly)"

	"Display the pop-up menu"

	| binding userSelection action |
	(encoder classEncoding instVarNames includes: proposedVariable) ifTrue: 
		[^InstanceVariableNode new 
			name: proposedVariable
			index: (encoder classEncoding allInstVarNames indexOf: proposedVariable)].

	"If we can't ask the user for correction, make it undeclared"
	self interactive ifFalse: [^encoder undeclared: proposedVariable].

	"First check to see if the requestor knows anything about the variable"
	(binding := cue requestor bindingOf: proposedVariable)
		ifNotNil: [^encoder global: binding name: proposedVariable].
	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spot first to: spot last.
	cue requestor select.

	"Build the menu with alternatives"
	action := UndeclaredVariable 
				signalFor: self
				name: proposedVariable
				inRange: spot.
	action ifNil: [^self fail].

	"Execute the selected action"
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.
	^action value! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:19' prior: 34172921!
declareUndeclaredTemps: methodNode
	"Declare any undeclared temps, declaring them at the smallest enclosing scope."

	| undeclared userSelection blocksToVars |
	(undeclared := encoder undeclaredTemps) isEmpty ifTrue:
		[^self].
	userSelection := cue requestor selectionInterval.
	blocksToVars := IdentityDictionary new.
	undeclared do:
		[:var|
		(blocksToVars
			at: (var tag == #method
					ifTrue: [methodNode block]
					ifFalse: [methodNode accept: (VariableScopeFinder new ofVariable: var)])
			ifAbsentPut: [SortedCollection new]) add: var name].
	(blocksToVars removeKey: methodNode block ifAbsent: []) ifNotNil:
		[:rootVars|
		rootVars do: [:varName| self pasteTempAtMethodLevel: varName]].
	(blocksToVars keys sorted: [:a :b| a tempsMark < b tempsMark]) do:
		[:block| | decl |
		decl := (blocksToVars at: block) reduce: [:a :b| a, ' ', b].
		block temporaries isEmpty
			ifTrue:
				[self substituteWord: ' | ', decl, ' |'
					wordInterval: (block tempsMark + 1 to: block tempsMark)
					offset: requestorOffset]
			ifFalse:
				[self substituteWord: decl, ' '
					wordInterval: (block tempsMark to: block tempsMark - 1)
					offset: requestorOffset]].
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + requestorOffset.
	ReparseAfterSourceEditing signal! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 11:45' prior: 37183770!
defineClass: className 
	"prompts the user to define a new class,  
	asks for it's category, and lets the users edit further  
	the definition"
	| sym cat def d2 |
	sym := className asSymbol.
	cat := UIManager default request: 'Enter class category : ' initialAnswer: self encoder classEncoding theNonMetaClass category.
	cat
		ifEmpty: [cat := 'Unknown'].
	def := 'Object subclass: #' , sym , '
		instanceVariableNames: '''' 
		classVariableNames: ''''
		poolDictionaries: ''''
		category: ''' , cat , ''''.
	d2 := UIManager default request: 'Edit class definition : ' initialAnswer: def.
	d2
		ifEmpty: [d2 := def].
	Compiler evaluate: d2.
	^ encoder
		global: (cue environment bindingOf: sym)
		name: sym! !
!Parser methodsFor: 'primitives' stamp: 'cwp 12/27/2012 11:46' prior: 37184567!
externalFunctionDeclaration
	"Parse the function declaration for a call to an external library."
	| descriptorClass callType modifier retType externalName args argType module fn |
	descriptorClass := cue environment
		valueOf: #ExternalFunction 
		ifAbsent: [^ false].
	callType := descriptorClass callingConventionFor: here.
	callType == nil ifTrue:[^false].
	[modifier := descriptorClass callingConventionModifierFor: token.
	 modifier notNil] whileTrue:
		[self advance.
		 callType := callType bitOr: modifier].
	"Parse return type"
	self advance.
	retType := self externalType: descriptorClass.
	retType == nil ifTrue:[^self expected:'return type'].
	"Parse function name or index"
	externalName := here.
	(self match: #string) 
		ifTrue:[externalName := externalName asSymbol]
		ifFalse:[(self match:#number) ifFalse:[^self expected:'function name or index']].
	(self matchToken: #'(') ifFalse:[^self expected:'argument list'].
	args := WriteStream on: Array new.
	[here == #')'] whileFalse:[
		argType := self externalType: descriptorClass.
		argType == nil ifTrue:[^self expected:'argument'].
		argType isVoid & argType isPointerType not ifFalse:[args nextPut: argType].
	].
	(self matchToken: #')') ifFalse:[^self expected:')'].
	(self matchToken: 'module:') ifTrue:[
		module := here.
		(self match: #string) ifFalse:[^self expected: 'String'].
		module := module asSymbol].
	Smalltalk at: #ExternalLibraryFunction ifPresent:[:xfn|
		fn := xfn name: externalName 
				module: module 
				callType: callType
				returnType: retType
				argumentTypes: args contents.
		self allocateLiteral: fn.
	].
	(self matchToken: 'error:')
		ifTrue:
			[| errorCodeVariable |
			 errorCodeVariable := here.
			(hereType == #string
			 or: [hereType == #word]) ifFalse:[^self expected: 'error code (a variable or string)'].
			 self advance.
			 self addPragma: (Pragma keyword: #primitive:error: arguments: (Array with: 120 with: errorCodeVariable)).
			 fn ifNotNil: [fn setErrorCodeName: errorCodeVariable]]
		ifFalse:
			[self addPragma: (Pragma keyword: #primitive: arguments: #(120))].
	^true
! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:19' prior: 58306169!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:22' prior: 58137223!
notify: string at: location
	cue requestor isNil
		ifTrue: [(encoder == self or: [encoder isNil]) ifTrue: [^ self fail  "failure setting up syntax error"].
				SyntaxErrorNotification
					inClass: encoder classEncoding
					category: cue category
					withCode: 
						(source contents asText
							copyReplaceFrom: location
							to: location - 1
							with: ((string , ' ->') asText allBold 
								addAttribute: TextColor red; yourself))
					doitFlag: doitFlag
					errorMessage: string
					location: location]
		ifFalse: [cue requestor
					notify: string , ' ->'
					at: location
					in: source].
	^self fail! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:17' prior: 34177108!
pasteTempAtMethodLevel: name
	| insertion delta theTextString characterBeforeMark |

	theTextString := cue requestor text string.
	characterBeforeMark := theTextString at: tempsMark-1 ifAbsent: [$ ].
	(theTextString at: tempsMark) = $| ifTrue: [
  		"Paste it before the second vertical bar"
		insertion := name, ' '.
		characterBeforeMark isSeparator ifFalse: [insertion := ' ', insertion].
		delta := 0.
	] ifFalse: [
		"No bars - insert some with CR, tab"
		insertion := '| ' , name , ' |',String cr.
		delta := 2.	"the bar and CR"
		characterBeforeMark = Character tab ifTrue: [
			insertion := insertion , String tab.
			delta := delta + 1.	"the tab"
		].
	].
	tempsMark := tempsMark +
		(self substituteWord: insertion
			wordInterval: (tempsMark to: tempsMark-1)
			offset: 0) - delta! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:16' prior: 52095305!
queryUndefined
	| varStart varName | 
	varName := parseNode key.
	varStart := self endOfLastToken + requestorOffset - varName size + 1.
	cue requestor selectFrom: varStart to: varStart + varName size - 1; select.
	(UndefinedVariable name: varName) ifFalse: [^ self fail]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38599341!
removeEmptyTempDeclarationsFrom: methodNode

	| sourceCode madeChanges tempsMarkHolder |
	sourceCode := cue requestor text asString.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	madeChanges := false.
	tempsMarkHolder do: [ :currentBlock | | tempsMarkChar0 tempsMarkChar1 tempsMarkChar2 end start |
		tempsMarkChar0 := (sourceCode at: currentBlock tempsMark).
		tempsMarkChar1 := (sourceCode at: currentBlock tempsMark - 1).
		tempsMarkChar2 := (sourceCode at: currentBlock tempsMark - 2).
		tempsMarkChar0 = $| & tempsMarkChar1 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 1].
		tempsMarkChar0 = $| & tempsMarkChar1 = $  & tempsMarkChar2 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 2].
		
		start notNil & end notNil ifTrue: [
			| lineStart lineEnd |
			lineStart := 1 + (sourceCode 
				lastIndexOf: Character cr 
				startingAt: start - 1
				ifAbsent: [ 0 ]).
			lineEnd := sourceCode 
				indexOf: Character cr
				startingAt: end + 1
				ifAbsent: [ sourceCode size ].
			((sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: lineStart) >= start 
				and: [ (sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: end + 1) > lineEnd ]) ifTrue: [
					start := lineStart.
					end := lineEnd ].
			cue requestor correctFrom: start to: end with: ''.
			madeChanges := true.
			currentBlock tempsMark: nil ] ].
	madeChanges ifTrue: [ReparseAfterSourceEditing signal]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38561281!
removeUnusedTemporaryNamed: temp from: str lookingAt: currentBlock movingTempMarksOf: someBlocks

	| start end |
	end := currentBlock tempsMark - 1.
	["Beginning at right temp marker..."
	start := end - temp size + 1.
	end < temp size or: [ (str at: start) = $| ]
		or: [ temp = (str copyFrom: start to: end) 
			and: [ ((str at: start - 1) = $| | (str at: start - 1) isSeparator) 
				& ((str at: end + 1) = $| | (str at: end + 1) isSeparator) ] ]]
		whileFalse: [ 
			"Search left for the unused temp"
			end := cue requestor nextTokenFrom: end direction: -1 ].
	(end < temp size or: [ (str at: start) = $| ])
		ifFalse: 
			[(str at: start - 1) = $ 
				ifTrue: [ start := start - 1 ].
			cue requestor correctFrom: start to: end with: ''.
			someBlocks do: [ :aBlock | aBlock tempsMark: aBlock tempsMark - (end - start + 1)].
			^true ].
	^false! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 38562194!
removeUnusedTemps: methodNode
	"Scan for unused temp names, and prompt the user about the prospect of removing each one found"

	| madeChanges tempsMarkHolder unusedTempNames tempMarkHoldersToChange |
	madeChanges := false.
	tempMarkHoldersToChange := OrderedCollection new.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	unusedTempNames := encoder unusedTempNames select: 
		[ :temp | (encoder lookupVariable: temp ifAbsent: [ ]) isUndefTemp 
				and: [ self queriedUnusedTemporaries at: temp ifAbsentPut: [UnusedVariable name: temp] ]].
	tempsMarkHolder do: [ :currentBlock | 
		tempMarkHoldersToChange add: currentBlock.
		unusedTempNames do: 
			[ :temp |
			(self 
				removeUnusedTemporaryNamed: temp 
				from: cue requestor text asString 
				lookingAt: currentBlock
				movingTempMarksOf: tempMarkHoldersToChange) ifTrue: [ madeChanges := true ]]].
	madeChanges
		ifTrue: [ self removeEmptyTempDeclarationsFrom: methodNode.
			ReparseAfterSourceEditing signal ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 34179326!
substituteWord: correctWord wordInterval: spot offset: o
	"Substitute the correctSelector into the (presumed interactive) receiver.
	 Update requestorOffset based on the delta size and answer the updated offset."

	cue requestor correctFrom: spot first + o to: spot last + o with: correctWord.
	requestorOffset := requestorOffset + correctWord size - spot size.
	^o + correctWord size - spot size! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34179807!
temporaries
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'
! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34180638!
temporariesIn: methodSelector
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance in: methodSelector)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:11' prior: 53971863!
compiledMethodFor: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code
	 into a method, and answers it.  If receiver is not nil, then the text can
	 refer to instance variables of that receiver (the Inspector uses this).
	 If aContext is not nil, the text can refer to temporaries in that context
	 (the Debugger uses this). If aRequestor is not nil, then it will receive a 
	 notify:at: message before the attempt to evaluate is aborted."

	| methodNode method theClass |
	theClass := (aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class.
	self from: textOrStream class: theClass context: aContext notifying: aRequestor.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].
	method := self interactive ifTrue: [ 	methodNode generateWithTempNames ] 
		ifFalse: [methodNode generate].
		
	logFlag ifTrue:
		[SystemChangeNotifier uniqueInstance evaluated: sourceStream contents context: aContext].
	^method! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:33' prior: 34363593!
format: aStream noPattern: noPattern ifFail: failBlock
	^(self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]) preen! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 10:08' prior: 58306325!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Compiler methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:10' prior: 50779387!
notify: aString at: location
	"Refer to the comment in Object|notify:."

	^ cue requestor == nil
		ifTrue: [SyntaxErrorNotification
					inClass: cue getClass
					category: cue category
					withCode: 
						(sourceStream contents
							copyReplaceFrom: location
							to: location - 1
							with: aString)
					doitFlag: false
					errorMessage: aString
					location: location]
		ifFalse: [cue requestor
					notify: aString
					at: location
					in: sourceStream]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:34' prior: 50777201!
parse: textOrStream in: aClass notifying: req
	"Compile the argument, textOrStream, with respect to the class, aClass, and
	 answer the MethodNode that is the root of the resulting parse tree.  Notify the
	 argument, req, if an error occurs. The failBlock is defaulted to an empty block."

	self from: textOrStream class: aClass context: nil notifying: req.
	^self parser
		parse: sourceStream
		cue: cue
		noPattern: false
		ifFail: []! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:09' prior: 36332471!
parser

	parser ifNil: [parser := (cue getClass ifNil: [self class]) newParser].
	^parser! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:37' prior: 50780779!
translate: aStream noPattern: noPattern ifFail: failBlock
	^self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:37' prior: 19124095!
translate: aStream noPattern: noPattern ifFail: failBlock parser: parser
	| tree |
	tree := parser
			parse: aStream
			cue: cue 
			noPattern: noPattern
			ifFail: [^ failBlock value].
	^ tree! !
!Encoder methodsFor: 'results' stamp: 'cwp 12/27/2012 10:26' prior: 50999892!
associationForClass
	| assoc |
	assoc := self environment associationAt: cue getClass name ifAbsent: [nil].
	^assoc value == cue getClass
		ifTrue: [assoc]
		ifFalse: [Association new value: cue getClass]! !
!Encoder methodsFor: 'temps' stamp: 'cwp 12/27/2012 10:25' prior: 20148386!
bindTemp: name in: methodSelector
	"Declare a temporary; error not if a field or class variable."
	scopeTable at: name ifPresent:[:node|
		"When non-interactive raise the error only if its a duplicate"
		(node isTemp or:[requestor interactive])
			ifTrue:[^self notify:'Name is already defined']
			ifFalse:[Transcript 
				show: '(', name, ' is shadowed in "' , cue getClass printString , '>>' , methodSelector printString , '")']].
	^self reallyBind: name! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:25' prior: 20149084!
classEncoding
	"This is a hack so that the parser may findout what class it was parsing for when it wants to create a syntax error view."
	^ cue getClass! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:39' prior: 20138819!
encodeLiteral: object

	^self
		name: object
		key: (cue  literalScannedAs: object notifying: self)
		class: LiteralNode
		type: LdLitType
		set: litSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 20139010!
encodeSelector: aSelector

	^self
		name: aSelector
		key: aSelector
		class: SelectorNode
		type: SendType
		set: selectorSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 58545123!
environment
	"Answer the environment of the current compilation context,
	 be it in a class or global (e.g. a workspace)"
	^cue environment! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 11:41' prior: 50994497!
lookupInPools: varName ifFound: assocBlock

	^Symbol
		hasInterned: varName
		ifTrue:
			[:sym|
			(cue bindingOf: sym)
				ifNil: [^false]
				ifNotNil: [:assoc| assocBlock value: assoc]]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 51004306!
possibleNamesFor: proposedName
	| results |
	results := cue getClass 
		possibleVariablesFor: proposedName 
		continuedFrom: nil.
	^ proposedName correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 50995012!
possibleVariablesFor: proposedVariable

	| results |
	results := proposedVariable correctAgainstDictionary: scopeTable
								continuedFrom: nil.
	proposedVariable first canBeGlobalVarInitial ifTrue:
		[ results := cue getClass possibleVariablesFor: proposedVariable
						continuedFrom: results ].
	^ proposedVariable correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:42' prior: 51002830!
undeclared: name
	| sym |
	requestor interactive ifTrue:
		[requestor requestor == #error: ifTrue:
			[requestor error: 'Undeclared'].
		 ^self notify: 'Undeclared'].
	"Allow knowlegeable clients to squash the undeclared warning if they want (e.g.
	 Diffing pretty printers that are simply formatting text).  As this breaks
	 compilation it should only be used by clients that want to discard the result
	 of the compilation.  To squash the warning use e.g.
		[Compiler format: code in: class notifying: nil decorated: false]
			on: UndeclaredVariableWarning
			do: [:ex| ex resume: false]"
	sym := name asSymbol.
	^(UndeclaredVariableWarning new name: name selector: selector class: cue getClass) signal
		ifTrue:
			[| undeclared |
			undeclared := cue environment undeclared.
			undeclared at: sym put: nil.
			self global: (undeclared associationAt: sym) name: sym]
		ifFalse:
			[self global: (Association key: sym) name: sym]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:23' prior: 51006007!
warnAboutShadowed: name
	requestor addWarning: name,' is shadowed'.
	selector ifNotNil:
		[Transcript cr; show: cue getClass name,'>>', selector, '(', name,' is shadowed)']! !

"Compiler"!
!SmalltalkImage methodsFor: 'housekeeping' stamp: 'cwp 6/22/2012 15:56' prior: 58497062!
cleanOutUndeclared 
	globals undeclared removeUnreferencedKeys! !
!SmalltalkImage methodsFor: 'special objects' stamp: 'cwp 6/22/2012 09:01' prior: 40515090!
recreateSpecialObjectsArray
	"Smalltalk recreateSpecialObjectsArray"
	
	"To external package developers:
	**** DO NOT OVERRIDE THIS METHOD.  *****
	If you are writing a plugin and need additional special object(s) for your own use, 
	use addGCRoot() function and use own, separate special objects registry "
	
	"The Special Objects Array is an array of objects used by the Squeak virtual machine.
	 Its contents are critical and accesses to it by the VM are unchecked, so don't even
	 think of playing here unless you know what you are doing."
	| newArray |
	newArray := Array new: 56.
	"Nil false and true get used throughout the interpreter"
	newArray at: 1 put: nil.
	newArray at: 2 put: false.
	newArray at: 3 put: true.
	"This association holds the active process (a ProcessScheduler)"
	newArray at: 4 put: (self bindingOf: #Processor).
	"Numerous classes below used for type checking and instantiation"
	newArray at: 5 put: Bitmap.
	newArray at: 6 put: SmallInteger.
	newArray at: 7 put: ByteString.
	newArray at: 8 put: Array.
	newArray at: 9 put: Smalltalk.
	newArray at: 10 put: Float.
	newArray at: 11 put: MethodContext.
	newArray at: 12 put: BlockContext.
	newArray at: 13 put: Point.
	newArray at: 14 put: LargePositiveInteger.
	newArray at: 15 put: Display.
	newArray at: 16 put: Message.
	newArray at: 17 put: CompiledMethod.
	newArray at: 18 put: (self specialObjectsArray at: 18).
	"(low space Semaphore)"
	newArray at: 19 put: Semaphore.
	newArray at: 20 put: Character.
	newArray at: 21 put: #doesNotUnderstand:.
	newArray at: 22 put: #cannotReturn:.
	newArray at: 23 put: nil. "This is the process signalling low space."
	"An array of the 32 selectors that are compiled as special bytecodes,
	 paired alternately with the number of arguments each takes."
	newArray at: 24 put: #(	#+ 1 #- 1 #< 1 #> 1 #<= 1 #>= 1 #= 1 #~= 1
							#* 1 #/ 1 #\\ 1 #@ 1 #bitShift: 1 #// 1 #bitAnd: 1 #bitOr: 1
							#at: 1 #at:put: 2 #size 0 #next 0 #nextPut: 1 #atEnd 0 #== 1 #class 0
							#blockCopy: 1 #value 0 #value: 1 #do: 1 #new 0 #new: 1 #x 0 #y 0 ).
	"An array of the 255 Characters in ascii order.
	 Cog inlines table into machine code at: prim so do not regenerate it."
	newArray at: 25 put: (self specialObjectsArray at: 25).
	newArray at: 26 put: #mustBeBoolean.
	newArray at: 27 put: ByteArray.
	newArray at: 28 put: Process.
	"An array of up to 31 classes whose instances will have compact headers"
	newArray at: 29 put: self compactClassesArray.
	newArray at: 30 put: (self specialObjectsArray at: 30). "(delay Semaphore)"
	newArray at: 31 put: (self specialObjectsArray at: 31). "(user interrupt Semaphore)"
	"Entries 32 - 34 unreferenced. Previously these contained prototype instances to be copied for fast initialization"
	newArray at: 32 put: nil. "was (Float new: 2)"
	newArray at: 33 put: nil. "was (LargePositiveInteger new: 4)"
	newArray at: 34 put: nil. "was Point new"
	newArray at: 35 put: #cannotInterpret:.
	"Note: This must be fixed once we start using context prototypes (yeah, right)"
	"(MethodContext new: CompiledMethod fullFrameSize)."
	newArray at: 36 put: (self specialObjectsArray at: 36). "Is the prototype MethodContext (unused by the VM)"
	newArray at: 37 put: BlockClosure.
	"(BlockContext new: CompiledMethod fullFrameSize)."
	newArray at: 38 put: (self specialObjectsArray at: 38). "Is the prototype BlockContext (unused by the VM)"
	"array of objects referred to by external code"
	newArray at: 39 put: (self specialObjectsArray at: 39).	"preserve external semaphores"
	newArray at: 40 put: nil. "Reserved for Mutex in Cog VMs"
	newArray at: 41 put: nil. "Reserved for a LinkedList instance for overlapped calls in CogMT"
	"finalization Semaphore"
	newArray at: 42 put: ((self specialObjectsArray at: 42) ifNil: [Semaphore new]).
	newArray at: 43 put: LargeNegativeInteger.
	"External objects for callout.
	 Note: Written so that one can actually completely remove the FFI."
	newArray at: 44 put: (self at: #ExternalAddress ifAbsent: []).
	newArray at: 45 put: (self at: #ExternalStructure ifAbsent: []).
	newArray at: 46 put: (self at: #ExternalData ifAbsent: []).
	newArray at: 47 put: (self at: #ExternalFunction ifAbsent: []).
	newArray at: 48 put: (self at: #ExternalLibrary ifAbsent: []).
	newArray at: 49 put: #aboutToReturn:through:.
	newArray at: 50 put: #run:with:in:.
	"51 reserved for immutability message"
	"newArray at: 51 put: #attemptToAssign:withIndex:."
	newArray at: 52 put: #(nil "nil => generic error" #'bad receiver'
							#'bad argument' #'bad index'
							#'bad number of arguments'
							#'inappropriate operation'  #'unsupported operation'
							#'no modification' #'insufficient object memory'
							#'insufficient C memory' #'not found' #'bad method'
							#'internal error in named primitive machinery'
							#'object may move').
	"53 to 55 are for Alien"
	newArray at: 53 put: (self at: #Alien ifAbsent: []).
	newArray at: 54 put: #invokeCallback:stack:registers:jmpbuf:.
	newArray at: 55 put: (self at: #UnsafeAlien ifAbsent: []).

	"Weak reference finalization"
	newArray at: 56 put: (self at: #WeakFinalizationList ifAbsent: []).

	"Now replace the interpreter's reference in one atomic operation"
	self specialObjectsArray becomeForward: newArray
	! !
!SmalltalkImage methodsFor: 'shrinking' stamp: 'cwp 6/22/2012 15:57' prior: 37288071!
unloadAllKnownPackages
	"Unload all packages we know how to unload and reload"

	"Prepare unloading"
	Smalltalk zapMVCprojects.
	Flaps disableGlobalFlaps: false.
	StandardScriptingSystem removeUnreferencedPlayers.
	Project removeAllButCurrent.
	#('Morphic-UserObjects' 'EToy-UserObjects' 'Morphic-Imported' )
		do: [:each | SystemOrganization removeSystemCategory: each].
	Smalltalk at: #ServiceRegistry ifPresent:[:aClass|
		SystemChangeNotifier uniqueInstance
			noMoreNotificationsFor: aClass.
	].
	World removeAllMorphs.

	"Go unloading"
	#(	'ReleaseBuilder' 'ScriptLoader'
		'311Deprecated' '39Deprecated'
		'Universes' 'SMLoader' 'SMBase' 'Installer-Core'
		'VersionNumberTests' 'VersionNumber'
		'Services-Base' 'PreferenceBrowser' 'Nebraska'
		'ToolBuilder-MVC' 'ST80'
		'CollectionsTests' 'GraphicsTests' 'KernelTests'  'MorphicTests' 
		'MultilingualTests' 'NetworkTests' 'ToolsTests' 'TraitsTests'
		'SystemChangeNotification-Tests' 'FlexibleVocabularies' 
		'EToys' 'Protocols' 'XML-Parser' 'Tests' 'SUnitGUI'
		'Help-Squeak' 'HelpSystem' 'SystemReporter'
	) do: [:pkgName| 
			(MCPackage named: pkgName) unload.
			MCMcmUpdater disableUpdatesOfPackage: pkgName.
			].
	"Traits use custom unload"
	Smalltalk at: #Trait ifPresent:[:aClass| aClass unloadTraits].

	"Post-unload cleanup"
	MCWorkingCopy flushObsoletePackageInfos.
	SystemOrganization removeSystemCategory: 'UserObjects'.
	Presenter defaultPresenterClass: nil.
	World dumpPresenter.
	ScheduledControllers := nil.
	Preferences removePreference: #allowEtoyUserCustomEvents.
	SystemOrganization removeEmptyCategories.
	ChangeSet removeChangeSetsNamedSuchThat:[:cs | (cs == ChangeSet current) not].
	globals undeclared removeUnreferencedKeys.
	StandardScriptingSystem initialize.
	MCFileBasedRepository flushAllCaches.
	MCDefinition clearInstances.
	Behavior flushObsoleteSubclasses.
	ChangeSet current clear.
	ChangeSet current name: 'Unnamed1'.
	Smalltalk flushClassNameCache.
	Smalltalk at: #Browser ifPresent:[:br| br initialize].
	DebuggerMethodMap voidMapCache.
	DataStream initialize.
	AppRegistry removeObsolete.
	FileServices removeObsolete.
	Preferences removeObsolete.
	TheWorldMenu removeObsolete.
	Smalltalk garbageCollect.
	Symbol compactSymbolTable.
	TheWorldMainDockingBar updateInstances.
	MorphicProject defaultFill: (Color gray: 0.9).
	World color: (Color gray: 0.9).
! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34' prior: 40472775!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:27' prior: 40496770!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!SystemDictionary methodsFor: 'dictionary access' stamp: 'cwp 6/22/2012 15:58' prior: 30574136!
at: aKey put: anObject 
	"Override from Dictionary to check Undeclared and fix up
	references to undeclared variables."
	| index element |
	(self includesKey: aKey) ifFalse: 
		[self declare: aKey from: (self at: #Undeclared).
		self flushClassNameCache].
	super at: aKey put: anObject.
	^ anObject! !

"System"!


From noreply at buildbot.pypy.org  Wed Jun 26 19:25:50 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 19:25:50 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: reactivated tinyBenchmarks
Message-ID: <20130626172550.0881E1C021A@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r483:e9b2d2da4ca8
Date: 2013-06-26 19:22 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/e9b2d2da4ca8/

Log:	reactivated tinyBenchmarks

diff too long, truncating to 2000 out of 80361 lines

diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
--- a/images/Squeak4.5-12568.changes
+++ b/images/Squeak4.5-12568.changes
@@ -36,4 +36,4 @@
 Workspace allInstances do: [:w | w topView delete].
 ReleaseBuilderFor4dot4 prepareNewBuild.
 Smalltalk snapshot: true andQuit: true.
-!

----End fileIn of a stream----!

----SNAPSHOT----{31 March 2013 . 3:27:34 pm} Squeak4.5-12327.image priorSource: 7430688!
!Installer methodsFor: 'squeakmap' stamp: 'fbs 1/28/2013 19:25' prior: 57597950!
packageAndVersionFrom: pkg
	| p |
	p := ReadStream on: pkg .
	^{(p upTo: $(). p upTo: $)} collect: [:s | s withBlanksTrimmed].! !

"Installer-Core"!
!Categorizer methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 16:58'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!ClassCategoryReader methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:21'!
scanFrom: aStream environment: anEnvironment
	"File in methods from the stream, aStream."
	| methodText |
	[methodText := aStream nextChunkText.
	 methodText size > 0] whileTrue:
		[class 
			compile: methodText 
			environment: anEnvironment
			classified: category
			withStamp: changeStamp 
			notifying: nil]! !
!ClassCommentReader methodsFor: 'as yet unclassified' stamp: 'cwp 6/20/2012 17:22'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!Metaclass methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:29'!
bindingOf: varName environment: anEnvironment 
	^ thisClass classBindingOf: varName environment: anEnvironment! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 20:03' prior: 22505876!
\\ aNumber 
	"Primitive. Take the receiver modulo the argument. The result is the
	remainder rounded towards negative infinity, of the receiver divided
	by the argument. Fail if the argument is 0. Fail if either the argument
	or the result is not a SmallInteger or a LargePositiveInteger less than
	2-to-the-30th (1073741824). Optional. See Object documentation whatIsAPrimitive."

	<primitive: 31>
	aNumber isInteger
		ifTrue:
			[| neg qr q r |
			neg := self negative == aNumber negative == false.
			qr := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: neg).
			q := qr first normalize.
			r := qr last normalize.
			^(q negative
				ifTrue: [r isZero not]
				ifFalse: [q isZero and: [neg]])
					ifTrue: [r + aNumber]
					ifFalse: [r]].
	^super \\ aNumber
	! !
!LargePositiveInteger methodsFor: 'converting' stamp: 'nice 1/27/2012 22:41' prior: 37616324!
asFloat
	"Answer a Float that best approximates the value of the receiver.
	This algorithm is optimized to process only the significant digits of a LargeInteger.
	And it does honour IEEE 754 round to nearest even mode in case of excess precision (see details below)."
	
	"How numbers are rounded in IEEE 754 default rounding mode:
	A shift is applied so that the highest 53 bits are placed before the floating point to form a mantissa.
	The trailing bits form the fraction part placed after the floating point.
	This fractional number must be rounded to the nearest integer.
	If fraction part is 2r0.1, exactly between two consecutive integers, there is a tie.
	The nearest even integer is chosen in this case.
	Examples (First 52bits of mantissa are omitted for brevity):
	2r0.00001 is rounded downward to 2r0
	2r1.00001 is rounded downward to 2r1
	2r0.1 is a tie and rounded to 2r0 (nearest even)
	2r1.1 is a tie and rounded to 2r10 (nearest even)
	2r0.10001 is rounded upward to 2r1
	2r1.10001 is rounded upward to 2r10
	Thus, if the next bit after floating point is 0, the mantissa is left unchanged.
	If next bit after floating point is 1, an odd mantissa is always rounded upper.
	An even mantissa is rounded upper only if the fraction part is not a tie."
	
	"Algorihm details:
	The floating point hardware can perform the rounding correctly with several excess bits as long as there is a single inexact operation.
	This can be obtained by splitting the mantissa plus excess bits in two part with less bits than Float precision.
	Note 1: the inexact flag in floating point hardware must not be trusted because in some cases the operations would be exact but would not take into account some bits that were truncated before the Floating point operations.
	Note 2: the floating point hardware is presumed configured in default rounding mode."
	
	| mantissa shift excess result n |

	"Check how many bits excess the maximum precision of a Float mantissa."
	excess := self highBitOfMagnitude - Float precision.
	excess > 7
		ifTrue:
			["Remove the excess bits but seven."
			mantissa := self bitShiftMagnitude: 7 - excess.
			shift := excess - 7.
			"An even mantissa with a single excess bit immediately following would be truncated.
			But this would not be correct if above shift has truncated some extra bits.
			Check this case, and round excess bits upper manually."
			((mantissa digitAt: 1) = 2r01000000 and: [self anyBitOfMagnitudeFrom: 1 to: shift])
				ifTrue: [mantissa := mantissa + 1]]
		ifFalse:
			[mantissa := self.
			shift := 0].

	"There will be a single inexact round off at last iteration"
	result := (mantissa digitAt: (n := mantissa digitLength)) asFloat.
	[(n := n - 1) > 0] whileTrue: [
		result := 256.0 * result + (mantissa digitAt: n) asFloat].
	^result timesTwoPower: shift.! !
!LargePositiveInteger methodsFor: 'private' stamp: 'nice 12/30/2012 14:25'!
primitiveQuo: anInteger 
	"Primitive. Divide the receiver by the argument and return the result.
	Round the result down towards zero to make it a whole integer. Fail if
	the argument is 0. Fail if either the argument or the result is not a
	SmallInteger or a LargePositiveInteger less than 2-to-the-30th (1073741824). Optional. See
	Object documentation whatIsAPrimitive."

	<primitive: 33>
	^nil! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 14:34'!
rem: aNumber 
	"Remainder defined in terms of quo:. See super rem:.
	This is defined only to speed up case of very large integers."

	(self primitiveQuo: aNumber)
		ifNotNil: [:quo | ^self - (quo * aNumber)].
	 aNumber isInteger
		ifTrue:
			[| ng rem |
			ng := self negative == aNumber negative == false.
			rem := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: ng) at: 2.
			^ rem normalize].
	^super rem: aNumber! !
!LargeNegativeInteger methodsFor: 'converting' stamp: 'nice 1/1/2013 15:42' prior: 37616204!
asFloat
	^super asFloat negated! !
!UndefinedObject methodsFor: 'class hierarchy' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:32'!
bindingOf: varName environment: anEnvironment
	^superclass bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:30'!
classBindingOf: varName environment: anEnvironment
	^self bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:37'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor
	"Postprocesses a literal scanned by Scanner scanToken (esp. xLitQuote).
	If scannedLiteral is not an association, answer it.
	Else, if it is of the form:
		nil->#NameOfMetaclass
	answer nil->theMetaclass, if any has that name, else report an error.
	Else, if it is of the form:
		#NameOfGlobalVariable->anythiEng
	answer the global, class, or pool association with that nameE, if any, else
	add it to Undeclared a answer the new Association."

	| key value |
	(scannedLiteral isVariableBinding)
		ifFalse: [^ scannedLiteral].
	key := scannedLiteral key.
	value := scannedLiteral value.
	key ifNil: "###<metaclass soleInstance name>"
		[(self bindingOf: value environment: anEnvironment) ifNotNil:
			[:assoc|
			(assoc value isKindOf: Behavior) ifTrue: 
				[^ nil->assoc value class]].
			 requestor notify: 'No such metaclass'.
			 ^false].
	(key isSymbol) ifTrue: "##<global var name>"
		[(self bindingOf: key environment: anEnvironment) ifNotNil:
			[:assoc | ^assoc].
		^ anEnvironment undeclared: key].
	requestor notify: '## must be followed by a non-local variable name'.
	^false

"	Form literalScannedAs: 14 notifying: nil 14
	Form literalScannedAs: #OneBitForm notiEfying: nil  OneBitForm
	Form literalScannedAs: ##OneBitForm notifying: nil  OneBitForm->a Form
	Form literalScannedAs: ##Form notifying: nil   Form->Form
	Form literalScannedAs: ###Form notifying: nil   nilE->Form class
"! !
!Fraction methodsFor: 'converting' stamp: 'nice 11/21/2011 22:34' prior: 37619655!
asFloat
	"Answer a Float that closely approximates the value of the receiver.
	This implementation will answer the closest floating point number to the receiver.
	In case of a tie, it will use the IEEE 754 round to nearest even mode.
	In case of overflow, it will answer +/- Float infinity."

	| a b mantissa exponent hasTruncatedBits lostBit n ha hb hm |
	a := numerator abs.
	b := denominator.	"denominator is always positive"
	ha := a highBitOfMagnitude.
	hb := b highBitOfMagnitude.
	
	"Number of bits to keep in mantissa plus one to handle rounding."
	n := 1 + Float precision.

	"If both numerator and denominator are represented exactly in floating point number,
	then fastest thing to do is to use hardwired float division."
	(ha < n and: [hb < n]) ifTrue: [^numerator asFloat / denominator asFloat].

	"Shift the fraction by a power of two exponent so as to obtain a mantissa with n bits.
	First guess is rough, the mantissa might have n+1 bits."
	exponent := ha - hb - n.
	exponent >= 0
		ifTrue: [b := b bitShift: exponent]
		ifFalse: [a := a bitShift: exponent negated].
	mantissa := a quo: b.
	hasTruncatedBits := a > (mantissa * b).
	hm := mantissa highBit.
	
	"Check for gradual underflow, in which case the mantissa will loose bits.
	Keep at least one bit to let underflow preserve the sign of zero."
	lostBit := Float emin - (exponent + hm - 1).
	lostBit > 0 ifTrue: [n := n - lostBit max: 1].

	"Remove excess bits in the mantissa."
	hm > n
		ifTrue:
			[exponent := exponent + hm - n.
			hasTruncatedBits := hasTruncatedBits or: [mantissa anyBitOfMagnitudeFrom: 1 to: hm - n].
			mantissa := mantissa bitShift: n - hm].

	"Check if mantissa must be rounded upward.
	The case of tie (mantissa odd & hasTruncatedBits not)
	will be handled by Integer>>asFloat."
	(hasTruncatedBits and: [mantissa odd])
		ifTrue: [mantissa := mantissa + 1].

	^ (self positive
			ifTrue: [mantissa asFloat]
			ifFalse: [mantissa asFloat negated])
		timesTwoPower: exponent! !
!Float methodsFor: 'arithmetic' stamp: 'nice 12/20/2012 23:16' prior: 20878776!
negated
	"Answer a Number that is the negation of the receiver.
	Implementation note: this version cares of negativeZero."

	^-1.0 * self! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:21'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor
	^ self 
		compile: text 
		environment: anEnvironment 
		classified: category 
		withStamp: changeStamp 
		notifying: requestor 
		logSource: self acceptsLoggingOfCompilation! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 12/27/2012 13:17'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor logSource: logSource
	| methodAndNode context methodNode |
	context := CompilationCue
		source: text
		class: self
		environment: anEnvironment
		category: category
		requestor: requestor.
	methodNode := self newCompiler compile: context ifFail: [^ nil].
	methodAndNode := CompiledMethodWithNode 
		generateMethodFromNode: methodNode 
		trailer: self defaultMethodTrailer.

	logSource ifTrue: [
		self logMethodSource: text forMethodWithNode: methodAndNode 
			inCategory: category withStamp: changeStamp notifying: requestor.
	].
	self addAndClassifySelector: methodAndNode selector withMethod: methodAndNode 
		method inProtocol: category notifying: requestor.
	self instanceSide noteCompilationOf: methodAndNode selector meta: self isClassSide.
	^ methodAndNode selector! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:47'!
bindingOf: varName environment: anEnvironment
	"Answer the binding of some variable resolved in the scope of the receiver"
	| aSymbol binding |
	aSymbol := varName asSymbol.

	"First look in classVar dictionary."
	binding := self classPool bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Next look in shared pools."
	self sharedPools do:[:pool | 
		binding := pool bindingOf: aSymbol.
		binding ifNotNil:[^binding].
	].

	"Next look in declared environment."
	binding := anEnvironment bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Finally look higher up the superclass chain and fail at the end."
	superclass == nil
		ifTrue: [^ nil]
		ifFalse: [^ superclass bindingOf: aSymbol].

! !

"Kernel"!

ParseNode subclass: #Encoder
	instanceVariableNames: 'scopeTable nTemps supered requestor class selector literalStream selectorSet litIndSet litSet sourceRanges globalSourceRanges addedSelectorAndMethodClassLiterals optimizedSelectors cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Encoder commentStamp: 'cwp 12/26/2012 23:29' prior: 36323851!
I encode names and literals into tree nodes with byte codes for the compiler. Byte codes for literals are not assigned until the tree-sizing pass of the compiler, because only then is it known which literals are actually needed. I also keep track of sourceCode ranges during parsing and code generation so I can provide an inverse map for the debugger.!

Scanner subclass: #Parser
	instanceVariableNames: 'here hereType hereMark hereEnd prevMark prevEnd encoder requestor parseNode failBlock requestorOffset tempsMark doitFlag properties category queriedUnusedTemporaries cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Parser commentStamp: 'cwp 12/26/2012 23:34' prior: 38557958!
I parse Smalltalk syntax and create a MethodNode that is the root of the parse tree. I look one token ahead.!

Object subclass: #CompilationCue
	instanceVariableNames: 'source context receiver class environment category requestor'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

Object subclass: #Compiler
	instanceVariableNames: 'sourceStream requestor class category context parser cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Compiler commentStamp: 'cwp 12/26/2012 23:17' prior: 59257505!
The compiler accepts Smalltalk source code and compiles it with respect to a given class. The user of the compiler supplies a context so that temporary variables are accessible during compilation. If there is an error, a requestor (usually a kind of StringHolderController) is sent the message notify:at:in: so that the error message can be displayed. If there is no error, then the result of compilation is a MethodNode, which is the root of a parse tree whose nodes are kinds of ParseNodes. The parse tree can be sent messages to (1) generate code for a CompiledMethod (this is done for compiling methods or evaluating expressions); (2) pretty-print the code (for formatting); or (3) produce a map from object code back to source code (used by debugger program-counter selection). See also Parser, Encoder, ParseNode.!
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:34'!
init: aCue notifying: anObject
	"The use of the variable requestor is a bit confusing here. This is
	*not* the original requestor, which is available through the cue.
	It's the Parser instance that is using the encoder."

	self setCue: aCue.
	requestor := anObject.
	nTemps := 0.
	supered := false.
	self initScopeAndLiteralTables.
	cue getClass variablesAndOffsetsDo:
		[:variable "<String|CFieldDefinition>" :offset "<Integer|nil>" |
		offset isNil
			ifTrue: [scopeTable at: variable name put: (FieldNode new fieldDefinition: variable)]
			ifFalse: [scopeTable
						at: variable
						put: (offset >= 0
								ifTrue: [InstanceVariableNode new
											name: variable index: offset]
								ifFalse: [MaybeContextInstanceVariableNode new
											name: variable index: offset negated])]].
	cue context ~~ nil ifTrue:
		[| homeNode |
		 homeNode := self bindTemp: self doItInContextName.
		 "0th temp = aContext passed as arg"
		 cue context tempNames withIndexDo:
			[:variable :index|
			scopeTable
				at: variable
				put: (MessageAsTempNode new
						receiver: homeNode
						selector: #namedTempAt:
						arguments: (Array with: (self encodeLiteral: index))
						precedence: 3
						from: self)]].
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/26/2012 23:30'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy instance variables for methods that
	don't use cue yet"
	class := cue getClass.! !
!Dictionary methodsFor: '*Compiler' stamp: 'cwp 6/22/2012 09:17'!
bindingOf: varName ifAbsent: aBlock

	^self associationAt: varName ifAbsent: aBlock! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:37'!
init: sourceStream cue: aCue failBlock: aBlock

	self setCue: aCue.
	failBlock := aBlock.
	requestorOffset := 0.
	super scan: sourceStream.
	prevMark := hereMark := mark.
	self advance
! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/26/2012 23:41'!
parse: sourceStream cue: aCue noPattern: noPattern ifFail: aBlock 
	"Answer a MethodNode for the argument, sourceStream, that is the root of
	 a parse tree. Parsing is done with respect to the CompilationCue to 
	 resolve variables. Errors in parsing are reported to the cue's requestor; 
	 otherwise aBlock is evaluated. The argument noPattern is a Boolean that is
	 true if the the sourceStream does not contain a method header (i.e., for DoIts)."

	| methNode repeatNeeded myStream s p subSelection |
	myStream := sourceStream.
	[repeatNeeded := false.
	 p := myStream position.
	 s := myStream upToEnd.
	 myStream position: p.
	 subSelection := aCue requestor notNil and: [aCue requestor selectionInterval = (p + 1 to: p + s size)].
	 self encoder init: aCue notifying: self.
	 self init: myStream cue: aCue failBlock: [^ aBlock value].
	 doitFlag := noPattern.
	 failBlock:= aBlock.
	 [methNode := self method: noPattern context: cue context] 
		on: ReparseAfterSourceEditing 
		do:	[ :ex |
			repeatNeeded := true.
			myStream := subSelection
							ifTrue:
								[ReadStream
									on: cue requestor text string
									from: cue requestor selectionInterval first
									to: cue requestor selectionInterval last]
							ifFalse:
								[ReadStream on: cue requestor text string]].
	 repeatNeeded] whileTrue:
		[encoder := self encoder class new].
	methNode sourceText: s.
	^methNode
! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:35'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy variables for methods that don't use cue yet."
	requestor := cue requestor.
	category := cue category.! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
class: aClass
	^ self 
		context: nil
		class: aClass
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
context: aContext class: aClass requestor: anObject
	^ self
		source: nil
		context: aContext
		receiver: nil
		class: aClass
		environment: (aClass ifNotNil: [aClass environment])
		category: nil
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aTextOrStream class: aClass environment: anEnvironment category: aString requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		receiver: nil
		class: aClass
		environment: anEnvironment
		category: aString
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
source: aTextOrStream context: aContext class: aClass category: aString requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		receiver: (aContext ifNotNil: [aContext receiver])
		class: aClass 
		environment: (aClass ifNotNil: [aClass environment])
		category: aString 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream context: aContext class: aClass requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		class: aClass 
		category: nil 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:55'!
source: aTextOrStream context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	^ self basicNew
		initializeWithSource: aTextOrStream 
		context: aContext 
		receiver: recObject 
		class: aClass 
		environment: anEnvironment 
		category: aString 
		requestor: reqObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aString environment: anEnvironment
	^ self 
		source: aString
		context: nil
		receiver: nil
		class: UndefinedObject
		environment: anEnvironment
		category: nil
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		class: nil
		requestor: anObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/20/2012 09:39'!
bindingOf: aSymbol
	^ class bindingOf: aSymbol environment: environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
category
	^ category! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 12/26/2012 23:19'!
context
	^ context! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
environment
	^ environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
getClass
	^ class! !
!CompilationCue methodsFor: 'initialization' stamp: 'cwp 12/26/2012 23:16'!
initializeWithSource: aTextOrString context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	self initialize.
	source := aTextOrString isStream ifTrue: [aTextOrString contents] ifFalse: [aTextOrString].
	context := aContext.
	receiver := recObject.
	class := aClass.
	environment := anEnvironment.
	category := aString.
	requestor := reqObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: anObject notifying: anEncoder
	^ class literalScannedAs: anObject environment: environment notifying: anEncoder! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
receiver
	^ receiver! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
requestor
	^ requestor! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
source
	^ source! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:44'!
sourceStream
	^ source readStream! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 6/20/2012 17:25'!
evaluate: aString environment: anEnvironment
	^ self 
		evaluate: aString 
		environment: anEnvironment 
		logged: false! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 12/27/2012 12:36'!
evaluate: aString environment: anEnvironment logged: aBoolean
	| cue |
	cue := CompilationCue
		source: aString
		environment: anEnvironment.
		
	^ self new
		evaluate: aString
		cue: cue
		ifFail: [^ nil]
		logged: aBoolean! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 13:18'!
compile: aCue ifFail: failBlock 
	"Answer a MethodNode. If the MethodNode can not be created, notify 
	the requestor in the contxt. If the requestor is nil, evaluate failBlock 
	instead. The MethodNode is the root  of a parse tree. It can be told 
	to generate a CompiledMethod to be installed in the method dictionary 
	of the class specified by the context."
	
	self setCue: aCue.
	self source: cue source.
	^self
		translate: sourceStream
		noPattern: false
		ifFail: failBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:06'!
evaluate: textOrStream cue: aCue ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	a method. Finally, the compiled method is invoked from here via 	withArgs:executeMethod:, hence the system no longer creates Doit method
	litter on errors."

	| methodNode method value toLog itsSelection itsSelectionString |
	self setCue: aCue.
	self source: textOrStream.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].

	method := self interactive
				ifTrue: [methodNode generateWithTempNames]
				ifFalse: [methodNode generate].

	value := cue receiver
				withArgs: (cue context ifNil: [#()] ifNotNil: [{cue context}])
				executeMethod: method.

	logFlag ifTrue:
		[toLog := ((cue requestor respondsTo: #selection)  
			and:[(itsSelection := cue requestor selection) notNil
			and:[(itsSelectionString := itsSelection asString) isEmptyOrNil not]])
				ifTrue:[itsSelectionString]
				ifFalse:[sourceStream contents].
		SystemChangeNotifier uniqueInstance evaluated: toLog context: cue context].
	^ value
! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:20'!
setCue: aCue
	cue := aCue.
	
	"Set legacy instance variables for methods that don't use cue yet."
	requestor := cue requestor.
	class := cue getClass.
	category := cue category.
	context := cue context.! !
!Compiler methodsFor: 'private' stamp: 'cwp 6/19/2012 21:58'!
source: textOrStream
	sourceStream := (textOrStream isKindOf: PositionableStream)
		ifTrue: [ textOrStream ]
		ifFalse: [ ReadStream on: textOrStream asString ]! !

"Compiler"!
!SmartRefStream class methodsFor: 'i/o' stamp: 'cwp 6/20/2012 17:42'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!SmartRefStream methodsFor: 'read write' stamp: 'cwp 6/20/2012 17:41'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!ImageSegment methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:23'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!PseudoClass methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| aString newTranslations assoc currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[Locale
		currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:26'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| newTranslations currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[| aString assoc |
	Locale currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!ObjectScanner methodsFor: 'scanning' stamp: 'cwp 6/20/2012 17:39'!
scanFrom: aByteStream environment: anEnvironment
	"This should probably be reimplemented using an environment
	for compilation. For now, don't change anything"
	^ self scanFrom: aByteStream! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 09:16'!
bindingOf: varName ifAbsent: aBlock
	"SystemDictionary includes Symbols only"
	^super bindingOf: varName asSymbol ifAbsent: aBlock! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 15:48'!
undeclared
	^ self at: #Undeclared! !

"System"!
!ExceptionTests methodsFor: 'testing-outer' stamp: 'fbs 1/1/2013 22:14' prior: 40840955!
expectedFailures
	^ #().! !

"Tests"!

ReleaseBuilder subclass: #ReleaseBuilderFor4dot5
	instanceVariableNames: ''
	classVariableNames: ''
	poolDictionaries: ''
	category: 'ReleaseBuilder'!
!ReleaseBuilderFor4dot5 commentStamp: 'fbs 1/1/2013 20:25' prior: 0!
The release builder for Squeak 4.5!
!ReleaseBuilder class methodsFor: 'scripts' stamp: 'fbs 12/31/2012 20:43'!
transferCurrentPackagesAsUser: username password: password
	"Copy the packages currently loaded in the image from the trunk repository to my releaseRepository."
	| trunkRep releaseRep |
	trunkRep := self trunkRepository.
	releaseRep := self releaseRepository
		user: username;
		password: password;
		yourself.
	MCWorkingCopy allManagers do:
		[ : eachWorkingCopy | eachWorkingCopy ancestors do:
			[ : eachVersionInfo | (releaseRep includesVersionNamed: eachVersionInfo versionName) ifFalse:
				[ (trunkRep versionWithInfo: eachVersionInfo)
					ifNil: [ Warning signal: eachVersionInfo name , ' not found in ', trunkRep ]
					ifNotNilDo: [ : ver | releaseRep storeVersion: ver ] ] ] ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
openWelcomeWorkspaces
	TheWorldMainDockingBar instance
		showWelcomeText: #squeakUserInterface
		label: 'Squeak User Interface'
		in: (40 @ 40 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #workingWithSqueak
		label: 'Working With Squeak'
		in: (80 @ 80 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #licenseInformation
		label: 'License Information'
		in: (120 @ 120 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeFutureDirections
		label: 'Future Directions'
		in: (160 @ 160 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeToSqueak
		label: 'Welcome to Squeak 4.5'
		in: (200 @ 200 extent: 500 @ 300)! !
!ReleaseBuilderFor4dot5 class methodsFor: 'scripts' stamp: 'fbs 1/1/2013 20:22'!
prepareNewBuild
	super prepareNewBuild.
	MCMockPackageInfo initialize.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:24'!
releaseRepository
	"At release time, change 'trunk' to 'squeak45'."
	^ MCHttpRepository
		location: 'http://source.squeak.org/trunk'
		user: 'squeak'
		password: 'squeak'! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:22'!
setDisplayExtent: extent
"Uncomment next line when the primitives become available in the Squeak VM."
"	DisplayScreen hostWindowSize: extent."
	Display extent = extent ifFalse: [ Warning signal: 'Display extent not set to ', extent ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
setPreferences
	Preferences 
		installBrightWindowColors ;
		setPreference: #scrollBarsWithoutMenuButton toValue: true ;
		setPreference: #swapMouseButtons toValue: true ;
		setPreference: #annotationPanes toValue: true ;
		setPreference: #showSplitterHandles toValue: false ;
		setPreference: #showBoundsInHalo toValue: true ;
		setPreference: #alternateHandlesLook toValue: false ;
		setPreference: #roundedMenuCorners toValue: false ;
		setPreference: #roundedWindowCorners toValue: false.
	PluggableButtonMorph roundedButtonCorners: false.
	FillInTheBlankMorph roundedDialogCorners: false.
	Workspace shouldStyle: false.
	NetNameResolver enableIPv6: true.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
switchToNewRepository
	| old44Repository |
	MCMcmUpdater defaultUpdateURL: self releaseRepository description.
	old44Repository := MCRepositoryGroup default repositories
				detect: [:each | each description includesSubString: 'squeak44'] ifNone: [nil].
	old44Repository
		ifNotNil: [MCRepositoryGroup default removeRepository: old44Repository].
	MCRepositoryGroup default addRepository: self releaseRepository! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
versionString
	^ 'Squeak4.5'.! !

ReleaseBuilder class removeSelector: #transferCurrentPackages!

"ReleaseBuilder"!
!Environment class methodsFor: 'as yet unclassified' stamp: 'cwp 1/1/2013 18:52' prior: 40834114!
initialize
	self install! !

"Environments"!
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:59' prior: 52081878!
initPattern: aString notifying: req return: aBlock

	| result |
	self
		init: (ReadStream on: aString asString)
		cue: (CompilationCue source: aString requestor: req)
		failBlock: [^nil].
	encoder := self.
	result := aBlock value: (self pattern: false inContext: nil).
	encoder := failBlock := nil.  "break cycles"
	^result! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:01' prior: 34175471!
parse: sourceStream class: class category: aCategory noPattern: noPattern context: aContext notifying: req ifFail: aBlock 
	| c |
	c := CompilationCue
			source: sourceStream
			context: aContext
			class: class
			category: aCategory
			requestor: req.
	^ self 
		parse: sourceStream 
		cue: c 
		noPattern: noPattern 
		ifFail: aBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:11' prior: 34183963!
evaluate: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	 a method. If aContext is not nil, the text can refer to temporaries in that
	 context (the Debugger uses this). If aRequestor is not nil, then it will receive
	 a notify:at: message before the attempt to evaluate is aborted. Finally, the 
	 compiled method is invoked from here via withArgs:executeMethod:, hence
	 the system no longer creates Doit method litter on errors."
	
	| theClass |
	theClass := ((aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class).
	self setCue: (CompilationCue
		source: textOrStream
		context: aContext
		receiver: receiver
		class: theClass
		environment: theClass environment
		category: nil
		requestor: aRequestor).
	^ self evaluate: textOrStream cue: cue ifFail: failBlock logged: logFlag! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:17' prior: 34185488!
from: textOrStream class: aClass classified: aCategory context: aContext notifying: req
	self source: textOrStream.
	self setCue: 
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			category: aCategory
			requestor: req)! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:55' prior: 50781309!
from: textOrStream class: aClass context: aContext notifying: req
	self source: textOrStream.
	self setCue:
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			requestor: req)
! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/27/2012 09:41' prior: 50996506!
init: aClass context: aContext notifying: anObject
	| c |
	c := CompilationCue
		context: aContext 
		class: aClass 
		requestor: nil.
	self init: c notifying: anObject! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:58' prior: 39061698!
temps: tempVars literals: lits class: cl 
	"Initialize this encoder for decompilation."

	self setCue: (CompilationCue class: cl).
	supered := false.
	nTemps := tempVars size.
	tempVars do: [:node | scopeTable at: node name put: node].
	literalStream := WriteStream on: (Array new: lits size).
	literalStream nextPutAll: lits.
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32.! !

"Compiler"!
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:48' prior: 36026010!
addClassVarName: aString 
	"Add the argument, aString, as a class variable of the receiver.
	Signal an error if the first character of aString is not capitalized,
	or if it is already a variable named in the class."
	| symbol oldState |
	oldState := self copy.
	aString first canBeGlobalVarInitial
		ifFalse: [^self error: aString, ' class variable name should be capitalized; proceed to include anyway.'].
	symbol := aString asSymbol.
	self withAllSubclasses do: 
		[:subclass | 
		(self canFindWithoutEnvironment: symbol) ifTrue: [
			(DuplicateVariableError new)
				superclass: superclass; "fake!!!!!!"
				variable: aString;
				signal: aString, ' is already defined']].
	classPool == nil ifTrue: [classPool := Dictionary new].
	(classPool includesKey: symbol) ifFalse: 
		["Pick up any refs in Undeclared"
		classPool declare: symbol from: environment undeclared.
		SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: oldState to: self]! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:48' prior: 54782024!
bindingOf: varName
	^ self bindingOf: varName environment: self environment! !
!Class methodsFor: 'organization' stamp: 'cwp 6/25/2012 18:25' prior: 54785804!
category
	"Answer the system organization category for the receiver. First check whether the
	category name stored in the ivar is still correct and only if this fails look it up
	(latter is much more expensive)"

	category ifNotNil: [ :symbol |
		((self environment organization listAtCategoryNamed: symbol) includes: self name)
			ifTrue: [ ^symbol ] ].
	category := self environment organization categoryOfElement: self name.
	^category! !
!Class methodsFor: 'initialize-release' stamp: 'cwp 6/22/2012 15:49' prior: 36027730!
declare: varString 
	"Declare class variables common to all instances. Answer whether 
	recompilation is advisable."

	| newVars conflicts |
	
	newVars := 
		(Scanner new scanFieldNames: varString)
			collect: [:x | x asSymbol].
	newVars do:
		[:var | var first canBeGlobalVarInitial
			ifFalse: [self error: var, ' class variable name should be capitalized; proceed to include anyway.']].
	conflicts := false.
	classPool == nil 
		ifFalse: [(classPool keys reject: [:x | newVars includes: x]) do: 
					[:var | self removeClassVarName: var]].
	(newVars reject: [:var | self classPool includesKey: var])
		do: [:var | "adding"
			"check if new vars defined elsewhere"
			(self canFindWithoutEnvironment: var) ifTrue: [
					(DuplicateVariableError new)
						superclass: superclass; "fake!!!!!!"
						variable: var;
						signal: var, ' is already defined'.
					conflicts := true]].
	newVars size > 0
		ifTrue: 
			[classPool := self classPool.
			"in case it was nil"
			newVars do: [:var | classPool declare: var from: environment undeclared]].
	^conflicts! !
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:49' prior: 54802475!
removeClassVarName: aString 
	"Remove the class variable whose name is the argument, aString, from 
	the names defined in the receiver, a class. Create an error notification if 
	aString is not a class variable or if it is still being used in the code of 
	the class."

	| aSymbol |
	aSymbol := aString asSymbol.
	(classPool includesKey: aSymbol)
		ifFalse: [^self error: aString, ' is not a class variable'].
	self withAllSubclasses do:[:subclass |
		(Array with: subclass with: subclass class) do:[:classOrMeta |
			(classOrMeta whichSelectorsReferTo: (classPool associationAt: aSymbol))
				isEmpty ifFalse: [
					InMidstOfFileinNotification signal ifTrue: [
						Transcript cr; show: self name, ' (' , aString , ' is Undeclared) '.
						^ environment undeclared declare: aSymbol from: classPool].
					(self confirm: (aString,' is still used in code of class ', classOrMeta name,
						'.\Is it okay to move it to Undeclared?') withCRs)
						ifTrue:[^Undeclared declare: aSymbol from: classPool]
						ifFalse:[^self]]]].
	classPool removeKey: aSymbol.
	classPool isEmpty ifTrue: [classPool := nil].
! !
!Class methodsFor: 'class name' stamp: 'cwp 6/22/2012 15:49' prior: 54796206!
rename: aString 
	"The new name of the receiver is the argument, aString."

	| oldName newName |
	(newName := aString asSymbol) = (oldName := self name)
		ifTrue: [^ self].
	(self environment includesKey: newName)
		ifTrue: [^ self error: newName , ' already exists'].
	(environment undeclared includesKey: newName)
		ifTrue: [self inform: 'There are references to, ' , aString printString , '
from Undeclared. Check them after this change.'].
	name := newName.
	self environment renameClass: self from: oldName! !
!ClassBuilder methodsFor: 'class definition' stamp: 'cwp 6/22/2012 01:05' prior: 39054430!
name: className inEnvironment: env subclassOf: newSuper type: type instanceVariableNames: instVarString classVariableNames: classVarString poolDictionaries: poolString category: category unsafe: unsafe
	"Define a new class in the given environment.
	If unsafe is true do not run any validation checks.
	This facility is provided to implement important system changes."
	| oldClass instVars classVars copyOfOldClass newClass |
 
	environ := env.
	instVars := Scanner new scanFieldNames: instVarString.
	classVars := (Scanner new scanFieldNames: classVarString) collect: [:x | x asSymbol].

	"Validate the proposed name"
	unsafe ifFalse:[(self validateClassName: className) ifFalse:[^nil]].
	oldClass := env at: className ifAbsent:[nil].
	oldClass isBehavior 
		ifFalse: [oldClass := nil]  "Already checked in #validateClassName:"
		ifTrue: [
			copyOfOldClass := oldClass copy.
			copyOfOldClass superclass addSubclass: copyOfOldClass].
	
	
	[ | newCategory needNew force organization oldCategory |
	unsafe ifFalse:[
		"Run validation checks so we know that we have a good chance for recompilation"
		(self validateSuperclass: newSuper forSubclass: oldClass) ifFalse:[^nil].
		(self validateInstvars: instVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateClassvars: classVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateSubclassFormat: type from: oldClass forSuper: newSuper extra: instVars size) ifFalse:[^nil]].

	"See if we need a new subclass"
	needNew := self needsSubclassOf: newSuper type: type instanceVariables: instVars from: oldClass.
	needNew == nil ifTrue:[^nil]. "some error"

	(needNew and:[unsafe not]) ifTrue:[
		"Make sure we don't redefine any dangerous classes"
		(self tooDangerousClasses includes: oldClass name) ifTrue:[
			self error: oldClass name, ' cannot be changed'.
		].
		"Check if the receiver should not be redefined"
		(oldClass ~~ nil and:[oldClass shouldNotBeRedefined]) ifTrue:[
			self notify: oldClass name asText allBold, 
						' should not be redefined. \Proceed to store over it.' withCRs]].

	needNew ifTrue:[
		"Create the new class"
		newClass := self 
			newSubclassOf: newSuper 
			type: type 
			instanceVariables: instVars
			from: oldClass.
		newClass == nil ifTrue:[^nil]. "Some error"
		newClass setName: className.
		newClass environment: environ.
	] ifFalse:[
		"Reuse the old class"
		newClass := oldClass.
	].

	"Install the class variables and pool dictionaries... "
	force := (newClass declare: classVarString) | (newClass sharing: poolString).

	"... classify ..."
	newCategory := category asSymbol.
	organization := environ ifNotNil:[environ organization].
	oldClass isNil ifFalse: [oldCategory := (organization categoryOfElement: oldClass name) asSymbol].
	organization classify: newClass name under: newCategory suppressIfDefault: true.
	
	"... recompile ..."
	newClass := self recompile: force from: oldClass to: newClass mutate: false.

	"... export if not yet done ..."
	(environ at: newClass name ifAbsent:[nil]) == newClass ifFalse:[
		[environ at: newClass name put: newClass]
			on: AttemptToWriteReadOnlyGlobal do:[:ex| ex resume: true].
		environ flushClassNameCache.
	].


	newClass doneCompiling.
	"... notify interested clients ..."
	oldClass isNil ifTrue: [
		SystemChangeNotifier uniqueInstance classAdded: newClass inCategory: newCategory.
		^ newClass].
	newCategory ~= oldCategory 
		ifTrue: [SystemChangeNotifier uniqueInstance class: newClass recategorizedFrom: oldCategory to: category]
		ifFalse: [SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: copyOfOldClass to: newClass.].
] ensure: 
		[copyOfOldClass ifNotNil: [copyOfOldClass superclass removeSubclass: copyOfOldClass].
		Behavior flushObsoleteSubclasses.
		].
	^newClass! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 22:57' prior: 18572019!
superclass: newSuper
	subclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat 
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class."
	| env |
	env := EnvironmentRequest signal ifNil: [newSuper environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: newSuper
		type: newSuper typeOfClass
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:01' prior: 50629912!
superclass: aClass
	variableByteSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable byte-sized nonpointer variables."
	| oldClassOrNil actualType env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a byte subclass of a class with named fields'].
	(aClass isVariable and: [aClass isWords])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with word fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with pointer fields'].
	oldClassOrNil := aClass environment at: t ifAbsent:[nil].
	actualType := (oldClassOrNil notNil
				   and: [oldClassOrNil typeOfClass == #compiledMethod])
					ifTrue: [#compiledMethod]
					ifFalse: [#bytes].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: actualType
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:03' prior: 18573442!
superclass: aClass
	variableSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable pointer variables."
	
	| env |
	aClass isBits ifTrue: 
		[^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #variable
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18574098!
superclass: aClass
	variableWordSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable word-sized nonpointer variables."
	| env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a word subclass of a class with named fields'].
	(aClass isVariable and: [aClass isBytes])
		ifTrue: [^self error: 'cannot make a word subclass of a class with byte fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a word subclass of a class with pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #words
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18575028!
superclass: aClass
	weakSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class (the receiver) in which the subclass is to 
	have weak indexable pointer variables."
	| env |
	aClass isBits 
		ifTrue: [^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #weak
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !

"Kernel"!
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 59135029!
ambiguousSelector: aString inRange: anInterval
	| correctedSelector userSelection offset intervalWithOffset |
	
	self interactive ifFalse: [
		"In non interactive mode, compile with backward comapatibility: $- is part of literal argument"
		Transcript cr; store: encoder classEncoding; nextPutAll:#'>>';store: encoder selector; show: ' would send ' , token , '-'.
		^super ambiguousSelector: aString inRange: anInterval].
	
	"handle the text selection"
	userSelection := cue requestor selectionInterval.
	intervalWithOffset := anInterval first + requestorOffset to: anInterval last + requestorOffset.
	cue requestor selectFrom: intervalWithOffset first to: intervalWithOffset last.
	cue requestor select.

	"Build the menu with alternatives"
	correctedSelector := AmbiguousSelector 
			signalName: aString
			inRange: intervalWithOffset.
	correctedSelector ifNil: [^self fail].

	"Execute the selected action"
	offset := self substituteWord: correctedSelector wordInterval: intervalWithOffset offset: 0.
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + offset.
	token := (correctedSelector readStream upTo: Character space) asSymbol! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 38558136!
collectTemporaryDeclarationsFrom: methodNode
	| tempsMarks str |
	tempsMarks := OrderedCollection new.
	str := cue requestor text asString.
	methodNode accept: (ParseNodeEnumerator
		ofBlock: [ :aNode | 
			| mark |
			(aNode class canUnderstand: #tempsMark) 
				ifTrue: 
					[mark := aNode tempsMark.
					(mark notNil and: [ mark between: 1 and: str size ] and: [ (str at: mark) = $| ])
						ifTrue: [ tempsMarks addLast: aNode ]]]).
	(tempsMark notNil and: [ tempsMark between: 1 and: str size ] and: [ (str at: tempsMark) = $| ])
						ifTrue: [ tempsMarks addLast: self ].
	^ tempsMarks sorted: [ :a :b | a tempsMark > b tempsMark ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 52096606!
correctSelector: proposedKeyword wordIntervals: spots exprInterval: expInt ifAbort: abortAction
	"Correct the proposedKeyword to some selector symbol, correcting the original text if such action is indicated.  abortAction is invoked if the proposedKeyword couldn't be converted into a valid selector.  Spots is an ordered collection of intervals within the test stream of the for each of the keyword parts."

	| correctSelector userSelection |
	"If we can't ask the user, assume that the keyword will be defined later"
	self interactive ifFalse: [^proposedKeyword asSymbol].

	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spots first first to: spots last last.
	cue requestor select.

	correctSelector := UnknownSelector name: proposedKeyword.
	correctSelector ifNil: [^abortAction value].

	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.

	self substituteSelector: correctSelector keywords wordIntervals: spots.
	^(proposedKeyword last ~~ $:
	   and: [correctSelector last == $:])
		ifTrue: [abortAction value]
		ifFalse: [correctSelector]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 33907242!
correctVariable: proposedVariable interval: spot 
	"Correct the proposedVariable to a known variable, or declare it as a new
	variable if such action is requested.  We support declaring lowercase
	variables as temps or inst-vars, and uppercase variables as Globals or 
	ClassVars, depending on whether the context is nil (class=UndefinedObject).
	Spot is the interval within the test stream of the variable.
	rr 3/4/2004 10:26 : adds the option to define a new class. "

	"Check if this is an i-var, that has been corrected already (ugly)"

	"Display the pop-up menu"

	| binding userSelection action |
	(encoder classEncoding instVarNames includes: proposedVariable) ifTrue: 
		[^InstanceVariableNode new 
			name: proposedVariable
			index: (encoder classEncoding allInstVarNames indexOf: proposedVariable)].

	"If we can't ask the user for correction, make it undeclared"
	self interactive ifFalse: [^encoder undeclared: proposedVariable].

	"First check to see if the requestor knows anything about the variable"
	(binding := cue requestor bindingOf: proposedVariable)
		ifNotNil: [^encoder global: binding name: proposedVariable].
	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spot first to: spot last.
	cue requestor select.

	"Build the menu with alternatives"
	action := UndeclaredVariable 
				signalFor: self
				name: proposedVariable
				inRange: spot.
	action ifNil: [^self fail].

	"Execute the selected action"
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.
	^action value! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:19' prior: 34172921!
declareUndeclaredTemps: methodNode
	"Declare any undeclared temps, declaring them at the smallest enclosing scope."

	| undeclared userSelection blocksToVars |
	(undeclared := encoder undeclaredTemps) isEmpty ifTrue:
		[^self].
	userSelection := cue requestor selectionInterval.
	blocksToVars := IdentityDictionary new.
	undeclared do:
		[:var|
		(blocksToVars
			at: (var tag == #method
					ifTrue: [methodNode block]
					ifFalse: [methodNode accept: (VariableScopeFinder new ofVariable: var)])
			ifAbsentPut: [SortedCollection new]) add: var name].
	(blocksToVars removeKey: methodNode block ifAbsent: []) ifNotNil:
		[:rootVars|
		rootVars do: [:varName| self pasteTempAtMethodLevel: varName]].
	(blocksToVars keys sorted: [:a :b| a tempsMark < b tempsMark]) do:
		[:block| | decl |
		decl := (blocksToVars at: block) reduce: [:a :b| a, ' ', b].
		block temporaries isEmpty
			ifTrue:
				[self substituteWord: ' | ', decl, ' |'
					wordInterval: (block tempsMark + 1 to: block tempsMark)
					offset: requestorOffset]
			ifFalse:
				[self substituteWord: decl, ' '
					wordInterval: (block tempsMark to: block tempsMark - 1)
					offset: requestorOffset]].
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + requestorOffset.
	ReparseAfterSourceEditing signal! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 11:45' prior: 37183770!
defineClass: className 
	"prompts the user to define a new class,  
	asks for it's category, and lets the users edit further  
	the definition"
	| sym cat def d2 |
	sym := className asSymbol.
	cat := UIManager default request: 'Enter class category : ' initialAnswer: self encoder classEncoding theNonMetaClass category.
	cat
		ifEmpty: [cat := 'Unknown'].
	def := 'Object subclass: #' , sym , '
		instanceVariableNames: '''' 
		classVariableNames: ''''
		poolDictionaries: ''''
		category: ''' , cat , ''''.
	d2 := UIManager default request: 'Edit class definition : ' initialAnswer: def.
	d2
		ifEmpty: [d2 := def].
	Compiler evaluate: d2.
	^ encoder
		global: (cue environment bindingOf: sym)
		name: sym! !
!Parser methodsFor: 'primitives' stamp: 'cwp 12/27/2012 11:46' prior: 37184567!
externalFunctionDeclaration
	"Parse the function declaration for a call to an external library."
	| descriptorClass callType modifier retType externalName args argType module fn |
	descriptorClass := cue environment
		valueOf: #ExternalFunction 
		ifAbsent: [^ false].
	callType := descriptorClass callingConventionFor: here.
	callType == nil ifTrue:[^false].
	[modifier := descriptorClass callingConventionModifierFor: token.
	 modifier notNil] whileTrue:
		[self advance.
		 callType := callType bitOr: modifier].
	"Parse return type"
	self advance.
	retType := self externalType: descriptorClass.
	retType == nil ifTrue:[^self expected:'return type'].
	"Parse function name or index"
	externalName := here.
	(self match: #string) 
		ifTrue:[externalName := externalName asSymbol]
		ifFalse:[(self match:#number) ifFalse:[^self expected:'function name or index']].
	(self matchToken: #'(') ifFalse:[^self expected:'argument list'].
	args := WriteStream on: Array new.
	[here == #')'] whileFalse:[
		argType := self externalType: descriptorClass.
		argType == nil ifTrue:[^self expected:'argument'].
		argType isVoid & argType isPointerType not ifFalse:[args nextPut: argType].
	].
	(self matchToken: #')') ifFalse:[^self expected:')'].
	(self matchToken: 'module:') ifTrue:[
		module := here.
		(self match: #string) ifFalse:[^self expected: 'String'].
		module := module asSymbol].
	Smalltalk at: #ExternalLibraryFunction ifPresent:[:xfn|
		fn := xfn name: externalName 
				module: module 
				callType: callType
				returnType: retType
				argumentTypes: args contents.
		self allocateLiteral: fn.
	].
	(self matchToken: 'error:')
		ifTrue:
			[| errorCodeVariable |
			 errorCodeVariable := here.
			(hereType == #string
			 or: [hereType == #word]) ifFalse:[^self expected: 'error code (a variable or string)'].
			 self advance.
			 self addPragma: (Pragma keyword: #primitive:error: arguments: (Array with: 120 with: errorCodeVariable)).
			 fn ifNotNil: [fn setErrorCodeName: errorCodeVariable]]
		ifFalse:
			[self addPragma: (Pragma keyword: #primitive: arguments: #(120))].
	^true
! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:19' prior: 58306169!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:22' prior: 58137223!
notify: string at: location
	cue requestor isNil
		ifTrue: [(encoder == self or: [encoder isNil]) ifTrue: [^ self fail  "failure setting up syntax error"].
				SyntaxErrorNotification
					inClass: encoder classEncoding
					category: cue category
					withCode: 
						(source contents asText
							copyReplaceFrom: location
							to: location - 1
							with: ((string , ' ->') asText allBold 
								addAttribute: TextColor red; yourself))
					doitFlag: doitFlag
					errorMessage: string
					location: location]
		ifFalse: [cue requestor
					notify: string , ' ->'
					at: location
					in: source].
	^self fail! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:17' prior: 34177108!
pasteTempAtMethodLevel: name
	| insertion delta theTextString characterBeforeMark |

	theTextString := cue requestor text string.
	characterBeforeMark := theTextString at: tempsMark-1 ifAbsent: [$ ].
	(theTextString at: tempsMark) = $| ifTrue: [
  		"Paste it before the second vertical bar"
		insertion := name, ' '.
		characterBeforeMark isSeparator ifFalse: [insertion := ' ', insertion].
		delta := 0.
	] ifFalse: [
		"No bars - insert some with CR, tab"
		insertion := '| ' , name , ' |',String cr.
		delta := 2.	"the bar and CR"
		characterBeforeMark = Character tab ifTrue: [
			insertion := insertion , String tab.
			delta := delta + 1.	"the tab"
		].
	].
	tempsMark := tempsMark +
		(self substituteWord: insertion
			wordInterval: (tempsMark to: tempsMark-1)
			offset: 0) - delta! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:16' prior: 52095305!
queryUndefined
	| varStart varName | 
	varName := parseNode key.
	varStart := self endOfLastToken + requestorOffset - varName size + 1.
	cue requestor selectFrom: varStart to: varStart + varName size - 1; select.
	(UndefinedVariable name: varName) ifFalse: [^ self fail]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38599341!
removeEmptyTempDeclarationsFrom: methodNode

	| sourceCode madeChanges tempsMarkHolder |
	sourceCode := cue requestor text asString.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	madeChanges := false.
	tempsMarkHolder do: [ :currentBlock | | tempsMarkChar0 tempsMarkChar1 tempsMarkChar2 end start |
		tempsMarkChar0 := (sourceCode at: currentBlock tempsMark).
		tempsMarkChar1 := (sourceCode at: currentBlock tempsMark - 1).
		tempsMarkChar2 := (sourceCode at: currentBlock tempsMark - 2).
		tempsMarkChar0 = $| & tempsMarkChar1 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 1].
		tempsMarkChar0 = $| & tempsMarkChar1 = $  & tempsMarkChar2 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 2].
		
		start notNil & end notNil ifTrue: [
			| lineStart lineEnd |
			lineStart := 1 + (sourceCode 
				lastIndexOf: Character cr 
				startingAt: start - 1
				ifAbsent: [ 0 ]).
			lineEnd := sourceCode 
				indexOf: Character cr
				startingAt: end + 1
				ifAbsent: [ sourceCode size ].
			((sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: lineStart) >= start 
				and: [ (sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: end + 1) > lineEnd ]) ifTrue: [
					start := lineStart.
					end := lineEnd ].
			cue requestor correctFrom: start to: end with: ''.
			madeChanges := true.
			currentBlock tempsMark: nil ] ].
	madeChanges ifTrue: [ReparseAfterSourceEditing signal]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38561281!
removeUnusedTemporaryNamed: temp from: str lookingAt: currentBlock movingTempMarksOf: someBlocks

	| start end |
	end := currentBlock tempsMark - 1.
	["Beginning at right temp marker..."
	start := end - temp size + 1.
	end < temp size or: [ (str at: start) = $| ]
		or: [ temp = (str copyFrom: start to: end) 
			and: [ ((str at: start - 1) = $| | (str at: start - 1) isSeparator) 
				& ((str at: end + 1) = $| | (str at: end + 1) isSeparator) ] ]]
		whileFalse: [ 
			"Search left for the unused temp"
			end := cue requestor nextTokenFrom: end direction: -1 ].
	(end < temp size or: [ (str at: start) = $| ])
		ifFalse: 
			[(str at: start - 1) = $ 
				ifTrue: [ start := start - 1 ].
			cue requestor correctFrom: start to: end with: ''.
			someBlocks do: [ :aBlock | aBlock tempsMark: aBlock tempsMark - (end - start + 1)].
			^true ].
	^false! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 38562194!
removeUnusedTemps: methodNode
	"Scan for unused temp names, and prompt the user about the prospect of removing each one found"

	| madeChanges tempsMarkHolder unusedTempNames tempMarkHoldersToChange |
	madeChanges := false.
	tempMarkHoldersToChange := OrderedCollection new.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	unusedTempNames := encoder unusedTempNames select: 
		[ :temp | (encoder lookupVariable: temp ifAbsent: [ ]) isUndefTemp 
				and: [ self queriedUnusedTemporaries at: temp ifAbsentPut: [UnusedVariable name: temp] ]].
	tempsMarkHolder do: [ :currentBlock | 
		tempMarkHoldersToChange add: currentBlock.
		unusedTempNames do: 
			[ :temp |
			(self 
				removeUnusedTemporaryNamed: temp 
				from: cue requestor text asString 
				lookingAt: currentBlock
				movingTempMarksOf: tempMarkHoldersToChange) ifTrue: [ madeChanges := true ]]].
	madeChanges
		ifTrue: [ self removeEmptyTempDeclarationsFrom: methodNode.
			ReparseAfterSourceEditing signal ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 34179326!
substituteWord: correctWord wordInterval: spot offset: o
	"Substitute the correctSelector into the (presumed interactive) receiver.
	 Update requestorOffset based on the delta size and answer the updated offset."

	cue requestor correctFrom: spot first + o to: spot last + o with: correctWord.
	requestorOffset := requestorOffset + correctWord size - spot size.
	^o + correctWord size - spot size! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34179807!
temporaries
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'
! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34180638!
temporariesIn: methodSelector
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance in: methodSelector)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:11' prior: 53971863!
compiledMethodFor: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code
	 into a method, and answers it.  If receiver is not nil, then the text can
	 refer to instance variables of that receiver (the Inspector uses this).
	 If aContext is not nil, the text can refer to temporaries in that context
	 (the Debugger uses this). If aRequestor is not nil, then it will receive a 
	 notify:at: message before the attempt to evaluate is aborted."

	| methodNode method theClass |
	theClass := (aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class.
	self from: textOrStream class: theClass context: aContext notifying: aRequestor.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].
	method := self interactive ifTrue: [ 	methodNode generateWithTempNames ] 
		ifFalse: [methodNode generate].
		
	logFlag ifTrue:
		[SystemChangeNotifier uniqueInstance evaluated: sourceStream contents context: aContext].
	^method! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:33' prior: 34363593!
format: aStream noPattern: noPattern ifFail: failBlock
	^(self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]) preen! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 10:08' prior: 58306325!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Compiler methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:10' prior: 50779387!
notify: aString at: location
	"Refer to the comment in Object|notify:."

	^ cue requestor == nil
		ifTrue: [SyntaxErrorNotification
					inClass: cue getClass
					category: cue category
					withCode: 
						(sourceStream contents
							copyReplaceFrom: location
							to: location - 1
							with: aString)
					doitFlag: false
					errorMessage: aString
					location: location]
		ifFalse: [cue requestor
					notify: aString
					at: location
					in: sourceStream]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:34' prior: 50777201!
parse: textOrStream in: aClass notifying: req
	"Compile the argument, textOrStream, with respect to the class, aClass, and
	 answer the MethodNode that is the root of the resulting parse tree.  Notify the
	 argument, req, if an error occurs. The failBlock is defaulted to an empty block."

	self from: textOrStream class: aClass context: nil notifying: req.
	^self parser
		parse: sourceStream
		cue: cue
		noPattern: false
		ifFail: []! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:09' prior: 36332471!
parser

	parser ifNil: [parser := (cue getClass ifNil: [self class]) newParser].
	^parser! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:37' prior: 50780779!
translate: aStream noPattern: noPattern ifFail: failBlock
	^self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:37' prior: 19124095!
translate: aStream noPattern: noPattern ifFail: failBlock parser: parser
	| tree |
	tree := parser
			parse: aStream
			cue: cue 
			noPattern: noPattern
			ifFail: [^ failBlock value].
	^ tree! !
!Encoder methodsFor: 'results' stamp: 'cwp 12/27/2012 10:26' prior: 50999892!
associationForClass
	| assoc |
	assoc := self environment associationAt: cue getClass name ifAbsent: [nil].
	^assoc value == cue getClass
		ifTrue: [assoc]
		ifFalse: [Association new value: cue getClass]! !
!Encoder methodsFor: 'temps' stamp: 'cwp 12/27/2012 10:25' prior: 20148386!
bindTemp: name in: methodSelector
	"Declare a temporary; error not if a field or class variable."
	scopeTable at: name ifPresent:[:node|
		"When non-interactive raise the error only if its a duplicate"
		(node isTemp or:[requestor interactive])
			ifTrue:[^self notify:'Name is already defined']
			ifFalse:[Transcript 
				show: '(', name, ' is shadowed in "' , cue getClass printString , '>>' , methodSelector printString , '")']].
	^self reallyBind: name! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:25' prior: 20149084!
classEncoding
	"This is a hack so that the parser may findout what class it was parsing for when it wants to create a syntax error view."
	^ cue getClass! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:39' prior: 20138819!
encodeLiteral: object

	^self
		name: object
		key: (cue  literalScannedAs: object notifying: self)
		class: LiteralNode
		type: LdLitType
		set: litSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 20139010!
encodeSelector: aSelector

	^self
		name: aSelector
		key: aSelector
		class: SelectorNode
		type: SendType
		set: selectorSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 58545123!
environment
	"Answer the environment of the current compilation context,
	 be it in a class or global (e.g. a workspace)"
	^cue environment! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 11:41' prior: 50994497!
lookupInPools: varName ifFound: assocBlock

	^Symbol
		hasInterned: varName
		ifTrue:
			[:sym|
			(cue bindingOf: sym)
				ifNil: [^false]
				ifNotNil: [:assoc| assocBlock value: assoc]]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 51004306!
possibleNamesFor: proposedName
	| results |
	results := cue getClass 
		possibleVariablesFor: proposedName 
		continuedFrom: nil.
	^ proposedName correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 50995012!
possibleVariablesFor: proposedVariable

	| results |
	results := proposedVariable correctAgainstDictionary: scopeTable
								continuedFrom: nil.
	proposedVariable first canBeGlobalVarInitial ifTrue:
		[ results := cue getClass possibleVariablesFor: proposedVariable
						continuedFrom: results ].
	^ proposedVariable correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:42' prior: 51002830!
undeclared: name
	| sym |
	requestor interactive ifTrue:
		[requestor requestor == #error: ifTrue:
			[requestor error: 'Undeclared'].
		 ^self notify: 'Undeclared'].
	"Allow knowlegeable clients to squash the undeclared warning if they want (e.g.
	 Diffing pretty printers that are simply formatting text).  As this breaks
	 compilation it should only be used by clients that want to discard the result
	 of the compilation.  To squash the warning use e.g.
		[Compiler format: code in: class notifying: nil decorated: false]
			on: UndeclaredVariableWarning
			do: [:ex| ex resume: false]"
	sym := name asSymbol.
	^(UndeclaredVariableWarning new name: name selector: selector class: cue getClass) signal
		ifTrue:
			[| undeclared |
			undeclared := cue environment undeclared.
			undeclared at: sym put: nil.
			self global: (undeclared associationAt: sym) name: sym]
		ifFalse:
			[self global: (Association key: sym) name: sym]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:23' prior: 51006007!
warnAboutShadowed: name
	requestor addWarning: name,' is shadowed'.
	selector ifNotNil:
		[Transcript cr; show: cue getClass name,'>>', selector, '(', name,' is shadowed)']! !

"Compiler"!
!SmalltalkImage methodsFor: 'housekeeping' stamp: 'cwp 6/22/2012 15:56' prior: 58497062!
cleanOutUndeclared 
	globals undeclared removeUnreferencedKeys! !
!SmalltalkImage methodsFor: 'special objects' stamp: 'cwp 6/22/2012 09:01' prior: 40515090!
recreateSpecialObjectsArray
	"Smalltalk recreateSpecialObjectsArray"
	
	"To external package developers:
	**** DO NOT OVERRIDE THIS METHOD.  *****
	If you are writing a plugin and need additional special object(s) for your own use, 
	use addGCRoot() function and use own, separate special objects registry "
	
	"The Special Objects Array is an array of objects used by the Squeak virtual machine.
	 Its contents are critical and accesses to it by the VM are unchecked, so don't even
	 think of playing here unless you know what you are doing."
	| newArray |
	newArray := Array new: 56.
	"Nil false and true get used throughout the interpreter"
	newArray at: 1 put: nil.
	newArray at: 2 put: false.
	newArray at: 3 put: true.
	"This association holds the active process (a ProcessScheduler)"
	newArray at: 4 put: (self bindingOf: #Processor).
	"Numerous classes below used for type checking and instantiation"
	newArray at: 5 put: Bitmap.
	newArray at: 6 put: SmallInteger.
	newArray at: 7 put: ByteString.
	newArray at: 8 put: Array.
	newArray at: 9 put: Smalltalk.
	newArray at: 10 put: Float.
	newArray at: 11 put: MethodContext.
	newArray at: 12 put: BlockContext.
	newArray at: 13 put: Point.
	newArray at: 14 put: LargePositiveInteger.
	newArray at: 15 put: Display.
	newArray at: 16 put: Message.
	newArray at: 17 put: CompiledMethod.
	newArray at: 18 put: (self specialObjectsArray at: 18).
	"(low space Semaphore)"
	newArray at: 19 put: Semaphore.
	newArray at: 20 put: Character.
	newArray at: 21 put: #doesNotUnderstand:.
	newArray at: 22 put: #cannotReturn:.
	newArray at: 23 put: nil. "This is the process signalling low space."
	"An array of the 32 selectors that are compiled as special bytecodes,
	 paired alternately with the number of arguments each takes."
	newArray at: 24 put: #(	#+ 1 #- 1 #< 1 #> 1 #<= 1 #>= 1 #= 1 #~= 1
							#* 1 #/ 1 #\\ 1 #@ 1 #bitShift: 1 #// 1 #bitAnd: 1 #bitOr: 1
							#at: 1 #at:put: 2 #size 0 #next 0 #nextPut: 1 #atEnd 0 #== 1 #class 0
							#blockCopy: 1 #value 0 #value: 1 #do: 1 #new 0 #new: 1 #x 0 #y 0 ).
	"An array of the 255 Characters in ascii order.
	 Cog inlines table into machine code at: prim so do not regenerate it."
	newArray at: 25 put: (self specialObjectsArray at: 25).
	newArray at: 26 put: #mustBeBoolean.
	newArray at: 27 put: ByteArray.
	newArray at: 28 put: Process.
	"An array of up to 31 classes whose instances will have compact headers"
	newArray at: 29 put: self compactClassesArray.
	newArray at: 30 put: (self specialObjectsArray at: 30). "(delay Semaphore)"
	newArray at: 31 put: (self specialObjectsArray at: 31). "(user interrupt Semaphore)"
	"Entries 32 - 34 unreferenced. Previously these contained prototype instances to be copied for fast initialization"
	newArray at: 32 put: nil. "was (Float new: 2)"
	newArray at: 33 put: nil. "was (LargePositiveInteger new: 4)"
	newArray at: 34 put: nil. "was Point new"
	newArray at: 35 put: #cannotInterpret:.
	"Note: This must be fixed once we start using context prototypes (yeah, right)"
	"(MethodContext new: CompiledMethod fullFrameSize)."
	newArray at: 36 put: (self specialObjectsArray at: 36). "Is the prototype MethodContext (unused by the VM)"
	newArray at: 37 put: BlockClosure.
	"(BlockContext new: CompiledMethod fullFrameSize)."
	newArray at: 38 put: (self specialObjectsArray at: 38). "Is the prototype BlockContext (unused by the VM)"
	"array of objects referred to by external code"
	newArray at: 39 put: (self specialObjectsArray at: 39).	"preserve external semaphores"
	newArray at: 40 put: nil. "Reserved for Mutex in Cog VMs"
	newArray at: 41 put: nil. "Reserved for a LinkedList instance for overlapped calls in CogMT"
	"finalization Semaphore"
	newArray at: 42 put: ((self specialObjectsArray at: 42) ifNil: [Semaphore new]).
	newArray at: 43 put: LargeNegativeInteger.
	"External objects for callout.
	 Note: Written so that one can actually completely remove the FFI."
	newArray at: 44 put: (self at: #ExternalAddress ifAbsent: []).
	newArray at: 45 put: (self at: #ExternalStructure ifAbsent: []).
	newArray at: 46 put: (self at: #ExternalData ifAbsent: []).
	newArray at: 47 put: (self at: #ExternalFunction ifAbsent: []).
	newArray at: 48 put: (self at: #ExternalLibrary ifAbsent: []).
	newArray at: 49 put: #aboutToReturn:through:.
	newArray at: 50 put: #run:with:in:.
	"51 reserved for immutability message"
	"newArray at: 51 put: #attemptToAssign:withIndex:."
	newArray at: 52 put: #(nil "nil => generic error" #'bad receiver'
							#'bad argument' #'bad index'
							#'bad number of arguments'
							#'inappropriate operation'  #'unsupported operation'
							#'no modification' #'insufficient object memory'
							#'insufficient C memory' #'not found' #'bad method'
							#'internal error in named primitive machinery'
							#'object may move').
	"53 to 55 are for Alien"
	newArray at: 53 put: (self at: #Alien ifAbsent: []).
	newArray at: 54 put: #invokeCallback:stack:registers:jmpbuf:.
	newArray at: 55 put: (self at: #UnsafeAlien ifAbsent: []).

	"Weak reference finalization"
	newArray at: 56 put: (self at: #WeakFinalizationList ifAbsent: []).

	"Now replace the interpreter's reference in one atomic operation"
	self specialObjectsArray becomeForward: newArray
	! !
!SmalltalkImage methodsFor: 'shrinking' stamp: 'cwp 6/22/2012 15:57' prior: 37288071!
unloadAllKnownPackages
	"Unload all packages we know how to unload and reload"

	"Prepare unloading"
	Smalltalk zapMVCprojects.
	Flaps disableGlobalFlaps: false.
	StandardScriptingSystem removeUnreferencedPlayers.
	Project removeAllButCurrent.
	#('Morphic-UserObjects' 'EToy-UserObjects' 'Morphic-Imported' )
		do: [:each | SystemOrganization removeSystemCategory: each].
	Smalltalk at: #ServiceRegistry ifPresent:[:aClass|
		SystemChangeNotifier uniqueInstance
			noMoreNotificationsFor: aClass.
	].
	World removeAllMorphs.

	"Go unloading"
	#(	'ReleaseBuilder' 'ScriptLoader'
		'311Deprecated' '39Deprecated'
		'Universes' 'SMLoader' 'SMBase' 'Installer-Core'
		'VersionNumberTests' 'VersionNumber'
		'Services-Base' 'PreferenceBrowser' 'Nebraska'
		'ToolBuilder-MVC' 'ST80'
		'CollectionsTests' 'GraphicsTests' 'KernelTests'  'MorphicTests' 
		'MultilingualTests' 'NetworkTests' 'ToolsTests' 'TraitsTests'
		'SystemChangeNotification-Tests' 'FlexibleVocabularies' 
		'EToys' 'Protocols' 'XML-Parser' 'Tests' 'SUnitGUI'
		'Help-Squeak' 'HelpSystem' 'SystemReporter'
	) do: [:pkgName| 
			(MCPackage named: pkgName) unload.
			MCMcmUpdater disableUpdatesOfPackage: pkgName.
			].
	"Traits use custom unload"
	Smalltalk at: #Trait ifPresent:[:aClass| aClass unloadTraits].

	"Post-unload cleanup"
	MCWorkingCopy flushObsoletePackageInfos.
	SystemOrganization removeSystemCategory: 'UserObjects'.
	Presenter defaultPresenterClass: nil.
	World dumpPresenter.
	ScheduledControllers := nil.
	Preferences removePreference: #allowEtoyUserCustomEvents.
	SystemOrganization removeEmptyCategories.
	ChangeSet removeChangeSetsNamedSuchThat:[:cs | (cs == ChangeSet current) not].
	globals undeclared removeUnreferencedKeys.
	StandardScriptingSystem initialize.
	MCFileBasedRepository flushAllCaches.
	MCDefinition clearInstances.
	Behavior flushObsoleteSubclasses.
	ChangeSet current clear.
	ChangeSet current name: 'Unnamed1'.
	Smalltalk flushClassNameCache.
	Smalltalk at: #Browser ifPresent:[:br| br initialize].
	DebuggerMethodMap voidMapCache.
	DataStream initialize.
	AppRegistry removeObsolete.
	FileServices removeObsolete.
	Preferences removeObsolete.
	TheWorldMenu removeObsolete.
	Smalltalk garbageCollect.
	Symbol compactSymbolTable.
	TheWorldMainDockingBar updateInstances.
	MorphicProject defaultFill: (Color gray: 0.9).
	World color: (Color gray: 0.9).
! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34' prior: 40472775!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:27' prior: 40496770!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!SystemDictionary methodsFor: 'dictionary access' stamp: 'cwp 6/22/2012 15:58' prior: 30574136!
at: aKey put: anObject 
	"Override from Dictionary to check Undeclared and fix up
	references to undeclared variables."
	| index element |
	(self includesKey: aKey) ifFalse: 
		[self declare: aKey from: (self at: #Undeclared).
		self flushClassNameCache].
	super at: aKey put: anObject.
	^ anObject! !

"System"!

CodeHolder subclass: #Browser
	instanceVariableNames: 'environment systemOrganizer classOrganizer metaClassOrganizer editSelection metaClassIndicated selectedSystemCategory selectedClassName selectedMessageName selectedMessageCategoryName'
	classVariableNames: 'ListClassesHierarchically RecentClasses'
	poolDictionaries: ''
	category: 'Tools-Browser'!
!Browser commentStamp: 'cwp 12/27/2012 11:09' prior: 36419432!
I represent a query path into the class descriptions, the software of the system.!
!Browser methodsFor: 'accessing' stamp: 'cwp 6/24/2012 23:20'!
selectEnvironment: anEnvironment 
	environment := anEnvironment.
	systemOrganizer := environment organization! !
!Browser methodsFor: 'system category list' stamp: 'cwp 6/24/2012 23:06' prior: 36467357!

From noreply at buildbot.pypy.org  Wed Jun 26 20:11:58 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 20:11:58 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: another step on the way to
	benchmarks
Message-ID: <20130626181158.29BFD1C0325@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r484:d8758c85c2ce
Date: 2013-06-26 20:05 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/d8758c85c2ce/

Log:	another step on the way to benchmarks

diff too long, truncating to 2000 out of 80508 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -166,7 +166,7 @@
             callback(self)
 
     def run(self, args):
-        print 'Calling %s (./%s) ...' % (self.name, " ".join([self.path] + args))
+        print 'Calling %s (%s) ...' % (self.name, " ".join([self.path] + args))
         pipe = subprocess.Popen(
             ["%s" % self.path] + args,
             stdout=subprocess.PIPE
@@ -220,13 +220,16 @@
 RSqueakVM = Project(
     "lang-smalltalk",
     executables=[
-        Executable("rsqueakvm", "./targetimageloadingsmalltalk-c"),
+        Executable("rsqueakvm", "bash"),
         # Executable("rsqueakvm-nojit", "./targetimageloadingsmalltalk-nojit-c")
     ],
-    arguments=["images/%s.image" % SqueakImage, '-m', 'runSPyBenchmarks']
+    arguments=["-c", "./targetimageloadingsmalltalk-c images/%s.image -m runSPyBenchmarks > >(tee stdout.log) 2> >(tee stderr.log >&2)" % SqueakImage]
 )
 
 
 if __name__ == "__main__":
-    for project in [Cog, RSqueakVM]:
-        project.post_results()
+    try:
+        for project in [RSqueakVM]:
+            project.post_results()
+    finally:
+        subprocess.Popen(["rm", '-r', "stackvm"])
diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
--- a/images/Squeak4.5-12568.changes
+++ b/images/Squeak4.5-12568.changes
@@ -36,4 +36,4 @@
 Workspace allInstances do: [:w | w topView delete].
 ReleaseBuilderFor4dot4 prepareNewBuild.
 Smalltalk snapshot: true andQuit: true.
-!

----End fileIn of a stream----!

----SNAPSHOT----{31 March 2013 . 3:27:34 pm} Squeak4.5-12327.image priorSource: 7430688!
!Installer methodsFor: 'squeakmap' stamp: 'fbs 1/28/2013 19:25' prior: 57597950!
packageAndVersionFrom: pkg
	| p |
	p := ReadStream on: pkg .
	^{(p upTo: $(). p upTo: $)} collect: [:s | s withBlanksTrimmed].! !

"Installer-Core"!
!Categorizer methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 16:58'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!ClassCategoryReader methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:21'!
scanFrom: aStream environment: anEnvironment
	"File in methods from the stream, aStream."
	| methodText |
	[methodText := aStream nextChunkText.
	 methodText size > 0] whileTrue:
		[class 
			compile: methodText 
			environment: anEnvironment
			classified: category
			withStamp: changeStamp 
			notifying: nil]! !
!ClassCommentReader methodsFor: 'as yet unclassified' stamp: 'cwp 6/20/2012 17:22'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!Metaclass methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:29'!
bindingOf: varName environment: anEnvironment 
	^ thisClass classBindingOf: varName environment: anEnvironment! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 20:03' prior: 22505876!
\\ aNumber 
	"Primitive. Take the receiver modulo the argument. The result is the
	remainder rounded towards negative infinity, of the receiver divided
	by the argument. Fail if the argument is 0. Fail if either the argument
	or the result is not a SmallInteger or a LargePositiveInteger less than
	2-to-the-30th (1073741824). Optional. See Object documentation whatIsAPrimitive."

	<primitive: 31>
	aNumber isInteger
		ifTrue:
			[| neg qr q r |
			neg := self negative == aNumber negative == false.
			qr := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: neg).
			q := qr first normalize.
			r := qr last normalize.
			^(q negative
				ifTrue: [r isZero not]
				ifFalse: [q isZero and: [neg]])
					ifTrue: [r + aNumber]
					ifFalse: [r]].
	^super \\ aNumber
	! !
!LargePositiveInteger methodsFor: 'converting' stamp: 'nice 1/27/2012 22:41' prior: 37616324!
asFloat
	"Answer a Float that best approximates the value of the receiver.
	This algorithm is optimized to process only the significant digits of a LargeInteger.
	And it does honour IEEE 754 round to nearest even mode in case of excess precision (see details below)."
	
	"How numbers are rounded in IEEE 754 default rounding mode:
	A shift is applied so that the highest 53 bits are placed before the floating point to form a mantissa.
	The trailing bits form the fraction part placed after the floating point.
	This fractional number must be rounded to the nearest integer.
	If fraction part is 2r0.1, exactly between two consecutive integers, there is a tie.
	The nearest even integer is chosen in this case.
	Examples (First 52bits of mantissa are omitted for brevity):
	2r0.00001 is rounded downward to 2r0
	2r1.00001 is rounded downward to 2r1
	2r0.1 is a tie and rounded to 2r0 (nearest even)
	2r1.1 is a tie and rounded to 2r10 (nearest even)
	2r0.10001 is rounded upward to 2r1
	2r1.10001 is rounded upward to 2r10
	Thus, if the next bit after floating point is 0, the mantissa is left unchanged.
	If next bit after floating point is 1, an odd mantissa is always rounded upper.
	An even mantissa is rounded upper only if the fraction part is not a tie."
	
	"Algorihm details:
	The floating point hardware can perform the rounding correctly with several excess bits as long as there is a single inexact operation.
	This can be obtained by splitting the mantissa plus excess bits in two part with less bits than Float precision.
	Note 1: the inexact flag in floating point hardware must not be trusted because in some cases the operations would be exact but would not take into account some bits that were truncated before the Floating point operations.
	Note 2: the floating point hardware is presumed configured in default rounding mode."
	
	| mantissa shift excess result n |

	"Check how many bits excess the maximum precision of a Float mantissa."
	excess := self highBitOfMagnitude - Float precision.
	excess > 7
		ifTrue:
			["Remove the excess bits but seven."
			mantissa := self bitShiftMagnitude: 7 - excess.
			shift := excess - 7.
			"An even mantissa with a single excess bit immediately following would be truncated.
			But this would not be correct if above shift has truncated some extra bits.
			Check this case, and round excess bits upper manually."
			((mantissa digitAt: 1) = 2r01000000 and: [self anyBitOfMagnitudeFrom: 1 to: shift])
				ifTrue: [mantissa := mantissa + 1]]
		ifFalse:
			[mantissa := self.
			shift := 0].

	"There will be a single inexact round off at last iteration"
	result := (mantissa digitAt: (n := mantissa digitLength)) asFloat.
	[(n := n - 1) > 0] whileTrue: [
		result := 256.0 * result + (mantissa digitAt: n) asFloat].
	^result timesTwoPower: shift.! !
!LargePositiveInteger methodsFor: 'private' stamp: 'nice 12/30/2012 14:25'!
primitiveQuo: anInteger 
	"Primitive. Divide the receiver by the argument and return the result.
	Round the result down towards zero to make it a whole integer. Fail if
	the argument is 0. Fail if either the argument or the result is not a
	SmallInteger or a LargePositiveInteger less than 2-to-the-30th (1073741824). Optional. See
	Object documentation whatIsAPrimitive."

	<primitive: 33>
	^nil! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 14:34'!
rem: aNumber 
	"Remainder defined in terms of quo:. See super rem:.
	This is defined only to speed up case of very large integers."

	(self primitiveQuo: aNumber)
		ifNotNil: [:quo | ^self - (quo * aNumber)].
	 aNumber isInteger
		ifTrue:
			[| ng rem |
			ng := self negative == aNumber negative == false.
			rem := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: ng) at: 2.
			^ rem normalize].
	^super rem: aNumber! !
!LargeNegativeInteger methodsFor: 'converting' stamp: 'nice 1/1/2013 15:42' prior: 37616204!
asFloat
	^super asFloat negated! !
!UndefinedObject methodsFor: 'class hierarchy' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:32'!
bindingOf: varName environment: anEnvironment
	^superclass bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:30'!
classBindingOf: varName environment: anEnvironment
	^self bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:37'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor
	"Postprocesses a literal scanned by Scanner scanToken (esp. xLitQuote).
	If scannedLiteral is not an association, answer it.
	Else, if it is of the form:
		nil->#NameOfMetaclass
	answer nil->theMetaclass, if any has that name, else report an error.
	Else, if it is of the form:
		#NameOfGlobalVariable->anythiEng
	answer the global, class, or pool association with that nameE, if any, else
	add it to Undeclared a answer the new Association."

	| key value |
	(scannedLiteral isVariableBinding)
		ifFalse: [^ scannedLiteral].
	key := scannedLiteral key.
	value := scannedLiteral value.
	key ifNil: "###<metaclass soleInstance name>"
		[(self bindingOf: value environment: anEnvironment) ifNotNil:
			[:assoc|
			(assoc value isKindOf: Behavior) ifTrue: 
				[^ nil->assoc value class]].
			 requestor notify: 'No such metaclass'.
			 ^false].
	(key isSymbol) ifTrue: "##<global var name>"
		[(self bindingOf: key environment: anEnvironment) ifNotNil:
			[:assoc | ^assoc].
		^ anEnvironment undeclared: key].
	requestor notify: '## must be followed by a non-local variable name'.
	^false

"	Form literalScannedAs: 14 notifying: nil 14
	Form literalScannedAs: #OneBitForm notiEfying: nil  OneBitForm
	Form literalScannedAs: ##OneBitForm notifying: nil  OneBitForm->a Form
	Form literalScannedAs: ##Form notifying: nil   Form->Form
	Form literalScannedAs: ###Form notifying: nil   nilE->Form class
"! !
!Fraction methodsFor: 'converting' stamp: 'nice 11/21/2011 22:34' prior: 37619655!
asFloat
	"Answer a Float that closely approximates the value of the receiver.
	This implementation will answer the closest floating point number to the receiver.
	In case of a tie, it will use the IEEE 754 round to nearest even mode.
	In case of overflow, it will answer +/- Float infinity."

	| a b mantissa exponent hasTruncatedBits lostBit n ha hb hm |
	a := numerator abs.
	b := denominator.	"denominator is always positive"
	ha := a highBitOfMagnitude.
	hb := b highBitOfMagnitude.
	
	"Number of bits to keep in mantissa plus one to handle rounding."
	n := 1 + Float precision.

	"If both numerator and denominator are represented exactly in floating point number,
	then fastest thing to do is to use hardwired float division."
	(ha < n and: [hb < n]) ifTrue: [^numerator asFloat / denominator asFloat].

	"Shift the fraction by a power of two exponent so as to obtain a mantissa with n bits.
	First guess is rough, the mantissa might have n+1 bits."
	exponent := ha - hb - n.
	exponent >= 0
		ifTrue: [b := b bitShift: exponent]
		ifFalse: [a := a bitShift: exponent negated].
	mantissa := a quo: b.
	hasTruncatedBits := a > (mantissa * b).
	hm := mantissa highBit.
	
	"Check for gradual underflow, in which case the mantissa will loose bits.
	Keep at least one bit to let underflow preserve the sign of zero."
	lostBit := Float emin - (exponent + hm - 1).
	lostBit > 0 ifTrue: [n := n - lostBit max: 1].

	"Remove excess bits in the mantissa."
	hm > n
		ifTrue:
			[exponent := exponent + hm - n.
			hasTruncatedBits := hasTruncatedBits or: [mantissa anyBitOfMagnitudeFrom: 1 to: hm - n].
			mantissa := mantissa bitShift: n - hm].

	"Check if mantissa must be rounded upward.
	The case of tie (mantissa odd & hasTruncatedBits not)
	will be handled by Integer>>asFloat."
	(hasTruncatedBits and: [mantissa odd])
		ifTrue: [mantissa := mantissa + 1].

	^ (self positive
			ifTrue: [mantissa asFloat]
			ifFalse: [mantissa asFloat negated])
		timesTwoPower: exponent! !
!Float methodsFor: 'arithmetic' stamp: 'nice 12/20/2012 23:16' prior: 20878776!
negated
	"Answer a Number that is the negation of the receiver.
	Implementation note: this version cares of negativeZero."

	^-1.0 * self! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:21'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor
	^ self 
		compile: text 
		environment: anEnvironment 
		classified: category 
		withStamp: changeStamp 
		notifying: requestor 
		logSource: self acceptsLoggingOfCompilation! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 12/27/2012 13:17'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor logSource: logSource
	| methodAndNode context methodNode |
	context := CompilationCue
		source: text
		class: self
		environment: anEnvironment
		category: category
		requestor: requestor.
	methodNode := self newCompiler compile: context ifFail: [^ nil].
	methodAndNode := CompiledMethodWithNode 
		generateMethodFromNode: methodNode 
		trailer: self defaultMethodTrailer.

	logSource ifTrue: [
		self logMethodSource: text forMethodWithNode: methodAndNode 
			inCategory: category withStamp: changeStamp notifying: requestor.
	].
	self addAndClassifySelector: methodAndNode selector withMethod: methodAndNode 
		method inProtocol: category notifying: requestor.
	self instanceSide noteCompilationOf: methodAndNode selector meta: self isClassSide.
	^ methodAndNode selector! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:47'!
bindingOf: varName environment: anEnvironment
	"Answer the binding of some variable resolved in the scope of the receiver"
	| aSymbol binding |
	aSymbol := varName asSymbol.

	"First look in classVar dictionary."
	binding := self classPool bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Next look in shared pools."
	self sharedPools do:[:pool | 
		binding := pool bindingOf: aSymbol.
		binding ifNotNil:[^binding].
	].

	"Next look in declared environment."
	binding := anEnvironment bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Finally look higher up the superclass chain and fail at the end."
	superclass == nil
		ifTrue: [^ nil]
		ifFalse: [^ superclass bindingOf: aSymbol].

! !

"Kernel"!

ParseNode subclass: #Encoder
	instanceVariableNames: 'scopeTable nTemps supered requestor class selector literalStream selectorSet litIndSet litSet sourceRanges globalSourceRanges addedSelectorAndMethodClassLiterals optimizedSelectors cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Encoder commentStamp: 'cwp 12/26/2012 23:29' prior: 36323851!
I encode names and literals into tree nodes with byte codes for the compiler. Byte codes for literals are not assigned until the tree-sizing pass of the compiler, because only then is it known which literals are actually needed. I also keep track of sourceCode ranges during parsing and code generation so I can provide an inverse map for the debugger.!

Scanner subclass: #Parser
	instanceVariableNames: 'here hereType hereMark hereEnd prevMark prevEnd encoder requestor parseNode failBlock requestorOffset tempsMark doitFlag properties category queriedUnusedTemporaries cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Parser commentStamp: 'cwp 12/26/2012 23:34' prior: 38557958!
I parse Smalltalk syntax and create a MethodNode that is the root of the parse tree. I look one token ahead.!

Object subclass: #CompilationCue
	instanceVariableNames: 'source context receiver class environment category requestor'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

Object subclass: #Compiler
	instanceVariableNames: 'sourceStream requestor class category context parser cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Compiler commentStamp: 'cwp 12/26/2012 23:17' prior: 59257505!
The compiler accepts Smalltalk source code and compiles it with respect to a given class. The user of the compiler supplies a context so that temporary variables are accessible during compilation. If there is an error, a requestor (usually a kind of StringHolderController) is sent the message notify:at:in: so that the error message can be displayed. If there is no error, then the result of compilation is a MethodNode, which is the root of a parse tree whose nodes are kinds of ParseNodes. The parse tree can be sent messages to (1) generate code for a CompiledMethod (this is done for compiling methods or evaluating expressions); (2) pretty-print the code (for formatting); or (3) produce a map from object code back to source code (used by debugger program-counter selection). See also Parser, Encoder, ParseNode.!
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:34'!
init: aCue notifying: anObject
	"The use of the variable requestor is a bit confusing here. This is
	*not* the original requestor, which is available through the cue.
	It's the Parser instance that is using the encoder."

	self setCue: aCue.
	requestor := anObject.
	nTemps := 0.
	supered := false.
	self initScopeAndLiteralTables.
	cue getClass variablesAndOffsetsDo:
		[:variable "<String|CFieldDefinition>" :offset "<Integer|nil>" |
		offset isNil
			ifTrue: [scopeTable at: variable name put: (FieldNode new fieldDefinition: variable)]
			ifFalse: [scopeTable
						at: variable
						put: (offset >= 0
								ifTrue: [InstanceVariableNode new
											name: variable index: offset]
								ifFalse: [MaybeContextInstanceVariableNode new
											name: variable index: offset negated])]].
	cue context ~~ nil ifTrue:
		[| homeNode |
		 homeNode := self bindTemp: self doItInContextName.
		 "0th temp = aContext passed as arg"
		 cue context tempNames withIndexDo:
			[:variable :index|
			scopeTable
				at: variable
				put: (MessageAsTempNode new
						receiver: homeNode
						selector: #namedTempAt:
						arguments: (Array with: (self encodeLiteral: index))
						precedence: 3
						from: self)]].
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/26/2012 23:30'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy instance variables for methods that
	don't use cue yet"
	class := cue getClass.! !
!Dictionary methodsFor: '*Compiler' stamp: 'cwp 6/22/2012 09:17'!
bindingOf: varName ifAbsent: aBlock

	^self associationAt: varName ifAbsent: aBlock! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:37'!
init: sourceStream cue: aCue failBlock: aBlock

	self setCue: aCue.
	failBlock := aBlock.
	requestorOffset := 0.
	super scan: sourceStream.
	prevMark := hereMark := mark.
	self advance
! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/26/2012 23:41'!
parse: sourceStream cue: aCue noPattern: noPattern ifFail: aBlock 
	"Answer a MethodNode for the argument, sourceStream, that is the root of
	 a parse tree. Parsing is done with respect to the CompilationCue to 
	 resolve variables. Errors in parsing are reported to the cue's requestor; 
	 otherwise aBlock is evaluated. The argument noPattern is a Boolean that is
	 true if the the sourceStream does not contain a method header (i.e., for DoIts)."

	| methNode repeatNeeded myStream s p subSelection |
	myStream := sourceStream.
	[repeatNeeded := false.
	 p := myStream position.
	 s := myStream upToEnd.
	 myStream position: p.
	 subSelection := aCue requestor notNil and: [aCue requestor selectionInterval = (p + 1 to: p + s size)].
	 self encoder init: aCue notifying: self.
	 self init: myStream cue: aCue failBlock: [^ aBlock value].
	 doitFlag := noPattern.
	 failBlock:= aBlock.
	 [methNode := self method: noPattern context: cue context] 
		on: ReparseAfterSourceEditing 
		do:	[ :ex |
			repeatNeeded := true.
			myStream := subSelection
							ifTrue:
								[ReadStream
									on: cue requestor text string
									from: cue requestor selectionInterval first
									to: cue requestor selectionInterval last]
							ifFalse:
								[ReadStream on: cue requestor text string]].
	 repeatNeeded] whileTrue:
		[encoder := self encoder class new].
	methNode sourceText: s.
	^methNode
! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:35'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy variables for methods that don't use cue yet."
	requestor := cue requestor.
	category := cue category.! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
class: aClass
	^ self 
		context: nil
		class: aClass
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
context: aContext class: aClass requestor: anObject
	^ self
		source: nil
		context: aContext
		receiver: nil
		class: aClass
		environment: (aClass ifNotNil: [aClass environment])
		category: nil
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aTextOrStream class: aClass environment: anEnvironment category: aString requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		receiver: nil
		class: aClass
		environment: anEnvironment
		category: aString
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
source: aTextOrStream context: aContext class: aClass category: aString requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		receiver: (aContext ifNotNil: [aContext receiver])
		class: aClass 
		environment: (aClass ifNotNil: [aClass environment])
		category: aString 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream context: aContext class: aClass requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		class: aClass 
		category: nil 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:55'!
source: aTextOrStream context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	^ self basicNew
		initializeWithSource: aTextOrStream 
		context: aContext 
		receiver: recObject 
		class: aClass 
		environment: anEnvironment 
		category: aString 
		requestor: reqObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aString environment: anEnvironment
	^ self 
		source: aString
		context: nil
		receiver: nil
		class: UndefinedObject
		environment: anEnvironment
		category: nil
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		class: nil
		requestor: anObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/20/2012 09:39'!
bindingOf: aSymbol
	^ class bindingOf: aSymbol environment: environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
category
	^ category! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 12/26/2012 23:19'!
context
	^ context! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
environment
	^ environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
getClass
	^ class! !
!CompilationCue methodsFor: 'initialization' stamp: 'cwp 12/26/2012 23:16'!
initializeWithSource: aTextOrString context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	self initialize.
	source := aTextOrString isStream ifTrue: [aTextOrString contents] ifFalse: [aTextOrString].
	context := aContext.
	receiver := recObject.
	class := aClass.
	environment := anEnvironment.
	category := aString.
	requestor := reqObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: anObject notifying: anEncoder
	^ class literalScannedAs: anObject environment: environment notifying: anEncoder! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
receiver
	^ receiver! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
requestor
	^ requestor! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
source
	^ source! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:44'!
sourceStream
	^ source readStream! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 6/20/2012 17:25'!
evaluate: aString environment: anEnvironment
	^ self 
		evaluate: aString 
		environment: anEnvironment 
		logged: false! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 12/27/2012 12:36'!
evaluate: aString environment: anEnvironment logged: aBoolean
	| cue |
	cue := CompilationCue
		source: aString
		environment: anEnvironment.
		
	^ self new
		evaluate: aString
		cue: cue
		ifFail: [^ nil]
		logged: aBoolean! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 13:18'!
compile: aCue ifFail: failBlock 
	"Answer a MethodNode. If the MethodNode can not be created, notify 
	the requestor in the contxt. If the requestor is nil, evaluate failBlock 
	instead. The MethodNode is the root  of a parse tree. It can be told 
	to generate a CompiledMethod to be installed in the method dictionary 
	of the class specified by the context."
	
	self setCue: aCue.
	self source: cue source.
	^self
		translate: sourceStream
		noPattern: false
		ifFail: failBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:06'!
evaluate: textOrStream cue: aCue ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	a method. Finally, the compiled method is invoked from here via 	withArgs:executeMethod:, hence the system no longer creates Doit method
	litter on errors."

	| methodNode method value toLog itsSelection itsSelectionString |
	self setCue: aCue.
	self source: textOrStream.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].

	method := self interactive
				ifTrue: [methodNode generateWithTempNames]
				ifFalse: [methodNode generate].

	value := cue receiver
				withArgs: (cue context ifNil: [#()] ifNotNil: [{cue context}])
				executeMethod: method.

	logFlag ifTrue:
		[toLog := ((cue requestor respondsTo: #selection)  
			and:[(itsSelection := cue requestor selection) notNil
			and:[(itsSelectionString := itsSelection asString) isEmptyOrNil not]])
				ifTrue:[itsSelectionString]
				ifFalse:[sourceStream contents].
		SystemChangeNotifier uniqueInstance evaluated: toLog context: cue context].
	^ value
! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:20'!
setCue: aCue
	cue := aCue.
	
	"Set legacy instance variables for methods that don't use cue yet."
	requestor := cue requestor.
	class := cue getClass.
	category := cue category.
	context := cue context.! !
!Compiler methodsFor: 'private' stamp: 'cwp 6/19/2012 21:58'!
source: textOrStream
	sourceStream := (textOrStream isKindOf: PositionableStream)
		ifTrue: [ textOrStream ]
		ifFalse: [ ReadStream on: textOrStream asString ]! !

"Compiler"!
!SmartRefStream class methodsFor: 'i/o' stamp: 'cwp 6/20/2012 17:42'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!SmartRefStream methodsFor: 'read write' stamp: 'cwp 6/20/2012 17:41'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!ImageSegment methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:23'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!PseudoClass methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| aString newTranslations assoc currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[Locale
		currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:26'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| newTranslations currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[| aString assoc |
	Locale currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!ObjectScanner methodsFor: 'scanning' stamp: 'cwp 6/20/2012 17:39'!
scanFrom: aByteStream environment: anEnvironment
	"This should probably be reimplemented using an environment
	for compilation. For now, don't change anything"
	^ self scanFrom: aByteStream! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 09:16'!
bindingOf: varName ifAbsent: aBlock
	"SystemDictionary includes Symbols only"
	^super bindingOf: varName asSymbol ifAbsent: aBlock! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 15:48'!
undeclared
	^ self at: #Undeclared! !

"System"!
!ExceptionTests methodsFor: 'testing-outer' stamp: 'fbs 1/1/2013 22:14' prior: 40840955!
expectedFailures
	^ #().! !

"Tests"!

ReleaseBuilder subclass: #ReleaseBuilderFor4dot5
	instanceVariableNames: ''
	classVariableNames: ''
	poolDictionaries: ''
	category: 'ReleaseBuilder'!
!ReleaseBuilderFor4dot5 commentStamp: 'fbs 1/1/2013 20:25' prior: 0!
The release builder for Squeak 4.5!
!ReleaseBuilder class methodsFor: 'scripts' stamp: 'fbs 12/31/2012 20:43'!
transferCurrentPackagesAsUser: username password: password
	"Copy the packages currently loaded in the image from the trunk repository to my releaseRepository."
	| trunkRep releaseRep |
	trunkRep := self trunkRepository.
	releaseRep := self releaseRepository
		user: username;
		password: password;
		yourself.
	MCWorkingCopy allManagers do:
		[ : eachWorkingCopy | eachWorkingCopy ancestors do:
			[ : eachVersionInfo | (releaseRep includesVersionNamed: eachVersionInfo versionName) ifFalse:
				[ (trunkRep versionWithInfo: eachVersionInfo)
					ifNil: [ Warning signal: eachVersionInfo name , ' not found in ', trunkRep ]
					ifNotNilDo: [ : ver | releaseRep storeVersion: ver ] ] ] ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
openWelcomeWorkspaces
	TheWorldMainDockingBar instance
		showWelcomeText: #squeakUserInterface
		label: 'Squeak User Interface'
		in: (40 @ 40 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #workingWithSqueak
		label: 'Working With Squeak'
		in: (80 @ 80 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #licenseInformation
		label: 'License Information'
		in: (120 @ 120 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeFutureDirections
		label: 'Future Directions'
		in: (160 @ 160 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeToSqueak
		label: 'Welcome to Squeak 4.5'
		in: (200 @ 200 extent: 500 @ 300)! !
!ReleaseBuilderFor4dot5 class methodsFor: 'scripts' stamp: 'fbs 1/1/2013 20:22'!
prepareNewBuild
	super prepareNewBuild.
	MCMockPackageInfo initialize.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:24'!
releaseRepository
	"At release time, change 'trunk' to 'squeak45'."
	^ MCHttpRepository
		location: 'http://source.squeak.org/trunk'
		user: 'squeak'
		password: 'squeak'! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:22'!
setDisplayExtent: extent
"Uncomment next line when the primitives become available in the Squeak VM."
"	DisplayScreen hostWindowSize: extent."
	Display extent = extent ifFalse: [ Warning signal: 'Display extent not set to ', extent ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
setPreferences
	Preferences 
		installBrightWindowColors ;
		setPreference: #scrollBarsWithoutMenuButton toValue: true ;
		setPreference: #swapMouseButtons toValue: true ;
		setPreference: #annotationPanes toValue: true ;
		setPreference: #showSplitterHandles toValue: false ;
		setPreference: #showBoundsInHalo toValue: true ;
		setPreference: #alternateHandlesLook toValue: false ;
		setPreference: #roundedMenuCorners toValue: false ;
		setPreference: #roundedWindowCorners toValue: false.
	PluggableButtonMorph roundedButtonCorners: false.
	FillInTheBlankMorph roundedDialogCorners: false.
	Workspace shouldStyle: false.
	NetNameResolver enableIPv6: true.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
switchToNewRepository
	| old44Repository |
	MCMcmUpdater defaultUpdateURL: self releaseRepository description.
	old44Repository := MCRepositoryGroup default repositories
				detect: [:each | each description includesSubString: 'squeak44'] ifNone: [nil].
	old44Repository
		ifNotNil: [MCRepositoryGroup default removeRepository: old44Repository].
	MCRepositoryGroup default addRepository: self releaseRepository! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
versionString
	^ 'Squeak4.5'.! !

ReleaseBuilder class removeSelector: #transferCurrentPackages!

"ReleaseBuilder"!
!Environment class methodsFor: 'as yet unclassified' stamp: 'cwp 1/1/2013 18:52' prior: 40834114!
initialize
	self install! !

"Environments"!
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:59' prior: 52081878!
initPattern: aString notifying: req return: aBlock

	| result |
	self
		init: (ReadStream on: aString asString)
		cue: (CompilationCue source: aString requestor: req)
		failBlock: [^nil].
	encoder := self.
	result := aBlock value: (self pattern: false inContext: nil).
	encoder := failBlock := nil.  "break cycles"
	^result! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:01' prior: 34175471!
parse: sourceStream class: class category: aCategory noPattern: noPattern context: aContext notifying: req ifFail: aBlock 
	| c |
	c := CompilationCue
			source: sourceStream
			context: aContext
			class: class
			category: aCategory
			requestor: req.
	^ self 
		parse: sourceStream 
		cue: c 
		noPattern: noPattern 
		ifFail: aBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:11' prior: 34183963!
evaluate: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	 a method. If aContext is not nil, the text can refer to temporaries in that
	 context (the Debugger uses this). If aRequestor is not nil, then it will receive
	 a notify:at: message before the attempt to evaluate is aborted. Finally, the 
	 compiled method is invoked from here via withArgs:executeMethod:, hence
	 the system no longer creates Doit method litter on errors."
	
	| theClass |
	theClass := ((aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class).
	self setCue: (CompilationCue
		source: textOrStream
		context: aContext
		receiver: receiver
		class: theClass
		environment: theClass environment
		category: nil
		requestor: aRequestor).
	^ self evaluate: textOrStream cue: cue ifFail: failBlock logged: logFlag! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:17' prior: 34185488!
from: textOrStream class: aClass classified: aCategory context: aContext notifying: req
	self source: textOrStream.
	self setCue: 
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			category: aCategory
			requestor: req)! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:55' prior: 50781309!
from: textOrStream class: aClass context: aContext notifying: req
	self source: textOrStream.
	self setCue:
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			requestor: req)
! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/27/2012 09:41' prior: 50996506!
init: aClass context: aContext notifying: anObject
	| c |
	c := CompilationCue
		context: aContext 
		class: aClass 
		requestor: nil.
	self init: c notifying: anObject! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:58' prior: 39061698!
temps: tempVars literals: lits class: cl 
	"Initialize this encoder for decompilation."

	self setCue: (CompilationCue class: cl).
	supered := false.
	nTemps := tempVars size.
	tempVars do: [:node | scopeTable at: node name put: node].
	literalStream := WriteStream on: (Array new: lits size).
	literalStream nextPutAll: lits.
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32.! !

"Compiler"!
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:48' prior: 36026010!
addClassVarName: aString 
	"Add the argument, aString, as a class variable of the receiver.
	Signal an error if the first character of aString is not capitalized,
	or if it is already a variable named in the class."
	| symbol oldState |
	oldState := self copy.
	aString first canBeGlobalVarInitial
		ifFalse: [^self error: aString, ' class variable name should be capitalized; proceed to include anyway.'].
	symbol := aString asSymbol.
	self withAllSubclasses do: 
		[:subclass | 
		(self canFindWithoutEnvironment: symbol) ifTrue: [
			(DuplicateVariableError new)
				superclass: superclass; "fake!!!!!!"
				variable: aString;
				signal: aString, ' is already defined']].
	classPool == nil ifTrue: [classPool := Dictionary new].
	(classPool includesKey: symbol) ifFalse: 
		["Pick up any refs in Undeclared"
		classPool declare: symbol from: environment undeclared.
		SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: oldState to: self]! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:48' prior: 54782024!
bindingOf: varName
	^ self bindingOf: varName environment: self environment! !
!Class methodsFor: 'organization' stamp: 'cwp 6/25/2012 18:25' prior: 54785804!
category
	"Answer the system organization category for the receiver. First check whether the
	category name stored in the ivar is still correct and only if this fails look it up
	(latter is much more expensive)"

	category ifNotNil: [ :symbol |
		((self environment organization listAtCategoryNamed: symbol) includes: self name)
			ifTrue: [ ^symbol ] ].
	category := self environment organization categoryOfElement: self name.
	^category! !
!Class methodsFor: 'initialize-release' stamp: 'cwp 6/22/2012 15:49' prior: 36027730!
declare: varString 
	"Declare class variables common to all instances. Answer whether 
	recompilation is advisable."

	| newVars conflicts |
	
	newVars := 
		(Scanner new scanFieldNames: varString)
			collect: [:x | x asSymbol].
	newVars do:
		[:var | var first canBeGlobalVarInitial
			ifFalse: [self error: var, ' class variable name should be capitalized; proceed to include anyway.']].
	conflicts := false.
	classPool == nil 
		ifFalse: [(classPool keys reject: [:x | newVars includes: x]) do: 
					[:var | self removeClassVarName: var]].
	(newVars reject: [:var | self classPool includesKey: var])
		do: [:var | "adding"
			"check if new vars defined elsewhere"
			(self canFindWithoutEnvironment: var) ifTrue: [
					(DuplicateVariableError new)
						superclass: superclass; "fake!!!!!!"
						variable: var;
						signal: var, ' is already defined'.
					conflicts := true]].
	newVars size > 0
		ifTrue: 
			[classPool := self classPool.
			"in case it was nil"
			newVars do: [:var | classPool declare: var from: environment undeclared]].
	^conflicts! !
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:49' prior: 54802475!
removeClassVarName: aString 
	"Remove the class variable whose name is the argument, aString, from 
	the names defined in the receiver, a class. Create an error notification if 
	aString is not a class variable or if it is still being used in the code of 
	the class."

	| aSymbol |
	aSymbol := aString asSymbol.
	(classPool includesKey: aSymbol)
		ifFalse: [^self error: aString, ' is not a class variable'].
	self withAllSubclasses do:[:subclass |
		(Array with: subclass with: subclass class) do:[:classOrMeta |
			(classOrMeta whichSelectorsReferTo: (classPool associationAt: aSymbol))
				isEmpty ifFalse: [
					InMidstOfFileinNotification signal ifTrue: [
						Transcript cr; show: self name, ' (' , aString , ' is Undeclared) '.
						^ environment undeclared declare: aSymbol from: classPool].
					(self confirm: (aString,' is still used in code of class ', classOrMeta name,
						'.\Is it okay to move it to Undeclared?') withCRs)
						ifTrue:[^Undeclared declare: aSymbol from: classPool]
						ifFalse:[^self]]]].
	classPool removeKey: aSymbol.
	classPool isEmpty ifTrue: [classPool := nil].
! !
!Class methodsFor: 'class name' stamp: 'cwp 6/22/2012 15:49' prior: 54796206!
rename: aString 
	"The new name of the receiver is the argument, aString."

	| oldName newName |
	(newName := aString asSymbol) = (oldName := self name)
		ifTrue: [^ self].
	(self environment includesKey: newName)
		ifTrue: [^ self error: newName , ' already exists'].
	(environment undeclared includesKey: newName)
		ifTrue: [self inform: 'There are references to, ' , aString printString , '
from Undeclared. Check them after this change.'].
	name := newName.
	self environment renameClass: self from: oldName! !
!ClassBuilder methodsFor: 'class definition' stamp: 'cwp 6/22/2012 01:05' prior: 39054430!
name: className inEnvironment: env subclassOf: newSuper type: type instanceVariableNames: instVarString classVariableNames: classVarString poolDictionaries: poolString category: category unsafe: unsafe
	"Define a new class in the given environment.
	If unsafe is true do not run any validation checks.
	This facility is provided to implement important system changes."
	| oldClass instVars classVars copyOfOldClass newClass |
 
	environ := env.
	instVars := Scanner new scanFieldNames: instVarString.
	classVars := (Scanner new scanFieldNames: classVarString) collect: [:x | x asSymbol].

	"Validate the proposed name"
	unsafe ifFalse:[(self validateClassName: className) ifFalse:[^nil]].
	oldClass := env at: className ifAbsent:[nil].
	oldClass isBehavior 
		ifFalse: [oldClass := nil]  "Already checked in #validateClassName:"
		ifTrue: [
			copyOfOldClass := oldClass copy.
			copyOfOldClass superclass addSubclass: copyOfOldClass].
	
	
	[ | newCategory needNew force organization oldCategory |
	unsafe ifFalse:[
		"Run validation checks so we know that we have a good chance for recompilation"
		(self validateSuperclass: newSuper forSubclass: oldClass) ifFalse:[^nil].
		(self validateInstvars: instVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateClassvars: classVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateSubclassFormat: type from: oldClass forSuper: newSuper extra: instVars size) ifFalse:[^nil]].

	"See if we need a new subclass"
	needNew := self needsSubclassOf: newSuper type: type instanceVariables: instVars from: oldClass.
	needNew == nil ifTrue:[^nil]. "some error"

	(needNew and:[unsafe not]) ifTrue:[
		"Make sure we don't redefine any dangerous classes"
		(self tooDangerousClasses includes: oldClass name) ifTrue:[
			self error: oldClass name, ' cannot be changed'.
		].
		"Check if the receiver should not be redefined"
		(oldClass ~~ nil and:[oldClass shouldNotBeRedefined]) ifTrue:[
			self notify: oldClass name asText allBold, 
						' should not be redefined. \Proceed to store over it.' withCRs]].

	needNew ifTrue:[
		"Create the new class"
		newClass := self 
			newSubclassOf: newSuper 
			type: type 
			instanceVariables: instVars
			from: oldClass.
		newClass == nil ifTrue:[^nil]. "Some error"
		newClass setName: className.
		newClass environment: environ.
	] ifFalse:[
		"Reuse the old class"
		newClass := oldClass.
	].

	"Install the class variables and pool dictionaries... "
	force := (newClass declare: classVarString) | (newClass sharing: poolString).

	"... classify ..."
	newCategory := category asSymbol.
	organization := environ ifNotNil:[environ organization].
	oldClass isNil ifFalse: [oldCategory := (organization categoryOfElement: oldClass name) asSymbol].
	organization classify: newClass name under: newCategory suppressIfDefault: true.
	
	"... recompile ..."
	newClass := self recompile: force from: oldClass to: newClass mutate: false.

	"... export if not yet done ..."
	(environ at: newClass name ifAbsent:[nil]) == newClass ifFalse:[
		[environ at: newClass name put: newClass]
			on: AttemptToWriteReadOnlyGlobal do:[:ex| ex resume: true].
		environ flushClassNameCache.
	].


	newClass doneCompiling.
	"... notify interested clients ..."
	oldClass isNil ifTrue: [
		SystemChangeNotifier uniqueInstance classAdded: newClass inCategory: newCategory.
		^ newClass].
	newCategory ~= oldCategory 
		ifTrue: [SystemChangeNotifier uniqueInstance class: newClass recategorizedFrom: oldCategory to: category]
		ifFalse: [SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: copyOfOldClass to: newClass.].
] ensure: 
		[copyOfOldClass ifNotNil: [copyOfOldClass superclass removeSubclass: copyOfOldClass].
		Behavior flushObsoleteSubclasses.
		].
	^newClass! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 22:57' prior: 18572019!
superclass: newSuper
	subclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat 
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class."
	| env |
	env := EnvironmentRequest signal ifNil: [newSuper environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: newSuper
		type: newSuper typeOfClass
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:01' prior: 50629912!
superclass: aClass
	variableByteSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable byte-sized nonpointer variables."
	| oldClassOrNil actualType env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a byte subclass of a class with named fields'].
	(aClass isVariable and: [aClass isWords])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with word fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with pointer fields'].
	oldClassOrNil := aClass environment at: t ifAbsent:[nil].
	actualType := (oldClassOrNil notNil
				   and: [oldClassOrNil typeOfClass == #compiledMethod])
					ifTrue: [#compiledMethod]
					ifFalse: [#bytes].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: actualType
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:03' prior: 18573442!
superclass: aClass
	variableSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable pointer variables."
	
	| env |
	aClass isBits ifTrue: 
		[^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #variable
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18574098!
superclass: aClass
	variableWordSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable word-sized nonpointer variables."
	| env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a word subclass of a class with named fields'].
	(aClass isVariable and: [aClass isBytes])
		ifTrue: [^self error: 'cannot make a word subclass of a class with byte fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a word subclass of a class with pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #words
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18575028!
superclass: aClass
	weakSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class (the receiver) in which the subclass is to 
	have weak indexable pointer variables."
	| env |
	aClass isBits 
		ifTrue: [^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #weak
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !

"Kernel"!
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 59135029!
ambiguousSelector: aString inRange: anInterval
	| correctedSelector userSelection offset intervalWithOffset |
	
	self interactive ifFalse: [
		"In non interactive mode, compile with backward comapatibility: $- is part of literal argument"
		Transcript cr; store: encoder classEncoding; nextPutAll:#'>>';store: encoder selector; show: ' would send ' , token , '-'.
		^super ambiguousSelector: aString inRange: anInterval].
	
	"handle the text selection"
	userSelection := cue requestor selectionInterval.
	intervalWithOffset := anInterval first + requestorOffset to: anInterval last + requestorOffset.
	cue requestor selectFrom: intervalWithOffset first to: intervalWithOffset last.
	cue requestor select.

	"Build the menu with alternatives"
	correctedSelector := AmbiguousSelector 
			signalName: aString
			inRange: intervalWithOffset.
	correctedSelector ifNil: [^self fail].

	"Execute the selected action"
	offset := self substituteWord: correctedSelector wordInterval: intervalWithOffset offset: 0.
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + offset.
	token := (correctedSelector readStream upTo: Character space) asSymbol! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 38558136!
collectTemporaryDeclarationsFrom: methodNode
	| tempsMarks str |
	tempsMarks := OrderedCollection new.
	str := cue requestor text asString.
	methodNode accept: (ParseNodeEnumerator
		ofBlock: [ :aNode | 
			| mark |
			(aNode class canUnderstand: #tempsMark) 
				ifTrue: 
					[mark := aNode tempsMark.
					(mark notNil and: [ mark between: 1 and: str size ] and: [ (str at: mark) = $| ])
						ifTrue: [ tempsMarks addLast: aNode ]]]).
	(tempsMark notNil and: [ tempsMark between: 1 and: str size ] and: [ (str at: tempsMark) = $| ])
						ifTrue: [ tempsMarks addLast: self ].
	^ tempsMarks sorted: [ :a :b | a tempsMark > b tempsMark ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 52096606!
correctSelector: proposedKeyword wordIntervals: spots exprInterval: expInt ifAbort: abortAction
	"Correct the proposedKeyword to some selector symbol, correcting the original text if such action is indicated.  abortAction is invoked if the proposedKeyword couldn't be converted into a valid selector.  Spots is an ordered collection of intervals within the test stream of the for each of the keyword parts."

	| correctSelector userSelection |
	"If we can't ask the user, assume that the keyword will be defined later"
	self interactive ifFalse: [^proposedKeyword asSymbol].

	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spots first first to: spots last last.
	cue requestor select.

	correctSelector := UnknownSelector name: proposedKeyword.
	correctSelector ifNil: [^abortAction value].

	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.

	self substituteSelector: correctSelector keywords wordIntervals: spots.
	^(proposedKeyword last ~~ $:
	   and: [correctSelector last == $:])
		ifTrue: [abortAction value]
		ifFalse: [correctSelector]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 33907242!
correctVariable: proposedVariable interval: spot 
	"Correct the proposedVariable to a known variable, or declare it as a new
	variable if such action is requested.  We support declaring lowercase
	variables as temps or inst-vars, and uppercase variables as Globals or 
	ClassVars, depending on whether the context is nil (class=UndefinedObject).
	Spot is the interval within the test stream of the variable.
	rr 3/4/2004 10:26 : adds the option to define a new class. "

	"Check if this is an i-var, that has been corrected already (ugly)"

	"Display the pop-up menu"

	| binding userSelection action |
	(encoder classEncoding instVarNames includes: proposedVariable) ifTrue: 
		[^InstanceVariableNode new 
			name: proposedVariable
			index: (encoder classEncoding allInstVarNames indexOf: proposedVariable)].

	"If we can't ask the user for correction, make it undeclared"
	self interactive ifFalse: [^encoder undeclared: proposedVariable].

	"First check to see if the requestor knows anything about the variable"
	(binding := cue requestor bindingOf: proposedVariable)
		ifNotNil: [^encoder global: binding name: proposedVariable].
	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spot first to: spot last.
	cue requestor select.

	"Build the menu with alternatives"
	action := UndeclaredVariable 
				signalFor: self
				name: proposedVariable
				inRange: spot.
	action ifNil: [^self fail].

	"Execute the selected action"
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.
	^action value! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:19' prior: 34172921!
declareUndeclaredTemps: methodNode
	"Declare any undeclared temps, declaring them at the smallest enclosing scope."

	| undeclared userSelection blocksToVars |
	(undeclared := encoder undeclaredTemps) isEmpty ifTrue:
		[^self].
	userSelection := cue requestor selectionInterval.
	blocksToVars := IdentityDictionary new.
	undeclared do:
		[:var|
		(blocksToVars
			at: (var tag == #method
					ifTrue: [methodNode block]
					ifFalse: [methodNode accept: (VariableScopeFinder new ofVariable: var)])
			ifAbsentPut: [SortedCollection new]) add: var name].
	(blocksToVars removeKey: methodNode block ifAbsent: []) ifNotNil:
		[:rootVars|
		rootVars do: [:varName| self pasteTempAtMethodLevel: varName]].
	(blocksToVars keys sorted: [:a :b| a tempsMark < b tempsMark]) do:
		[:block| | decl |
		decl := (blocksToVars at: block) reduce: [:a :b| a, ' ', b].
		block temporaries isEmpty
			ifTrue:
				[self substituteWord: ' | ', decl, ' |'
					wordInterval: (block tempsMark + 1 to: block tempsMark)
					offset: requestorOffset]
			ifFalse:
				[self substituteWord: decl, ' '
					wordInterval: (block tempsMark to: block tempsMark - 1)
					offset: requestorOffset]].
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + requestorOffset.
	ReparseAfterSourceEditing signal! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 11:45' prior: 37183770!
defineClass: className 
	"prompts the user to define a new class,  
	asks for it's category, and lets the users edit further  
	the definition"
	| sym cat def d2 |
	sym := className asSymbol.
	cat := UIManager default request: 'Enter class category : ' initialAnswer: self encoder classEncoding theNonMetaClass category.
	cat
		ifEmpty: [cat := 'Unknown'].
	def := 'Object subclass: #' , sym , '
		instanceVariableNames: '''' 
		classVariableNames: ''''
		poolDictionaries: ''''
		category: ''' , cat , ''''.
	d2 := UIManager default request: 'Edit class definition : ' initialAnswer: def.
	d2
		ifEmpty: [d2 := def].
	Compiler evaluate: d2.
	^ encoder
		global: (cue environment bindingOf: sym)
		name: sym! !
!Parser methodsFor: 'primitives' stamp: 'cwp 12/27/2012 11:46' prior: 37184567!
externalFunctionDeclaration
	"Parse the function declaration for a call to an external library."
	| descriptorClass callType modifier retType externalName args argType module fn |
	descriptorClass := cue environment
		valueOf: #ExternalFunction 
		ifAbsent: [^ false].
	callType := descriptorClass callingConventionFor: here.
	callType == nil ifTrue:[^false].
	[modifier := descriptorClass callingConventionModifierFor: token.
	 modifier notNil] whileTrue:
		[self advance.
		 callType := callType bitOr: modifier].
	"Parse return type"
	self advance.
	retType := self externalType: descriptorClass.
	retType == nil ifTrue:[^self expected:'return type'].
	"Parse function name or index"
	externalName := here.
	(self match: #string) 
		ifTrue:[externalName := externalName asSymbol]
		ifFalse:[(self match:#number) ifFalse:[^self expected:'function name or index']].
	(self matchToken: #'(') ifFalse:[^self expected:'argument list'].
	args := WriteStream on: Array new.
	[here == #')'] whileFalse:[
		argType := self externalType: descriptorClass.
		argType == nil ifTrue:[^self expected:'argument'].
		argType isVoid & argType isPointerType not ifFalse:[args nextPut: argType].
	].
	(self matchToken: #')') ifFalse:[^self expected:')'].
	(self matchToken: 'module:') ifTrue:[
		module := here.
		(self match: #string) ifFalse:[^self expected: 'String'].
		module := module asSymbol].
	Smalltalk at: #ExternalLibraryFunction ifPresent:[:xfn|
		fn := xfn name: externalName 
				module: module 
				callType: callType
				returnType: retType
				argumentTypes: args contents.
		self allocateLiteral: fn.
	].
	(self matchToken: 'error:')
		ifTrue:
			[| errorCodeVariable |
			 errorCodeVariable := here.
			(hereType == #string
			 or: [hereType == #word]) ifFalse:[^self expected: 'error code (a variable or string)'].
			 self advance.
			 self addPragma: (Pragma keyword: #primitive:error: arguments: (Array with: 120 with: errorCodeVariable)).
			 fn ifNotNil: [fn setErrorCodeName: errorCodeVariable]]
		ifFalse:
			[self addPragma: (Pragma keyword: #primitive: arguments: #(120))].
	^true
! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:19' prior: 58306169!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:22' prior: 58137223!
notify: string at: location
	cue requestor isNil
		ifTrue: [(encoder == self or: [encoder isNil]) ifTrue: [^ self fail  "failure setting up syntax error"].
				SyntaxErrorNotification
					inClass: encoder classEncoding
					category: cue category
					withCode: 
						(source contents asText
							copyReplaceFrom: location
							to: location - 1
							with: ((string , ' ->') asText allBold 
								addAttribute: TextColor red; yourself))
					doitFlag: doitFlag
					errorMessage: string
					location: location]
		ifFalse: [cue requestor
					notify: string , ' ->'
					at: location
					in: source].
	^self fail! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:17' prior: 34177108!
pasteTempAtMethodLevel: name
	| insertion delta theTextString characterBeforeMark |

	theTextString := cue requestor text string.
	characterBeforeMark := theTextString at: tempsMark-1 ifAbsent: [$ ].
	(theTextString at: tempsMark) = $| ifTrue: [
  		"Paste it before the second vertical bar"
		insertion := name, ' '.
		characterBeforeMark isSeparator ifFalse: [insertion := ' ', insertion].
		delta := 0.
	] ifFalse: [
		"No bars - insert some with CR, tab"
		insertion := '| ' , name , ' |',String cr.
		delta := 2.	"the bar and CR"
		characterBeforeMark = Character tab ifTrue: [
			insertion := insertion , String tab.
			delta := delta + 1.	"the tab"
		].
	].
	tempsMark := tempsMark +
		(self substituteWord: insertion
			wordInterval: (tempsMark to: tempsMark-1)
			offset: 0) - delta! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:16' prior: 52095305!
queryUndefined
	| varStart varName | 
	varName := parseNode key.
	varStart := self endOfLastToken + requestorOffset - varName size + 1.
	cue requestor selectFrom: varStart to: varStart + varName size - 1; select.
	(UndefinedVariable name: varName) ifFalse: [^ self fail]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38599341!
removeEmptyTempDeclarationsFrom: methodNode

	| sourceCode madeChanges tempsMarkHolder |
	sourceCode := cue requestor text asString.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	madeChanges := false.
	tempsMarkHolder do: [ :currentBlock | | tempsMarkChar0 tempsMarkChar1 tempsMarkChar2 end start |
		tempsMarkChar0 := (sourceCode at: currentBlock tempsMark).
		tempsMarkChar1 := (sourceCode at: currentBlock tempsMark - 1).
		tempsMarkChar2 := (sourceCode at: currentBlock tempsMark - 2).
		tempsMarkChar0 = $| & tempsMarkChar1 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 1].
		tempsMarkChar0 = $| & tempsMarkChar1 = $  & tempsMarkChar2 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 2].
		
		start notNil & end notNil ifTrue: [
			| lineStart lineEnd |
			lineStart := 1 + (sourceCode 
				lastIndexOf: Character cr 
				startingAt: start - 1
				ifAbsent: [ 0 ]).
			lineEnd := sourceCode 
				indexOf: Character cr
				startingAt: end + 1
				ifAbsent: [ sourceCode size ].
			((sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: lineStart) >= start 
				and: [ (sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: end + 1) > lineEnd ]) ifTrue: [
					start := lineStart.
					end := lineEnd ].
			cue requestor correctFrom: start to: end with: ''.
			madeChanges := true.
			currentBlock tempsMark: nil ] ].
	madeChanges ifTrue: [ReparseAfterSourceEditing signal]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38561281!
removeUnusedTemporaryNamed: temp from: str lookingAt: currentBlock movingTempMarksOf: someBlocks

	| start end |
	end := currentBlock tempsMark - 1.
	["Beginning at right temp marker..."
	start := end - temp size + 1.
	end < temp size or: [ (str at: start) = $| ]
		or: [ temp = (str copyFrom: start to: end) 
			and: [ ((str at: start - 1) = $| | (str at: start - 1) isSeparator) 
				& ((str at: end + 1) = $| | (str at: end + 1) isSeparator) ] ]]
		whileFalse: [ 
			"Search left for the unused temp"
			end := cue requestor nextTokenFrom: end direction: -1 ].
	(end < temp size or: [ (str at: start) = $| ])
		ifFalse: 
			[(str at: start - 1) = $ 
				ifTrue: [ start := start - 1 ].
			cue requestor correctFrom: start to: end with: ''.
			someBlocks do: [ :aBlock | aBlock tempsMark: aBlock tempsMark - (end - start + 1)].
			^true ].
	^false! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 38562194!
removeUnusedTemps: methodNode
	"Scan for unused temp names, and prompt the user about the prospect of removing each one found"

	| madeChanges tempsMarkHolder unusedTempNames tempMarkHoldersToChange |
	madeChanges := false.
	tempMarkHoldersToChange := OrderedCollection new.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	unusedTempNames := encoder unusedTempNames select: 
		[ :temp | (encoder lookupVariable: temp ifAbsent: [ ]) isUndefTemp 
				and: [ self queriedUnusedTemporaries at: temp ifAbsentPut: [UnusedVariable name: temp] ]].
	tempsMarkHolder do: [ :currentBlock | 
		tempMarkHoldersToChange add: currentBlock.
		unusedTempNames do: 
			[ :temp |
			(self 
				removeUnusedTemporaryNamed: temp 
				from: cue requestor text asString 
				lookingAt: currentBlock
				movingTempMarksOf: tempMarkHoldersToChange) ifTrue: [ madeChanges := true ]]].
	madeChanges
		ifTrue: [ self removeEmptyTempDeclarationsFrom: methodNode.
			ReparseAfterSourceEditing signal ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 34179326!
substituteWord: correctWord wordInterval: spot offset: o
	"Substitute the correctSelector into the (presumed interactive) receiver.
	 Update requestorOffset based on the delta size and answer the updated offset."

	cue requestor correctFrom: spot first + o to: spot last + o with: correctWord.
	requestorOffset := requestorOffset + correctWord size - spot size.
	^o + correctWord size - spot size! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34179807!
temporaries
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'
! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34180638!
temporariesIn: methodSelector
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance in: methodSelector)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:11' prior: 53971863!
compiledMethodFor: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code
	 into a method, and answers it.  If receiver is not nil, then the text can
	 refer to instance variables of that receiver (the Inspector uses this).
	 If aContext is not nil, the text can refer to temporaries in that context
	 (the Debugger uses this). If aRequestor is not nil, then it will receive a 
	 notify:at: message before the attempt to evaluate is aborted."

	| methodNode method theClass |
	theClass := (aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class.
	self from: textOrStream class: theClass context: aContext notifying: aRequestor.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].
	method := self interactive ifTrue: [ 	methodNode generateWithTempNames ] 
		ifFalse: [methodNode generate].
		
	logFlag ifTrue:
		[SystemChangeNotifier uniqueInstance evaluated: sourceStream contents context: aContext].
	^method! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:33' prior: 34363593!
format: aStream noPattern: noPattern ifFail: failBlock
	^(self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]) preen! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 10:08' prior: 58306325!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Compiler methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:10' prior: 50779387!
notify: aString at: location
	"Refer to the comment in Object|notify:."

	^ cue requestor == nil
		ifTrue: [SyntaxErrorNotification
					inClass: cue getClass
					category: cue category
					withCode: 
						(sourceStream contents
							copyReplaceFrom: location
							to: location - 1
							with: aString)
					doitFlag: false
					errorMessage: aString
					location: location]
		ifFalse: [cue requestor
					notify: aString
					at: location
					in: sourceStream]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:34' prior: 50777201!
parse: textOrStream in: aClass notifying: req
	"Compile the argument, textOrStream, with respect to the class, aClass, and
	 answer the MethodNode that is the root of the resulting parse tree.  Notify the
	 argument, req, if an error occurs. The failBlock is defaulted to an empty block."

	self from: textOrStream class: aClass context: nil notifying: req.
	^self parser
		parse: sourceStream
		cue: cue
		noPattern: false
		ifFail: []! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:09' prior: 36332471!
parser

	parser ifNil: [parser := (cue getClass ifNil: [self class]) newParser].
	^parser! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:37' prior: 50780779!
translate: aStream noPattern: noPattern ifFail: failBlock
	^self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:37' prior: 19124095!
translate: aStream noPattern: noPattern ifFail: failBlock parser: parser
	| tree |
	tree := parser
			parse: aStream
			cue: cue 
			noPattern: noPattern
			ifFail: [^ failBlock value].
	^ tree! !
!Encoder methodsFor: 'results' stamp: 'cwp 12/27/2012 10:26' prior: 50999892!
associationForClass
	| assoc |
	assoc := self environment associationAt: cue getClass name ifAbsent: [nil].
	^assoc value == cue getClass
		ifTrue: [assoc]
		ifFalse: [Association new value: cue getClass]! !
!Encoder methodsFor: 'temps' stamp: 'cwp 12/27/2012 10:25' prior: 20148386!
bindTemp: name in: methodSelector
	"Declare a temporary; error not if a field or class variable."
	scopeTable at: name ifPresent:[:node|
		"When non-interactive raise the error only if its a duplicate"
		(node isTemp or:[requestor interactive])
			ifTrue:[^self notify:'Name is already defined']
			ifFalse:[Transcript 
				show: '(', name, ' is shadowed in "' , cue getClass printString , '>>' , methodSelector printString , '")']].
	^self reallyBind: name! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:25' prior: 20149084!
classEncoding
	"This is a hack so that the parser may findout what class it was parsing for when it wants to create a syntax error view."
	^ cue getClass! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:39' prior: 20138819!
encodeLiteral: object

	^self
		name: object
		key: (cue  literalScannedAs: object notifying: self)
		class: LiteralNode
		type: LdLitType
		set: litSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 20139010!
encodeSelector: aSelector

	^self
		name: aSelector
		key: aSelector
		class: SelectorNode
		type: SendType
		set: selectorSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 58545123!
environment
	"Answer the environment of the current compilation context,
	 be it in a class or global (e.g. a workspace)"
	^cue environment! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 11:41' prior: 50994497!
lookupInPools: varName ifFound: assocBlock

	^Symbol
		hasInterned: varName
		ifTrue:
			[:sym|
			(cue bindingOf: sym)
				ifNil: [^false]
				ifNotNil: [:assoc| assocBlock value: assoc]]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 51004306!
possibleNamesFor: proposedName
	| results |
	results := cue getClass 
		possibleVariablesFor: proposedName 
		continuedFrom: nil.
	^ proposedName correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 50995012!
possibleVariablesFor: proposedVariable

	| results |
	results := proposedVariable correctAgainstDictionary: scopeTable
								continuedFrom: nil.
	proposedVariable first canBeGlobalVarInitial ifTrue:
		[ results := cue getClass possibleVariablesFor: proposedVariable
						continuedFrom: results ].
	^ proposedVariable correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:42' prior: 51002830!
undeclared: name
	| sym |
	requestor interactive ifTrue:
		[requestor requestor == #error: ifTrue:
			[requestor error: 'Undeclared'].
		 ^self notify: 'Undeclared'].
	"Allow knowlegeable clients to squash the undeclared warning if they want (e.g.
	 Diffing pretty printers that are simply formatting text).  As this breaks
	 compilation it should only be used by clients that want to discard the result
	 of the compilation.  To squash the warning use e.g.
		[Compiler format: code in: class notifying: nil decorated: false]
			on: UndeclaredVariableWarning
			do: [:ex| ex resume: false]"
	sym := name asSymbol.
	^(UndeclaredVariableWarning new name: name selector: selector class: cue getClass) signal
		ifTrue:
			[| undeclared |
			undeclared := cue environment undeclared.
			undeclared at: sym put: nil.
			self global: (undeclared associationAt: sym) name: sym]
		ifFalse:
			[self global: (Association key: sym) name: sym]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:23' prior: 51006007!
warnAboutShadowed: name
	requestor addWarning: name,' is shadowed'.
	selector ifNotNil:
		[Transcript cr; show: cue getClass name,'>>', selector, '(', name,' is shadowed)']! !

"Compiler"!
!SmalltalkImage methodsFor: 'housekeeping' stamp: 'cwp 6/22/2012 15:56' prior: 58497062!
cleanOutUndeclared 
	globals undeclared removeUnreferencedKeys! !
!SmalltalkImage methodsFor: 'special objects' stamp: 'cwp 6/22/2012 09:01' prior: 40515090!
recreateSpecialObjectsArray
	"Smalltalk recreateSpecialObjectsArray"
	
	"To external package developers:
	**** DO NOT OVERRIDE THIS METHOD.  *****
	If you are writing a plugin and need additional special object(s) for your own use, 
	use addGCRoot() function and use own, separate special objects registry "
	
	"The Special Objects Array is an array of objects used by the Squeak virtual machine.
	 Its contents are critical and accesses to it by the VM are unchecked, so don't even
	 think of playing here unless you know what you are doing."
	| newArray |
	newArray := Array new: 56.
	"Nil false and true get used throughout the interpreter"
	newArray at: 1 put: nil.
	newArray at: 2 put: false.
	newArray at: 3 put: true.
	"This association holds the active process (a ProcessScheduler)"
	newArray at: 4 put: (self bindingOf: #Processor).
	"Numerous classes below used for type checking and instantiation"
	newArray at: 5 put: Bitmap.
	newArray at: 6 put: SmallInteger.
	newArray at: 7 put: ByteString.
	newArray at: 8 put: Array.
	newArray at: 9 put: Smalltalk.
	newArray at: 10 put: Float.
	newArray at: 11 put: MethodContext.
	newArray at: 12 put: BlockContext.
	newArray at: 13 put: Point.
	newArray at: 14 put: LargePositiveInteger.
	newArray at: 15 put: Display.
	newArray at: 16 put: Message.
	newArray at: 17 put: CompiledMethod.
	newArray at: 18 put: (self specialObjectsArray at: 18).
	"(low space Semaphore)"
	newArray at: 19 put: Semaphore.
	newArray at: 20 put: Character.
	newArray at: 21 put: #doesNotUnderstand:.
	newArray at: 22 put: #cannotReturn:.
	newArray at: 23 put: nil. "This is the process signalling low space."
	"An array of the 32 selectors that are compiled as special bytecodes,
	 paired alternately with the number of arguments each takes."
	newArray at: 24 put: #(	#+ 1 #- 1 #< 1 #> 1 #<= 1 #>= 1 #= 1 #~= 1
							#* 1 #/ 1 #\\ 1 #@ 1 #bitShift: 1 #// 1 #bitAnd: 1 #bitOr: 1
							#at: 1 #at:put: 2 #size 0 #next 0 #nextPut: 1 #atEnd 0 #== 1 #class 0
							#blockCopy: 1 #value 0 #value: 1 #do: 1 #new 0 #new: 1 #x 0 #y 0 ).
	"An array of the 255 Characters in ascii order.
	 Cog inlines table into machine code at: prim so do not regenerate it."
	newArray at: 25 put: (self specialObjectsArray at: 25).
	newArray at: 26 put: #mustBeBoolean.
	newArray at: 27 put: ByteArray.
	newArray at: 28 put: Process.
	"An array of up to 31 classes whose instances will have compact headers"
	newArray at: 29 put: self compactClassesArray.
	newArray at: 30 put: (self specialObjectsArray at: 30). "(delay Semaphore)"
	newArray at: 31 put: (self specialObjectsArray at: 31). "(user interrupt Semaphore)"
	"Entries 32 - 34 unreferenced. Previously these contained prototype instances to be copied for fast initialization"
	newArray at: 32 put: nil. "was (Float new: 2)"
	newArray at: 33 put: nil. "was (LargePositiveInteger new: 4)"
	newArray at: 34 put: nil. "was Point new"
	newArray at: 35 put: #cannotInterpret:.
	"Note: This must be fixed once we start using context prototypes (yeah, right)"
	"(MethodContext new: CompiledMethod fullFrameSize)."
	newArray at: 36 put: (self specialObjectsArray at: 36). "Is the prototype MethodContext (unused by the VM)"
	newArray at: 37 put: BlockClosure.
	"(BlockContext new: CompiledMethod fullFrameSize)."
	newArray at: 38 put: (self specialObjectsArray at: 38). "Is the prototype BlockContext (unused by the VM)"
	"array of objects referred to by external code"
	newArray at: 39 put: (self specialObjectsArray at: 39).	"preserve external semaphores"
	newArray at: 40 put: nil. "Reserved for Mutex in Cog VMs"
	newArray at: 41 put: nil. "Reserved for a LinkedList instance for overlapped calls in CogMT"
	"finalization Semaphore"
	newArray at: 42 put: ((self specialObjectsArray at: 42) ifNil: [Semaphore new]).
	newArray at: 43 put: LargeNegativeInteger.
	"External objects for callout.
	 Note: Written so that one can actually completely remove the FFI."
	newArray at: 44 put: (self at: #ExternalAddress ifAbsent: []).
	newArray at: 45 put: (self at: #ExternalStructure ifAbsent: []).
	newArray at: 46 put: (self at: #ExternalData ifAbsent: []).
	newArray at: 47 put: (self at: #ExternalFunction ifAbsent: []).
	newArray at: 48 put: (self at: #ExternalLibrary ifAbsent: []).
	newArray at: 49 put: #aboutToReturn:through:.
	newArray at: 50 put: #run:with:in:.
	"51 reserved for immutability message"
	"newArray at: 51 put: #attemptToAssign:withIndex:."
	newArray at: 52 put: #(nil "nil => generic error" #'bad receiver'
							#'bad argument' #'bad index'
							#'bad number of arguments'
							#'inappropriate operation'  #'unsupported operation'
							#'no modification' #'insufficient object memory'
							#'insufficient C memory' #'not found' #'bad method'
							#'internal error in named primitive machinery'
							#'object may move').
	"53 to 55 are for Alien"
	newArray at: 53 put: (self at: #Alien ifAbsent: []).
	newArray at: 54 put: #invokeCallback:stack:registers:jmpbuf:.
	newArray at: 55 put: (self at: #UnsafeAlien ifAbsent: []).

	"Weak reference finalization"
	newArray at: 56 put: (self at: #WeakFinalizationList ifAbsent: []).

	"Now replace the interpreter's reference in one atomic operation"
	self specialObjectsArray becomeForward: newArray
	! !
!SmalltalkImage methodsFor: 'shrinking' stamp: 'cwp 6/22/2012 15:57' prior: 37288071!
unloadAllKnownPackages
	"Unload all packages we know how to unload and reload"

	"Prepare unloading"
	Smalltalk zapMVCprojects.
	Flaps disableGlobalFlaps: false.
	StandardScriptingSystem removeUnreferencedPlayers.
	Project removeAllButCurrent.
	#('Morphic-UserObjects' 'EToy-UserObjects' 'Morphic-Imported' )
		do: [:each | SystemOrganization removeSystemCategory: each].
	Smalltalk at: #ServiceRegistry ifPresent:[:aClass|
		SystemChangeNotifier uniqueInstance
			noMoreNotificationsFor: aClass.
	].
	World removeAllMorphs.

	"Go unloading"
	#(	'ReleaseBuilder' 'ScriptLoader'
		'311Deprecated' '39Deprecated'
		'Universes' 'SMLoader' 'SMBase' 'Installer-Core'
		'VersionNumberTests' 'VersionNumber'
		'Services-Base' 'PreferenceBrowser' 'Nebraska'
		'ToolBuilder-MVC' 'ST80'
		'CollectionsTests' 'GraphicsTests' 'KernelTests'  'MorphicTests' 
		'MultilingualTests' 'NetworkTests' 'ToolsTests' 'TraitsTests'
		'SystemChangeNotification-Tests' 'FlexibleVocabularies' 
		'EToys' 'Protocols' 'XML-Parser' 'Tests' 'SUnitGUI'
		'Help-Squeak' 'HelpSystem' 'SystemReporter'
	) do: [:pkgName| 
			(MCPackage named: pkgName) unload.
			MCMcmUpdater disableUpdatesOfPackage: pkgName.
			].
	"Traits use custom unload"
	Smalltalk at: #Trait ifPresent:[:aClass| aClass unloadTraits].

	"Post-unload cleanup"
	MCWorkingCopy flushObsoletePackageInfos.
	SystemOrganization removeSystemCategory: 'UserObjects'.
	Presenter defaultPresenterClass: nil.
	World dumpPresenter.
	ScheduledControllers := nil.
	Preferences removePreference: #allowEtoyUserCustomEvents.
	SystemOrganization removeEmptyCategories.
	ChangeSet removeChangeSetsNamedSuchThat:[:cs | (cs == ChangeSet current) not].
	globals undeclared removeUnreferencedKeys.
	StandardScriptingSystem initialize.
	MCFileBasedRepository flushAllCaches.
	MCDefinition clearInstances.
	Behavior flushObsoleteSubclasses.
	ChangeSet current clear.
	ChangeSet current name: 'Unnamed1'.
	Smalltalk flushClassNameCache.
	Smalltalk at: #Browser ifPresent:[:br| br initialize].
	DebuggerMethodMap voidMapCache.
	DataStream initialize.
	AppRegistry removeObsolete.
	FileServices removeObsolete.
	Preferences removeObsolete.
	TheWorldMenu removeObsolete.
	Smalltalk garbageCollect.
	Symbol compactSymbolTable.
	TheWorldMainDockingBar updateInstances.
	MorphicProject defaultFill: (Color gray: 0.9).

From noreply at buildbot.pypy.org  Wed Jun 26 21:09:43 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Jun 2013 21:09:43 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: typo
Message-ID: <20130626190944.036871C021A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r65007:5e9069975ee9
Date: 2013-06-26 21:09 +0200
http://bitbucket.org/pypy/pypy/changeset/5e9069975ee9/

Log:	typo

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1329,7 +1329,7 @@
         fail_locs = [self.loc(v) for v in op.getfailargs()]
         self.assembler.store_force_descr(op, fail_locs,
                                          self.fm.get_frame_depth())
-        self.possibly_free_vars(guard_op.getfailargs())
+        self.possibly_free_vars(op.getfailargs())
 
     def consider_keepalive(self, op):
         pass

From noreply at buildbot.pypy.org  Wed Jun 26 21:19:07 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 21:19:07 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Clean up sweep_pages()
Message-ID: <20130626191907.29A711C021A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r291:6a50b88e3e3a
Date: 2013-06-26 19:17 +0200
http://bitbucket.org/pypy/stmgc/changeset/6a50b88e3e3a/

Log:	Clean up sweep_pages()

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1494,7 +1494,6 @@
           assert(descriptor_array_free_list >= 0);
           assert(pd->stolen_objects.size == 0);
           assert(pd->stolen_young_stubs.size == 0);
-          pd->shutdown = 0;
           /* there may be a thread holding the collection lock
              because it steals a stub belonging to the thread
              that previously owned this descriptor.
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -156,9 +156,10 @@
     /* Allocates an object of the given 'size'.  This will never run
        a collection: you need to call stmgcpage_possibly_major_collect(0)
        when you know you're at a safe point. */
+    struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
+
     if (size <= GC_SMALL_REQUEST_THRESHOLD) {
         gcptr result;
-        struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
         int size_class = (size + WORD - 1) / WORD;
         assert(0 < size_class && size_class < GC_SMALL_REQUESTS);
 
@@ -175,6 +176,7 @@
     else {
         gcptr result = (gcptr)alloc_tracked_memory(size);
         dprintf(("stmgcpage_malloc(BIG %zu): %p\n", size, result));
+        g2l_insert(&gcp->nonsmall_objects, result, result);
         return result;
     }
 }
@@ -186,8 +188,9 @@
 void stmgcpage_free(gcptr obj)
 {
     size_t size = stmgc_size(obj);
+    struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
+
     if (size <= GC_SMALL_REQUEST_THRESHOLD) {
-        struct tx_public_descriptor *gcp = LOCAL_GCPAGES();
         int size_class = (size + WORD - 1) / WORD;
         assert(0 < size_class && size_class < GC_SMALL_REQUESTS);
 
@@ -199,7 +202,8 @@
         //stm_dbgmem_not_used(obj, size_class * WORD, 0);
     }
     else {
-        stm_free(obj, stmgc_size(obj));
+        g2l_delete_item(&gcp->nonsmall_objects, obj);
+        stm_free(obj, size);
     }
 }
 
@@ -475,17 +479,17 @@
 
 /***** Major collections: sweeping *****/
 
-static void sweep_pages(struct tx_public_descriptor *gcp, int size_class,
-                        page_header_t *lpage)
+static void sweep_pages(struct tx_public_descriptor *gcp, int size_class)
 {
     int objs_per_page = nblocks_for_size[size_class];
     revision_t obj_size = size_class * WORD;
-    gcptr freelist = gcp->free_loc_for_size[size_class];
-    page_header_t *lpagenext;
+    gcptr freelist = NULL;
+    page_header_t *lpage, *lpagenext;
+    page_header_t *surviving_pages = NULL;
     int j;
     gcptr p;
 
-    for (; lpage; lpage = lpagenext) {
+    for (lpage = gcp->pages_for_size[size_class]; lpage; lpage = lpagenext) {
         lpagenext = lpage->next_page;
         /* sweep 'page': any object with GCFLAG_VISITED stays alive
            and the flag is removed; other locations are marked as free. */
@@ -497,8 +501,8 @@
         }
         if (j < objs_per_page) {
             /* the page contains at least one object that stays alive */
-            lpage->next_page = gcp->pages_for_size[size_class];
-            gcp->pages_for_size[size_class] = lpage;
+            lpage->next_page = surviving_pages;
+            surviving_pages = lpage;
             p = (gcptr)(lpage + 1);
             for (j = 0; j < objs_per_page; j++) {
                 if (p->h_tid & GCFLAG_VISITED) {
@@ -541,19 +545,15 @@
             count_global_pages--;
         }
     }
+    gcp->pages_for_size[size_class] = surviving_pages;
     gcp->free_loc_for_size[size_class] = freelist;
 }
 
 static void free_unused_local_pages(struct tx_public_descriptor *gcp)
 {
     int i;
-    page_header_t *lpage;
-
     for (i = 1; i < GC_SMALL_REQUESTS; i++) {
-        lpage = gcp->pages_for_size[i];
-        gcp->pages_for_size[i] = NULL;
-        gcp->free_loc_for_size[i] = NULL;
-        sweep_pages(gcp, i, lpage);
+        sweep_pages(gcp, i);
     }
 }
 
@@ -568,20 +568,14 @@
 static void free_closed_thread_descriptors(void)
 {
     int i;
-    page_header_t *gpage;
     struct tx_public_descriptor *gcp;
     revision_t index = -1;
 
     while ((gcp = stm_get_free_public_descriptor(&index)) != NULL) {
-        if (gcp->shutdown)
-            continue;
 
-        for (i = 1; i < GC_SMALL_REQUESTS; i++) {
-            gpage = gcp->pages_for_size[i];
-            sweep_pages(gcp, i, gpage);
-        }
+        free_unused_local_pages(gcp);
+
         assert(gcp->collection_lock == 0);
-        gcp->shutdown = 1;
         assert(gcp->stolen_objects.size == 0);
         assert(gcp->stolen_young_stubs.size == 0);
         gcptrlist_delete(&gcp->stolen_objects);
diff --git a/c4/gcpage.h b/c4/gcpage.h
--- a/c4/gcpage.h
+++ b/c4/gcpage.h
@@ -62,10 +62,10 @@
     /* For statistics */                                                \
     size_t count_pages;                                                 \
                                                                         \
-    /* Set to 1 when a major collection shuts down a                    \
-       public_descriptor after the corresponding thread was             \
-       already finished */                                              \
-    int shutdown;
+    /* A set of all non-small objects (outside the nursery).            \
+       We could also have a single global set, but this avoids          \
+       locking in stmgcpage_malloc/free. */                             \
+    struct G2L nonsmall_objects;
 
 
 #define LOCAL_GCPAGES()  (thread_descriptor->public_descriptor)
diff --git a/c4/lists.c b/c4/lists.c
--- a/c4/lists.c
+++ b/c4/lists.c
@@ -116,6 +116,17 @@
   *(char **)p = (char *)wlog;
 }
 
+void g2l_delete_item(struct G2L *g2l, gcptr addr)
+{
+    wlog_t *entry;
+    G2L_FIND(*g2l, addr, entry, goto missing);
+    g2l_mark_as_deleted(entry);
+    return;
+
+ missing:
+    stm_fatalerror("g2l_delete_item: item %p not in dict", addr);
+}
+
 /************************************************************/
 
 void gcptrlist_delete(struct GcPtrList *gcptrlist)
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -102,6 +102,7 @@
 
 wlog_t *_g2l_find(char *entry, gcptr addr);
 void g2l_insert(struct G2L *g2l, gcptr addr, gcptr val);
+void g2l_delete_item(struct G2L *g2l, gcptr addr);
 
 static inline int g2l_contains(struct G2L *g2l, gcptr addr)
 {
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -424,3 +424,17 @@
     assert p1b == p1
     lib.stm_commit_transaction()
     lib.stm_begin_inevitable_transaction()
+
+def test_big_old_object_collect():
+    p1 = oalloc(HDR + 50 * WORD)
+    lib.stm_push_root(p1)
+    major_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    check_not_free(p1)
+    #
+    major_collect()
+    check_free_old(p1)
+    #
+    major_collect()
+    check_free_old(p1)

From noreply at buildbot.pypy.org  Wed Jun 26 21:19:11 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 21:19:11 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Finish support for big objects
 (missing: LARGE objects, the ones too big for the nursery)
Message-ID: <20130626191911.6714F1C021A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r292:aa1ae03711e1
Date: 2013-06-26 21:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/aa1ae03711e1/

Log:	Finish support for big objects (missing: LARGE objects, the ones too
	big for the nursery)

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -552,9 +552,24 @@
 static void free_unused_local_pages(struct tx_public_descriptor *gcp)
 {
     int i;
+    wlog_t *item;
+
     for (i = 1; i < GC_SMALL_REQUESTS; i++) {
         sweep_pages(gcp, i);
     }
+
+    G2L_LOOP_FORWARD(gcp->nonsmall_objects, item) {
+
+        gcptr p = item->addr;
+        if (p->h_tid & GCFLAG_VISITED) {
+            p->h_tid &= ~GCFLAG_VISITED;
+        }
+        else {
+            g2l_mark_as_deleted(item);
+            stm_free(p, stmgc_size(p));
+        }
+
+    } G2L_LOOP_END;
 }
 
 static void free_all_unused_local_pages(void)
@@ -567,7 +582,6 @@
 
 static void free_closed_thread_descriptors(void)
 {
-    int i;
     struct tx_public_descriptor *gcp;
     revision_t index = -1;
 
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -133,7 +133,8 @@
     int gettid(gcptr obj)
     {
         int result = stm_get_tid(obj);
-        assert(0 <= result && result < 521);
+        assert((0 <= result && result < 2000) ||
+               (42142 <= result && result < 42142 + 2000));
         return result;
     }
 
@@ -144,14 +145,14 @@
 
     gcptr rawgetptr(gcptr obj, long index)
     {
-        assert(gettid(obj) > 421 + index);
+        assert(gettid(obj) > 42142 + index);
         return ((gcptr *)(obj + 1))[index];
     }
 
     void rawsetptr(gcptr obj, long index, gcptr newvalue)
     {
         fprintf(stderr, "%p->[%ld] = %p\n", obj, index, newvalue);
-        assert(gettid(obj) > 421 + index);
+        assert(gettid(obj) > 42142 + index);
         ((gcptr *)(obj + 1))[index] = newvalue;
     }
 
@@ -249,13 +250,13 @@
 
     size_t stmcb_size(gcptr obj)
     {
-        if (gettid(obj) < 421) {
+        if (gettid(obj) < 42142) {
             /* basic case: tid equals 42 plus the size of the object */
             assert(gettid(obj) >= 42 + sizeof(struct stm_object_s));
             return gettid(obj) - 42;
         }
         else {
-            int nrefs = gettid(obj) - 421;
+            int nrefs = gettid(obj) - 42142;
             assert(nrefs < 100);
             return sizeof(*obj) + nrefs * sizeof(gcptr);
         }
@@ -264,11 +265,11 @@
     void stmcb_trace(gcptr obj, void visit(gcptr *))
     {
         int i;
-        if (gettid(obj) < 421) {
+        if (gettid(obj) < 42142) {
             /* basic case: no references */
             return;
         }
-        for (i=0; i < gettid(obj) - 421; i++) {
+        for (i=0; i < gettid(obj) - 42142; i++) {
             gcptr *ref = ((gcptr *)(obj + 1)) + i;
             visit(ref);
         }
@@ -459,7 +460,7 @@
     p = lib.stmgcpage_malloc(HDR + WORD * nrefs)
     p.h_tid = GCFLAG_OLD | GCFLAG_WRITE_BARRIER
     p.h_revision = -sys.maxint
-    lib.settid(p, 421 + nrefs)
+    lib.settid(p, 42142 + nrefs)
     for i in range(nrefs):
         rawsetptr(p, i, ffi.NULL)
     return p
@@ -475,7 +476,7 @@
 
 def nalloc_refs(nrefs):
     "Allocate a fresh object from the nursery, with nrefs pointers"
-    p = lib.stm_allocate(HDR + WORD * nrefs, 421 + nrefs)
+    p = lib.stm_allocate(HDR + WORD * nrefs, 42142 + nrefs)
     assert p.h_revision == lib.get_private_rev_num()
     for i in range(nrefs):
         assert rawgetptr(p, i) == ffi.NULL   # must already be zero-filled
@@ -493,10 +494,10 @@
 def palloc_refs(nrefs, prehash=None):
     "Get a ``prebuilt'' object with nrefs pointers."
     if prehash is None:
-        p = lib.pseudoprebuilt(HDR + WORD * nrefs, 421 + nrefs)
+        p = lib.pseudoprebuilt(HDR + WORD * nrefs, 42142 + nrefs)
     else:
         p = lib.pseudoprebuilt_with_hash(HDR + WORD * nrefs,
-                                         421 + nrefs, prehash)
+                                         42142 + nrefs, prehash)
     return p
 
 gettid = lib.gettid
@@ -518,7 +519,8 @@
 
 def check_not_free(p):
     #print >> sys.stderr, "[checking %r..." % p,
-    assert 42 < (p.h_tid & 0xFFFF) < 521
+    assert ((42 < (p.h_tid & 0xFFFF) < 2000) or
+            (42142 <= (p.h_tid & 0xFFFF) < 42142 + 2000))
     #print >> sys.stderr, "ok]"
 
 def check_nursery_free(p):
@@ -539,7 +541,7 @@
         DEBUG_WORD(0x55), DEBUG_WORD(0xAA))
 
 def check_prebuilt(p):
-    assert 42 < (p.h_tid & 0xFFFF) < 521
+    check_not_free(p)
     assert p.h_tid & GCFLAG_PREBUILT_ORIGINAL
 
 def delegate(p1, p2):
diff --git a/c4/test/test_gcpage.py b/c4/test/test_gcpage.py
--- a/c4/test/test_gcpage.py
+++ b/c4/test/test_gcpage.py
@@ -415,26 +415,29 @@
     check_not_free(p1b)
 
 def test_big_old_object():
-    p1 = oalloc(HDR + 50 * WORD)
+    for words in range(80):
+        p1 = oalloc(HDR + words * WORD)
     # assert did not crash
 
 def test_big_old_object_free():
-    p1 = oalloc(HDR + 50 * WORD)
-    p1b = lib.stm_write_barrier(p1)
-    assert p1b == p1
-    lib.stm_commit_transaction()
-    lib.stm_begin_inevitable_transaction()
+    for words in range(80):
+        p1 = oalloc(HDR + words * WORD)
+        p1b = lib.stm_write_barrier(p1)
+        assert p1b == p1
+        lib.stm_commit_transaction()
+        lib.stm_begin_inevitable_transaction()
 
 def test_big_old_object_collect():
-    p1 = oalloc(HDR + 50 * WORD)
-    lib.stm_push_root(p1)
-    major_collect()
-    p1b = lib.stm_pop_root()
-    assert p1b == p1
-    check_not_free(p1)
-    #
-    major_collect()
-    check_free_old(p1)
-    #
-    major_collect()
-    check_free_old(p1)
+    for words in range(80):
+        p1 = oalloc(HDR + words * WORD)
+        lib.stm_push_root(p1)
+        major_collect()
+        p1b = lib.stm_pop_root()
+        assert p1b == p1
+        check_not_free(p1)
+        #
+        major_collect()
+        check_free_old(p1)
+        #
+        major_collect()
+        check_free_old(p1)

From noreply at buildbot.pypy.org  Wed Jun 26 21:48:39 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Wed, 26 Jun 2013 21:48:39 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: readded cog/stackvm and the
	shootout-tests
Message-ID: <20130626194839.AB2121C021A@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r485:9cf4ae1ceb86
Date: 2013-06-26 21:47 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/9cf4ae1ceb86/

Log:	readded cog/stackvm and the shootout-tests

diff too long, truncating to 2000 out of 80615 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -229,7 +229,7 @@
 
 if __name__ == "__main__":
     try:
-        for project in [RSqueakVM]:
+        for project in [Cog, RSqueakVM]:
             project.post_results()
     finally:
         subprocess.Popen(["rm", '-r', "stackvm"])
diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
--- a/images/Squeak4.5-12568.changes
+++ b/images/Squeak4.5-12568.changes
@@ -36,4 +36,4 @@
 Workspace allInstances do: [:w | w topView delete].
 ReleaseBuilderFor4dot4 prepareNewBuild.
 Smalltalk snapshot: true andQuit: true.
-!

----End fileIn of a stream----!

----SNAPSHOT----{31 March 2013 . 3:27:34 pm} Squeak4.5-12327.image priorSource: 7430688!
!Installer methodsFor: 'squeakmap' stamp: 'fbs 1/28/2013 19:25' prior: 57597950!
packageAndVersionFrom: pkg
	| p |
	p := ReadStream on: pkg .
	^{(p upTo: $(). p upTo: $)} collect: [:s | s withBlanksTrimmed].! !

"Installer-Core"!
!Categorizer methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 16:58'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!ClassCategoryReader methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:21'!
scanFrom: aStream environment: anEnvironment
	"File in methods from the stream, aStream."
	| methodText |
	[methodText := aStream nextChunkText.
	 methodText size > 0] whileTrue:
		[class 
			compile: methodText 
			environment: anEnvironment
			classified: category
			withStamp: changeStamp 
			notifying: nil]! !
!ClassCommentReader methodsFor: 'as yet unclassified' stamp: 'cwp 6/20/2012 17:22'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!Metaclass methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:29'!
bindingOf: varName environment: anEnvironment 
	^ thisClass classBindingOf: varName environment: anEnvironment! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 20:03' prior: 22505876!
\\ aNumber 
	"Primitive. Take the receiver modulo the argument. The result is the
	remainder rounded towards negative infinity, of the receiver divided
	by the argument. Fail if the argument is 0. Fail if either the argument
	or the result is not a SmallInteger or a LargePositiveInteger less than
	2-to-the-30th (1073741824). Optional. See Object documentation whatIsAPrimitive."

	<primitive: 31>
	aNumber isInteger
		ifTrue:
			[| neg qr q r |
			neg := self negative == aNumber negative == false.
			qr := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: neg).
			q := qr first normalize.
			r := qr last normalize.
			^(q negative
				ifTrue: [r isZero not]
				ifFalse: [q isZero and: [neg]])
					ifTrue: [r + aNumber]
					ifFalse: [r]].
	^super \\ aNumber
	! !
!LargePositiveInteger methodsFor: 'converting' stamp: 'nice 1/27/2012 22:41' prior: 37616324!
asFloat
	"Answer a Float that best approximates the value of the receiver.
	This algorithm is optimized to process only the significant digits of a LargeInteger.
	And it does honour IEEE 754 round to nearest even mode in case of excess precision (see details below)."
	
	"How numbers are rounded in IEEE 754 default rounding mode:
	A shift is applied so that the highest 53 bits are placed before the floating point to form a mantissa.
	The trailing bits form the fraction part placed after the floating point.
	This fractional number must be rounded to the nearest integer.
	If fraction part is 2r0.1, exactly between two consecutive integers, there is a tie.
	The nearest even integer is chosen in this case.
	Examples (First 52bits of mantissa are omitted for brevity):
	2r0.00001 is rounded downward to 2r0
	2r1.00001 is rounded downward to 2r1
	2r0.1 is a tie and rounded to 2r0 (nearest even)
	2r1.1 is a tie and rounded to 2r10 (nearest even)
	2r0.10001 is rounded upward to 2r1
	2r1.10001 is rounded upward to 2r10
	Thus, if the next bit after floating point is 0, the mantissa is left unchanged.
	If next bit after floating point is 1, an odd mantissa is always rounded upper.
	An even mantissa is rounded upper only if the fraction part is not a tie."
	
	"Algorihm details:
	The floating point hardware can perform the rounding correctly with several excess bits as long as there is a single inexact operation.
	This can be obtained by splitting the mantissa plus excess bits in two part with less bits than Float precision.
	Note 1: the inexact flag in floating point hardware must not be trusted because in some cases the operations would be exact but would not take into account some bits that were truncated before the Floating point operations.
	Note 2: the floating point hardware is presumed configured in default rounding mode."
	
	| mantissa shift excess result n |

	"Check how many bits excess the maximum precision of a Float mantissa."
	excess := self highBitOfMagnitude - Float precision.
	excess > 7
		ifTrue:
			["Remove the excess bits but seven."
			mantissa := self bitShiftMagnitude: 7 - excess.
			shift := excess - 7.
			"An even mantissa with a single excess bit immediately following would be truncated.
			But this would not be correct if above shift has truncated some extra bits.
			Check this case, and round excess bits upper manually."
			((mantissa digitAt: 1) = 2r01000000 and: [self anyBitOfMagnitudeFrom: 1 to: shift])
				ifTrue: [mantissa := mantissa + 1]]
		ifFalse:
			[mantissa := self.
			shift := 0].

	"There will be a single inexact round off at last iteration"
	result := (mantissa digitAt: (n := mantissa digitLength)) asFloat.
	[(n := n - 1) > 0] whileTrue: [
		result := 256.0 * result + (mantissa digitAt: n) asFloat].
	^result timesTwoPower: shift.! !
!LargePositiveInteger methodsFor: 'private' stamp: 'nice 12/30/2012 14:25'!
primitiveQuo: anInteger 
	"Primitive. Divide the receiver by the argument and return the result.
	Round the result down towards zero to make it a whole integer. Fail if
	the argument is 0. Fail if either the argument or the result is not a
	SmallInteger or a LargePositiveInteger less than 2-to-the-30th (1073741824). Optional. See
	Object documentation whatIsAPrimitive."

	<primitive: 33>
	^nil! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 14:34'!
rem: aNumber 
	"Remainder defined in terms of quo:. See super rem:.
	This is defined only to speed up case of very large integers."

	(self primitiveQuo: aNumber)
		ifNotNil: [:quo | ^self - (quo * aNumber)].
	 aNumber isInteger
		ifTrue:
			[| ng rem |
			ng := self negative == aNumber negative == false.
			rem := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: ng) at: 2.
			^ rem normalize].
	^super rem: aNumber! !
!LargeNegativeInteger methodsFor: 'converting' stamp: 'nice 1/1/2013 15:42' prior: 37616204!
asFloat
	^super asFloat negated! !
!UndefinedObject methodsFor: 'class hierarchy' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:32'!
bindingOf: varName environment: anEnvironment
	^superclass bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:30'!
classBindingOf: varName environment: anEnvironment
	^self bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:37'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor
	"Postprocesses a literal scanned by Scanner scanToken (esp. xLitQuote).
	If scannedLiteral is not an association, answer it.
	Else, if it is of the form:
		nil->#NameOfMetaclass
	answer nil->theMetaclass, if any has that name, else report an error.
	Else, if it is of the form:
		#NameOfGlobalVariable->anythiEng
	answer the global, class, or pool association with that nameE, if any, else
	add it to Undeclared a answer the new Association."

	| key value |
	(scannedLiteral isVariableBinding)
		ifFalse: [^ scannedLiteral].
	key := scannedLiteral key.
	value := scannedLiteral value.
	key ifNil: "###<metaclass soleInstance name>"
		[(self bindingOf: value environment: anEnvironment) ifNotNil:
			[:assoc|
			(assoc value isKindOf: Behavior) ifTrue: 
				[^ nil->assoc value class]].
			 requestor notify: 'No such metaclass'.
			 ^false].
	(key isSymbol) ifTrue: "##<global var name>"
		[(self bindingOf: key environment: anEnvironment) ifNotNil:
			[:assoc | ^assoc].
		^ anEnvironment undeclared: key].
	requestor notify: '## must be followed by a non-local variable name'.
	^false

"	Form literalScannedAs: 14 notifying: nil 14
	Form literalScannedAs: #OneBitForm notiEfying: nil  OneBitForm
	Form literalScannedAs: ##OneBitForm notifying: nil  OneBitForm->a Form
	Form literalScannedAs: ##Form notifying: nil   Form->Form
	Form literalScannedAs: ###Form notifying: nil   nilE->Form class
"! !
!Fraction methodsFor: 'converting' stamp: 'nice 11/21/2011 22:34' prior: 37619655!
asFloat
	"Answer a Float that closely approximates the value of the receiver.
	This implementation will answer the closest floating point number to the receiver.
	In case of a tie, it will use the IEEE 754 round to nearest even mode.
	In case of overflow, it will answer +/- Float infinity."

	| a b mantissa exponent hasTruncatedBits lostBit n ha hb hm |
	a := numerator abs.
	b := denominator.	"denominator is always positive"
	ha := a highBitOfMagnitude.
	hb := b highBitOfMagnitude.
	
	"Number of bits to keep in mantissa plus one to handle rounding."
	n := 1 + Float precision.

	"If both numerator and denominator are represented exactly in floating point number,
	then fastest thing to do is to use hardwired float division."
	(ha < n and: [hb < n]) ifTrue: [^numerator asFloat / denominator asFloat].

	"Shift the fraction by a power of two exponent so as to obtain a mantissa with n bits.
	First guess is rough, the mantissa might have n+1 bits."
	exponent := ha - hb - n.
	exponent >= 0
		ifTrue: [b := b bitShift: exponent]
		ifFalse: [a := a bitShift: exponent negated].
	mantissa := a quo: b.
	hasTruncatedBits := a > (mantissa * b).
	hm := mantissa highBit.
	
	"Check for gradual underflow, in which case the mantissa will loose bits.
	Keep at least one bit to let underflow preserve the sign of zero."
	lostBit := Float emin - (exponent + hm - 1).
	lostBit > 0 ifTrue: [n := n - lostBit max: 1].

	"Remove excess bits in the mantissa."
	hm > n
		ifTrue:
			[exponent := exponent + hm - n.
			hasTruncatedBits := hasTruncatedBits or: [mantissa anyBitOfMagnitudeFrom: 1 to: hm - n].
			mantissa := mantissa bitShift: n - hm].

	"Check if mantissa must be rounded upward.
	The case of tie (mantissa odd & hasTruncatedBits not)
	will be handled by Integer>>asFloat."
	(hasTruncatedBits and: [mantissa odd])
		ifTrue: [mantissa := mantissa + 1].

	^ (self positive
			ifTrue: [mantissa asFloat]
			ifFalse: [mantissa asFloat negated])
		timesTwoPower: exponent! !
!Float methodsFor: 'arithmetic' stamp: 'nice 12/20/2012 23:16' prior: 20878776!
negated
	"Answer a Number that is the negation of the receiver.
	Implementation note: this version cares of negativeZero."

	^-1.0 * self! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:21'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor
	^ self 
		compile: text 
		environment: anEnvironment 
		classified: category 
		withStamp: changeStamp 
		notifying: requestor 
		logSource: self acceptsLoggingOfCompilation! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 12/27/2012 13:17'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor logSource: logSource
	| methodAndNode context methodNode |
	context := CompilationCue
		source: text
		class: self
		environment: anEnvironment
		category: category
		requestor: requestor.
	methodNode := self newCompiler compile: context ifFail: [^ nil].
	methodAndNode := CompiledMethodWithNode 
		generateMethodFromNode: methodNode 
		trailer: self defaultMethodTrailer.

	logSource ifTrue: [
		self logMethodSource: text forMethodWithNode: methodAndNode 
			inCategory: category withStamp: changeStamp notifying: requestor.
	].
	self addAndClassifySelector: methodAndNode selector withMethod: methodAndNode 
		method inProtocol: category notifying: requestor.
	self instanceSide noteCompilationOf: methodAndNode selector meta: self isClassSide.
	^ methodAndNode selector! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:47'!
bindingOf: varName environment: anEnvironment
	"Answer the binding of some variable resolved in the scope of the receiver"
	| aSymbol binding |
	aSymbol := varName asSymbol.

	"First look in classVar dictionary."
	binding := self classPool bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Next look in shared pools."
	self sharedPools do:[:pool | 
		binding := pool bindingOf: aSymbol.
		binding ifNotNil:[^binding].
	].

	"Next look in declared environment."
	binding := anEnvironment bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Finally look higher up the superclass chain and fail at the end."
	superclass == nil
		ifTrue: [^ nil]
		ifFalse: [^ superclass bindingOf: aSymbol].

! !

"Kernel"!

ParseNode subclass: #Encoder
	instanceVariableNames: 'scopeTable nTemps supered requestor class selector literalStream selectorSet litIndSet litSet sourceRanges globalSourceRanges addedSelectorAndMethodClassLiterals optimizedSelectors cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Encoder commentStamp: 'cwp 12/26/2012 23:29' prior: 36323851!
I encode names and literals into tree nodes with byte codes for the compiler. Byte codes for literals are not assigned until the tree-sizing pass of the compiler, because only then is it known which literals are actually needed. I also keep track of sourceCode ranges during parsing and code generation so I can provide an inverse map for the debugger.!

Scanner subclass: #Parser
	instanceVariableNames: 'here hereType hereMark hereEnd prevMark prevEnd encoder requestor parseNode failBlock requestorOffset tempsMark doitFlag properties category queriedUnusedTemporaries cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Parser commentStamp: 'cwp 12/26/2012 23:34' prior: 38557958!
I parse Smalltalk syntax and create a MethodNode that is the root of the parse tree. I look one token ahead.!

Object subclass: #CompilationCue
	instanceVariableNames: 'source context receiver class environment category requestor'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

Object subclass: #Compiler
	instanceVariableNames: 'sourceStream requestor class category context parser cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Compiler commentStamp: 'cwp 12/26/2012 23:17' prior: 59257505!
The compiler accepts Smalltalk source code and compiles it with respect to a given class. The user of the compiler supplies a context so that temporary variables are accessible during compilation. If there is an error, a requestor (usually a kind of StringHolderController) is sent the message notify:at:in: so that the error message can be displayed. If there is no error, then the result of compilation is a MethodNode, which is the root of a parse tree whose nodes are kinds of ParseNodes. The parse tree can be sent messages to (1) generate code for a CompiledMethod (this is done for compiling methods or evaluating expressions); (2) pretty-print the code (for formatting); or (3) produce a map from object code back to source code (used by debugger program-counter selection). See also Parser, Encoder, ParseNode.!
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:34'!
init: aCue notifying: anObject
	"The use of the variable requestor is a bit confusing here. This is
	*not* the original requestor, which is available through the cue.
	It's the Parser instance that is using the encoder."

	self setCue: aCue.
	requestor := anObject.
	nTemps := 0.
	supered := false.
	self initScopeAndLiteralTables.
	cue getClass variablesAndOffsetsDo:
		[:variable "<String|CFieldDefinition>" :offset "<Integer|nil>" |
		offset isNil
			ifTrue: [scopeTable at: variable name put: (FieldNode new fieldDefinition: variable)]
			ifFalse: [scopeTable
						at: variable
						put: (offset >= 0
								ifTrue: [InstanceVariableNode new
											name: variable index: offset]
								ifFalse: [MaybeContextInstanceVariableNode new
											name: variable index: offset negated])]].
	cue context ~~ nil ifTrue:
		[| homeNode |
		 homeNode := self bindTemp: self doItInContextName.
		 "0th temp = aContext passed as arg"
		 cue context tempNames withIndexDo:
			[:variable :index|
			scopeTable
				at: variable
				put: (MessageAsTempNode new
						receiver: homeNode
						selector: #namedTempAt:
						arguments: (Array with: (self encodeLiteral: index))
						precedence: 3
						from: self)]].
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/26/2012 23:30'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy instance variables for methods that
	don't use cue yet"
	class := cue getClass.! !
!Dictionary methodsFor: '*Compiler' stamp: 'cwp 6/22/2012 09:17'!
bindingOf: varName ifAbsent: aBlock

	^self associationAt: varName ifAbsent: aBlock! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:37'!
init: sourceStream cue: aCue failBlock: aBlock

	self setCue: aCue.
	failBlock := aBlock.
	requestorOffset := 0.
	super scan: sourceStream.
	prevMark := hereMark := mark.
	self advance
! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/26/2012 23:41'!
parse: sourceStream cue: aCue noPattern: noPattern ifFail: aBlock 
	"Answer a MethodNode for the argument, sourceStream, that is the root of
	 a parse tree. Parsing is done with respect to the CompilationCue to 
	 resolve variables. Errors in parsing are reported to the cue's requestor; 
	 otherwise aBlock is evaluated. The argument noPattern is a Boolean that is
	 true if the the sourceStream does not contain a method header (i.e., for DoIts)."

	| methNode repeatNeeded myStream s p subSelection |
	myStream := sourceStream.
	[repeatNeeded := false.
	 p := myStream position.
	 s := myStream upToEnd.
	 myStream position: p.
	 subSelection := aCue requestor notNil and: [aCue requestor selectionInterval = (p + 1 to: p + s size)].
	 self encoder init: aCue notifying: self.
	 self init: myStream cue: aCue failBlock: [^ aBlock value].
	 doitFlag := noPattern.
	 failBlock:= aBlock.
	 [methNode := self method: noPattern context: cue context] 
		on: ReparseAfterSourceEditing 
		do:	[ :ex |
			repeatNeeded := true.
			myStream := subSelection
							ifTrue:
								[ReadStream
									on: cue requestor text string
									from: cue requestor selectionInterval first
									to: cue requestor selectionInterval last]
							ifFalse:
								[ReadStream on: cue requestor text string]].
	 repeatNeeded] whileTrue:
		[encoder := self encoder class new].
	methNode sourceText: s.
	^methNode
! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:35'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy variables for methods that don't use cue yet."
	requestor := cue requestor.
	category := cue category.! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
class: aClass
	^ self 
		context: nil
		class: aClass
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
context: aContext class: aClass requestor: anObject
	^ self
		source: nil
		context: aContext
		receiver: nil
		class: aClass
		environment: (aClass ifNotNil: [aClass environment])
		category: nil
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aTextOrStream class: aClass environment: anEnvironment category: aString requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		receiver: nil
		class: aClass
		environment: anEnvironment
		category: aString
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
source: aTextOrStream context: aContext class: aClass category: aString requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		receiver: (aContext ifNotNil: [aContext receiver])
		class: aClass 
		environment: (aClass ifNotNil: [aClass environment])
		category: aString 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream context: aContext class: aClass requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		class: aClass 
		category: nil 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:55'!
source: aTextOrStream context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	^ self basicNew
		initializeWithSource: aTextOrStream 
		context: aContext 
		receiver: recObject 
		class: aClass 
		environment: anEnvironment 
		category: aString 
		requestor: reqObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aString environment: anEnvironment
	^ self 
		source: aString
		context: nil
		receiver: nil
		class: UndefinedObject
		environment: anEnvironment
		category: nil
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		class: nil
		requestor: anObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/20/2012 09:39'!
bindingOf: aSymbol
	^ class bindingOf: aSymbol environment: environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
category
	^ category! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 12/26/2012 23:19'!
context
	^ context! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
environment
	^ environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
getClass
	^ class! !
!CompilationCue methodsFor: 'initialization' stamp: 'cwp 12/26/2012 23:16'!
initializeWithSource: aTextOrString context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	self initialize.
	source := aTextOrString isStream ifTrue: [aTextOrString contents] ifFalse: [aTextOrString].
	context := aContext.
	receiver := recObject.
	class := aClass.
	environment := anEnvironment.
	category := aString.
	requestor := reqObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: anObject notifying: anEncoder
	^ class literalScannedAs: anObject environment: environment notifying: anEncoder! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
receiver
	^ receiver! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
requestor
	^ requestor! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
source
	^ source! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:44'!
sourceStream
	^ source readStream! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 6/20/2012 17:25'!
evaluate: aString environment: anEnvironment
	^ self 
		evaluate: aString 
		environment: anEnvironment 
		logged: false! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 12/27/2012 12:36'!
evaluate: aString environment: anEnvironment logged: aBoolean
	| cue |
	cue := CompilationCue
		source: aString
		environment: anEnvironment.
		
	^ self new
		evaluate: aString
		cue: cue
		ifFail: [^ nil]
		logged: aBoolean! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 13:18'!
compile: aCue ifFail: failBlock 
	"Answer a MethodNode. If the MethodNode can not be created, notify 
	the requestor in the contxt. If the requestor is nil, evaluate failBlock 
	instead. The MethodNode is the root  of a parse tree. It can be told 
	to generate a CompiledMethod to be installed in the method dictionary 
	of the class specified by the context."
	
	self setCue: aCue.
	self source: cue source.
	^self
		translate: sourceStream
		noPattern: false
		ifFail: failBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:06'!
evaluate: textOrStream cue: aCue ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	a method. Finally, the compiled method is invoked from here via 	withArgs:executeMethod:, hence the system no longer creates Doit method
	litter on errors."

	| methodNode method value toLog itsSelection itsSelectionString |
	self setCue: aCue.
	self source: textOrStream.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].

	method := self interactive
				ifTrue: [methodNode generateWithTempNames]
				ifFalse: [methodNode generate].

	value := cue receiver
				withArgs: (cue context ifNil: [#()] ifNotNil: [{cue context}])
				executeMethod: method.

	logFlag ifTrue:
		[toLog := ((cue requestor respondsTo: #selection)  
			and:[(itsSelection := cue requestor selection) notNil
			and:[(itsSelectionString := itsSelection asString) isEmptyOrNil not]])
				ifTrue:[itsSelectionString]
				ifFalse:[sourceStream contents].
		SystemChangeNotifier uniqueInstance evaluated: toLog context: cue context].
	^ value
! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:20'!
setCue: aCue
	cue := aCue.
	
	"Set legacy instance variables for methods that don't use cue yet."
	requestor := cue requestor.
	class := cue getClass.
	category := cue category.
	context := cue context.! !
!Compiler methodsFor: 'private' stamp: 'cwp 6/19/2012 21:58'!
source: textOrStream
	sourceStream := (textOrStream isKindOf: PositionableStream)
		ifTrue: [ textOrStream ]
		ifFalse: [ ReadStream on: textOrStream asString ]! !

"Compiler"!
!SmartRefStream class methodsFor: 'i/o' stamp: 'cwp 6/20/2012 17:42'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!SmartRefStream methodsFor: 'read write' stamp: 'cwp 6/20/2012 17:41'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!ImageSegment methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:23'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!PseudoClass methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| aString newTranslations assoc currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[Locale
		currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:26'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| newTranslations currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[| aString assoc |
	Locale currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!ObjectScanner methodsFor: 'scanning' stamp: 'cwp 6/20/2012 17:39'!
scanFrom: aByteStream environment: anEnvironment
	"This should probably be reimplemented using an environment
	for compilation. For now, don't change anything"
	^ self scanFrom: aByteStream! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 09:16'!
bindingOf: varName ifAbsent: aBlock
	"SystemDictionary includes Symbols only"
	^super bindingOf: varName asSymbol ifAbsent: aBlock! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 15:48'!
undeclared
	^ self at: #Undeclared! !

"System"!
!ExceptionTests methodsFor: 'testing-outer' stamp: 'fbs 1/1/2013 22:14' prior: 40840955!
expectedFailures
	^ #().! !

"Tests"!

ReleaseBuilder subclass: #ReleaseBuilderFor4dot5
	instanceVariableNames: ''
	classVariableNames: ''
	poolDictionaries: ''
	category: 'ReleaseBuilder'!
!ReleaseBuilderFor4dot5 commentStamp: 'fbs 1/1/2013 20:25' prior: 0!
The release builder for Squeak 4.5!
!ReleaseBuilder class methodsFor: 'scripts' stamp: 'fbs 12/31/2012 20:43'!
transferCurrentPackagesAsUser: username password: password
	"Copy the packages currently loaded in the image from the trunk repository to my releaseRepository."
	| trunkRep releaseRep |
	trunkRep := self trunkRepository.
	releaseRep := self releaseRepository
		user: username;
		password: password;
		yourself.
	MCWorkingCopy allManagers do:
		[ : eachWorkingCopy | eachWorkingCopy ancestors do:
			[ : eachVersionInfo | (releaseRep includesVersionNamed: eachVersionInfo versionName) ifFalse:
				[ (trunkRep versionWithInfo: eachVersionInfo)
					ifNil: [ Warning signal: eachVersionInfo name , ' not found in ', trunkRep ]
					ifNotNilDo: [ : ver | releaseRep storeVersion: ver ] ] ] ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
openWelcomeWorkspaces
	TheWorldMainDockingBar instance
		showWelcomeText: #squeakUserInterface
		label: 'Squeak User Interface'
		in: (40 @ 40 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #workingWithSqueak
		label: 'Working With Squeak'
		in: (80 @ 80 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #licenseInformation
		label: 'License Information'
		in: (120 @ 120 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeFutureDirections
		label: 'Future Directions'
		in: (160 @ 160 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeToSqueak
		label: 'Welcome to Squeak 4.5'
		in: (200 @ 200 extent: 500 @ 300)! !
!ReleaseBuilderFor4dot5 class methodsFor: 'scripts' stamp: 'fbs 1/1/2013 20:22'!
prepareNewBuild
	super prepareNewBuild.
	MCMockPackageInfo initialize.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:24'!
releaseRepository
	"At release time, change 'trunk' to 'squeak45'."
	^ MCHttpRepository
		location: 'http://source.squeak.org/trunk'
		user: 'squeak'
		password: 'squeak'! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:22'!
setDisplayExtent: extent
"Uncomment next line when the primitives become available in the Squeak VM."
"	DisplayScreen hostWindowSize: extent."
	Display extent = extent ifFalse: [ Warning signal: 'Display extent not set to ', extent ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
setPreferences
	Preferences 
		installBrightWindowColors ;
		setPreference: #scrollBarsWithoutMenuButton toValue: true ;
		setPreference: #swapMouseButtons toValue: true ;
		setPreference: #annotationPanes toValue: true ;
		setPreference: #showSplitterHandles toValue: false ;
		setPreference: #showBoundsInHalo toValue: true ;
		setPreference: #alternateHandlesLook toValue: false ;
		setPreference: #roundedMenuCorners toValue: false ;
		setPreference: #roundedWindowCorners toValue: false.
	PluggableButtonMorph roundedButtonCorners: false.
	FillInTheBlankMorph roundedDialogCorners: false.
	Workspace shouldStyle: false.
	NetNameResolver enableIPv6: true.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
switchToNewRepository
	| old44Repository |
	MCMcmUpdater defaultUpdateURL: self releaseRepository description.
	old44Repository := MCRepositoryGroup default repositories
				detect: [:each | each description includesSubString: 'squeak44'] ifNone: [nil].
	old44Repository
		ifNotNil: [MCRepositoryGroup default removeRepository: old44Repository].
	MCRepositoryGroup default addRepository: self releaseRepository! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
versionString
	^ 'Squeak4.5'.! !

ReleaseBuilder class removeSelector: #transferCurrentPackages!

"ReleaseBuilder"!
!Environment class methodsFor: 'as yet unclassified' stamp: 'cwp 1/1/2013 18:52' prior: 40834114!
initialize
	self install! !

"Environments"!
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:59' prior: 52081878!
initPattern: aString notifying: req return: aBlock

	| result |
	self
		init: (ReadStream on: aString asString)
		cue: (CompilationCue source: aString requestor: req)
		failBlock: [^nil].
	encoder := self.
	result := aBlock value: (self pattern: false inContext: nil).
	encoder := failBlock := nil.  "break cycles"
	^result! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:01' prior: 34175471!
parse: sourceStream class: class category: aCategory noPattern: noPattern context: aContext notifying: req ifFail: aBlock 
	| c |
	c := CompilationCue
			source: sourceStream
			context: aContext
			class: class
			category: aCategory
			requestor: req.
	^ self 
		parse: sourceStream 
		cue: c 
		noPattern: noPattern 
		ifFail: aBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:11' prior: 34183963!
evaluate: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	 a method. If aContext is not nil, the text can refer to temporaries in that
	 context (the Debugger uses this). If aRequestor is not nil, then it will receive
	 a notify:at: message before the attempt to evaluate is aborted. Finally, the 
	 compiled method is invoked from here via withArgs:executeMethod:, hence
	 the system no longer creates Doit method litter on errors."
	
	| theClass |
	theClass := ((aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class).
	self setCue: (CompilationCue
		source: textOrStream
		context: aContext
		receiver: receiver
		class: theClass
		environment: theClass environment
		category: nil
		requestor: aRequestor).
	^ self evaluate: textOrStream cue: cue ifFail: failBlock logged: logFlag! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:17' prior: 34185488!
from: textOrStream class: aClass classified: aCategory context: aContext notifying: req
	self source: textOrStream.
	self setCue: 
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			category: aCategory
			requestor: req)! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:55' prior: 50781309!
from: textOrStream class: aClass context: aContext notifying: req
	self source: textOrStream.
	self setCue:
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			requestor: req)
! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/27/2012 09:41' prior: 50996506!
init: aClass context: aContext notifying: anObject
	| c |
	c := CompilationCue
		context: aContext 
		class: aClass 
		requestor: nil.
	self init: c notifying: anObject! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:58' prior: 39061698!
temps: tempVars literals: lits class: cl 
	"Initialize this encoder for decompilation."

	self setCue: (CompilationCue class: cl).
	supered := false.
	nTemps := tempVars size.
	tempVars do: [:node | scopeTable at: node name put: node].
	literalStream := WriteStream on: (Array new: lits size).
	literalStream nextPutAll: lits.
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32.! !

"Compiler"!
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:48' prior: 36026010!
addClassVarName: aString 
	"Add the argument, aString, as a class variable of the receiver.
	Signal an error if the first character of aString is not capitalized,
	or if it is already a variable named in the class."
	| symbol oldState |
	oldState := self copy.
	aString first canBeGlobalVarInitial
		ifFalse: [^self error: aString, ' class variable name should be capitalized; proceed to include anyway.'].
	symbol := aString asSymbol.
	self withAllSubclasses do: 
		[:subclass | 
		(self canFindWithoutEnvironment: symbol) ifTrue: [
			(DuplicateVariableError new)
				superclass: superclass; "fake!!!!!!"
				variable: aString;
				signal: aString, ' is already defined']].
	classPool == nil ifTrue: [classPool := Dictionary new].
	(classPool includesKey: symbol) ifFalse: 
		["Pick up any refs in Undeclared"
		classPool declare: symbol from: environment undeclared.
		SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: oldState to: self]! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:48' prior: 54782024!
bindingOf: varName
	^ self bindingOf: varName environment: self environment! !
!Class methodsFor: 'organization' stamp: 'cwp 6/25/2012 18:25' prior: 54785804!
category
	"Answer the system organization category for the receiver. First check whether the
	category name stored in the ivar is still correct and only if this fails look it up
	(latter is much more expensive)"

	category ifNotNil: [ :symbol |
		((self environment organization listAtCategoryNamed: symbol) includes: self name)
			ifTrue: [ ^symbol ] ].
	category := self environment organization categoryOfElement: self name.
	^category! !
!Class methodsFor: 'initialize-release' stamp: 'cwp 6/22/2012 15:49' prior: 36027730!
declare: varString 
	"Declare class variables common to all instances. Answer whether 
	recompilation is advisable."

	| newVars conflicts |
	
	newVars := 
		(Scanner new scanFieldNames: varString)
			collect: [:x | x asSymbol].
	newVars do:
		[:var | var first canBeGlobalVarInitial
			ifFalse: [self error: var, ' class variable name should be capitalized; proceed to include anyway.']].
	conflicts := false.
	classPool == nil 
		ifFalse: [(classPool keys reject: [:x | newVars includes: x]) do: 
					[:var | self removeClassVarName: var]].
	(newVars reject: [:var | self classPool includesKey: var])
		do: [:var | "adding"
			"check if new vars defined elsewhere"
			(self canFindWithoutEnvironment: var) ifTrue: [
					(DuplicateVariableError new)
						superclass: superclass; "fake!!!!!!"
						variable: var;
						signal: var, ' is already defined'.
					conflicts := true]].
	newVars size > 0
		ifTrue: 
			[classPool := self classPool.
			"in case it was nil"
			newVars do: [:var | classPool declare: var from: environment undeclared]].
	^conflicts! !
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:49' prior: 54802475!
removeClassVarName: aString 
	"Remove the class variable whose name is the argument, aString, from 
	the names defined in the receiver, a class. Create an error notification if 
	aString is not a class variable or if it is still being used in the code of 
	the class."

	| aSymbol |
	aSymbol := aString asSymbol.
	(classPool includesKey: aSymbol)
		ifFalse: [^self error: aString, ' is not a class variable'].
	self withAllSubclasses do:[:subclass |
		(Array with: subclass with: subclass class) do:[:classOrMeta |
			(classOrMeta whichSelectorsReferTo: (classPool associationAt: aSymbol))
				isEmpty ifFalse: [
					InMidstOfFileinNotification signal ifTrue: [
						Transcript cr; show: self name, ' (' , aString , ' is Undeclared) '.
						^ environment undeclared declare: aSymbol from: classPool].
					(self confirm: (aString,' is still used in code of class ', classOrMeta name,
						'.\Is it okay to move it to Undeclared?') withCRs)
						ifTrue:[^Undeclared declare: aSymbol from: classPool]
						ifFalse:[^self]]]].
	classPool removeKey: aSymbol.
	classPool isEmpty ifTrue: [classPool := nil].
! !
!Class methodsFor: 'class name' stamp: 'cwp 6/22/2012 15:49' prior: 54796206!
rename: aString 
	"The new name of the receiver is the argument, aString."

	| oldName newName |
	(newName := aString asSymbol) = (oldName := self name)
		ifTrue: [^ self].
	(self environment includesKey: newName)
		ifTrue: [^ self error: newName , ' already exists'].
	(environment undeclared includesKey: newName)
		ifTrue: [self inform: 'There are references to, ' , aString printString , '
from Undeclared. Check them after this change.'].
	name := newName.
	self environment renameClass: self from: oldName! !
!ClassBuilder methodsFor: 'class definition' stamp: 'cwp 6/22/2012 01:05' prior: 39054430!
name: className inEnvironment: env subclassOf: newSuper type: type instanceVariableNames: instVarString classVariableNames: classVarString poolDictionaries: poolString category: category unsafe: unsafe
	"Define a new class in the given environment.
	If unsafe is true do not run any validation checks.
	This facility is provided to implement important system changes."
	| oldClass instVars classVars copyOfOldClass newClass |
 
	environ := env.
	instVars := Scanner new scanFieldNames: instVarString.
	classVars := (Scanner new scanFieldNames: classVarString) collect: [:x | x asSymbol].

	"Validate the proposed name"
	unsafe ifFalse:[(self validateClassName: className) ifFalse:[^nil]].
	oldClass := env at: className ifAbsent:[nil].
	oldClass isBehavior 
		ifFalse: [oldClass := nil]  "Already checked in #validateClassName:"
		ifTrue: [
			copyOfOldClass := oldClass copy.
			copyOfOldClass superclass addSubclass: copyOfOldClass].
	
	
	[ | newCategory needNew force organization oldCategory |
	unsafe ifFalse:[
		"Run validation checks so we know that we have a good chance for recompilation"
		(self validateSuperclass: newSuper forSubclass: oldClass) ifFalse:[^nil].
		(self validateInstvars: instVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateClassvars: classVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateSubclassFormat: type from: oldClass forSuper: newSuper extra: instVars size) ifFalse:[^nil]].

	"See if we need a new subclass"
	needNew := self needsSubclassOf: newSuper type: type instanceVariables: instVars from: oldClass.
	needNew == nil ifTrue:[^nil]. "some error"

	(needNew and:[unsafe not]) ifTrue:[
		"Make sure we don't redefine any dangerous classes"
		(self tooDangerousClasses includes: oldClass name) ifTrue:[
			self error: oldClass name, ' cannot be changed'.
		].
		"Check if the receiver should not be redefined"
		(oldClass ~~ nil and:[oldClass shouldNotBeRedefined]) ifTrue:[
			self notify: oldClass name asText allBold, 
						' should not be redefined. \Proceed to store over it.' withCRs]].

	needNew ifTrue:[
		"Create the new class"
		newClass := self 
			newSubclassOf: newSuper 
			type: type 
			instanceVariables: instVars
			from: oldClass.
		newClass == nil ifTrue:[^nil]. "Some error"
		newClass setName: className.
		newClass environment: environ.
	] ifFalse:[
		"Reuse the old class"
		newClass := oldClass.
	].

	"Install the class variables and pool dictionaries... "
	force := (newClass declare: classVarString) | (newClass sharing: poolString).

	"... classify ..."
	newCategory := category asSymbol.
	organization := environ ifNotNil:[environ organization].
	oldClass isNil ifFalse: [oldCategory := (organization categoryOfElement: oldClass name) asSymbol].
	organization classify: newClass name under: newCategory suppressIfDefault: true.
	
	"... recompile ..."
	newClass := self recompile: force from: oldClass to: newClass mutate: false.

	"... export if not yet done ..."
	(environ at: newClass name ifAbsent:[nil]) == newClass ifFalse:[
		[environ at: newClass name put: newClass]
			on: AttemptToWriteReadOnlyGlobal do:[:ex| ex resume: true].
		environ flushClassNameCache.
	].


	newClass doneCompiling.
	"... notify interested clients ..."
	oldClass isNil ifTrue: [
		SystemChangeNotifier uniqueInstance classAdded: newClass inCategory: newCategory.
		^ newClass].
	newCategory ~= oldCategory 
		ifTrue: [SystemChangeNotifier uniqueInstance class: newClass recategorizedFrom: oldCategory to: category]
		ifFalse: [SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: copyOfOldClass to: newClass.].
] ensure: 
		[copyOfOldClass ifNotNil: [copyOfOldClass superclass removeSubclass: copyOfOldClass].
		Behavior flushObsoleteSubclasses.
		].
	^newClass! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 22:57' prior: 18572019!
superclass: newSuper
	subclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat 
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class."
	| env |
	env := EnvironmentRequest signal ifNil: [newSuper environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: newSuper
		type: newSuper typeOfClass
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:01' prior: 50629912!
superclass: aClass
	variableByteSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable byte-sized nonpointer variables."
	| oldClassOrNil actualType env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a byte subclass of a class with named fields'].
	(aClass isVariable and: [aClass isWords])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with word fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with pointer fields'].
	oldClassOrNil := aClass environment at: t ifAbsent:[nil].
	actualType := (oldClassOrNil notNil
				   and: [oldClassOrNil typeOfClass == #compiledMethod])
					ifTrue: [#compiledMethod]
					ifFalse: [#bytes].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: actualType
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:03' prior: 18573442!
superclass: aClass
	variableSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable pointer variables."
	
	| env |
	aClass isBits ifTrue: 
		[^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #variable
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18574098!
superclass: aClass
	variableWordSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable word-sized nonpointer variables."
	| env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a word subclass of a class with named fields'].
	(aClass isVariable and: [aClass isBytes])
		ifTrue: [^self error: 'cannot make a word subclass of a class with byte fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a word subclass of a class with pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #words
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18575028!
superclass: aClass
	weakSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class (the receiver) in which the subclass is to 
	have weak indexable pointer variables."
	| env |
	aClass isBits 
		ifTrue: [^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #weak
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !

"Kernel"!
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 59135029!
ambiguousSelector: aString inRange: anInterval
	| correctedSelector userSelection offset intervalWithOffset |
	
	self interactive ifFalse: [
		"In non interactive mode, compile with backward comapatibility: $- is part of literal argument"
		Transcript cr; store: encoder classEncoding; nextPutAll:#'>>';store: encoder selector; show: ' would send ' , token , '-'.
		^super ambiguousSelector: aString inRange: anInterval].
	
	"handle the text selection"
	userSelection := cue requestor selectionInterval.
	intervalWithOffset := anInterval first + requestorOffset to: anInterval last + requestorOffset.
	cue requestor selectFrom: intervalWithOffset first to: intervalWithOffset last.
	cue requestor select.

	"Build the menu with alternatives"
	correctedSelector := AmbiguousSelector 
			signalName: aString
			inRange: intervalWithOffset.
	correctedSelector ifNil: [^self fail].

	"Execute the selected action"
	offset := self substituteWord: correctedSelector wordInterval: intervalWithOffset offset: 0.
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + offset.
	token := (correctedSelector readStream upTo: Character space) asSymbol! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 38558136!
collectTemporaryDeclarationsFrom: methodNode
	| tempsMarks str |
	tempsMarks := OrderedCollection new.
	str := cue requestor text asString.
	methodNode accept: (ParseNodeEnumerator
		ofBlock: [ :aNode | 
			| mark |
			(aNode class canUnderstand: #tempsMark) 
				ifTrue: 
					[mark := aNode tempsMark.
					(mark notNil and: [ mark between: 1 and: str size ] and: [ (str at: mark) = $| ])
						ifTrue: [ tempsMarks addLast: aNode ]]]).
	(tempsMark notNil and: [ tempsMark between: 1 and: str size ] and: [ (str at: tempsMark) = $| ])
						ifTrue: [ tempsMarks addLast: self ].
	^ tempsMarks sorted: [ :a :b | a tempsMark > b tempsMark ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 52096606!
correctSelector: proposedKeyword wordIntervals: spots exprInterval: expInt ifAbort: abortAction
	"Correct the proposedKeyword to some selector symbol, correcting the original text if such action is indicated.  abortAction is invoked if the proposedKeyword couldn't be converted into a valid selector.  Spots is an ordered collection of intervals within the test stream of the for each of the keyword parts."

	| correctSelector userSelection |
	"If we can't ask the user, assume that the keyword will be defined later"
	self interactive ifFalse: [^proposedKeyword asSymbol].

	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spots first first to: spots last last.
	cue requestor select.

	correctSelector := UnknownSelector name: proposedKeyword.
	correctSelector ifNil: [^abortAction value].

	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.

	self substituteSelector: correctSelector keywords wordIntervals: spots.
	^(proposedKeyword last ~~ $:
	   and: [correctSelector last == $:])
		ifTrue: [abortAction value]
		ifFalse: [correctSelector]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 33907242!
correctVariable: proposedVariable interval: spot 
	"Correct the proposedVariable to a known variable, or declare it as a new
	variable if such action is requested.  We support declaring lowercase
	variables as temps or inst-vars, and uppercase variables as Globals or 
	ClassVars, depending on whether the context is nil (class=UndefinedObject).
	Spot is the interval within the test stream of the variable.
	rr 3/4/2004 10:26 : adds the option to define a new class. "

	"Check if this is an i-var, that has been corrected already (ugly)"

	"Display the pop-up menu"

	| binding userSelection action |
	(encoder classEncoding instVarNames includes: proposedVariable) ifTrue: 
		[^InstanceVariableNode new 
			name: proposedVariable
			index: (encoder classEncoding allInstVarNames indexOf: proposedVariable)].

	"If we can't ask the user for correction, make it undeclared"
	self interactive ifFalse: [^encoder undeclared: proposedVariable].

	"First check to see if the requestor knows anything about the variable"
	(binding := cue requestor bindingOf: proposedVariable)
		ifNotNil: [^encoder global: binding name: proposedVariable].
	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spot first to: spot last.
	cue requestor select.

	"Build the menu with alternatives"
	action := UndeclaredVariable 
				signalFor: self
				name: proposedVariable
				inRange: spot.
	action ifNil: [^self fail].

	"Execute the selected action"
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.
	^action value! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:19' prior: 34172921!
declareUndeclaredTemps: methodNode
	"Declare any undeclared temps, declaring them at the smallest enclosing scope."

	| undeclared userSelection blocksToVars |
	(undeclared := encoder undeclaredTemps) isEmpty ifTrue:
		[^self].
	userSelection := cue requestor selectionInterval.
	blocksToVars := IdentityDictionary new.
	undeclared do:
		[:var|
		(blocksToVars
			at: (var tag == #method
					ifTrue: [methodNode block]
					ifFalse: [methodNode accept: (VariableScopeFinder new ofVariable: var)])
			ifAbsentPut: [SortedCollection new]) add: var name].
	(blocksToVars removeKey: methodNode block ifAbsent: []) ifNotNil:
		[:rootVars|
		rootVars do: [:varName| self pasteTempAtMethodLevel: varName]].
	(blocksToVars keys sorted: [:a :b| a tempsMark < b tempsMark]) do:
		[:block| | decl |
		decl := (blocksToVars at: block) reduce: [:a :b| a, ' ', b].
		block temporaries isEmpty
			ifTrue:
				[self substituteWord: ' | ', decl, ' |'
					wordInterval: (block tempsMark + 1 to: block tempsMark)
					offset: requestorOffset]
			ifFalse:
				[self substituteWord: decl, ' '
					wordInterval: (block tempsMark to: block tempsMark - 1)
					offset: requestorOffset]].
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + requestorOffset.
	ReparseAfterSourceEditing signal! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 11:45' prior: 37183770!
defineClass: className 
	"prompts the user to define a new class,  
	asks for it's category, and lets the users edit further  
	the definition"
	| sym cat def d2 |
	sym := className asSymbol.
	cat := UIManager default request: 'Enter class category : ' initialAnswer: self encoder classEncoding theNonMetaClass category.
	cat
		ifEmpty: [cat := 'Unknown'].
	def := 'Object subclass: #' , sym , '
		instanceVariableNames: '''' 
		classVariableNames: ''''
		poolDictionaries: ''''
		category: ''' , cat , ''''.
	d2 := UIManager default request: 'Edit class definition : ' initialAnswer: def.
	d2
		ifEmpty: [d2 := def].
	Compiler evaluate: d2.
	^ encoder
		global: (cue environment bindingOf: sym)
		name: sym! !
!Parser methodsFor: 'primitives' stamp: 'cwp 12/27/2012 11:46' prior: 37184567!
externalFunctionDeclaration
	"Parse the function declaration for a call to an external library."
	| descriptorClass callType modifier retType externalName args argType module fn |
	descriptorClass := cue environment
		valueOf: #ExternalFunction 
		ifAbsent: [^ false].
	callType := descriptorClass callingConventionFor: here.
	callType == nil ifTrue:[^false].
	[modifier := descriptorClass callingConventionModifierFor: token.
	 modifier notNil] whileTrue:
		[self advance.
		 callType := callType bitOr: modifier].
	"Parse return type"
	self advance.
	retType := self externalType: descriptorClass.
	retType == nil ifTrue:[^self expected:'return type'].
	"Parse function name or index"
	externalName := here.
	(self match: #string) 
		ifTrue:[externalName := externalName asSymbol]
		ifFalse:[(self match:#number) ifFalse:[^self expected:'function name or index']].
	(self matchToken: #'(') ifFalse:[^self expected:'argument list'].
	args := WriteStream on: Array new.
	[here == #')'] whileFalse:[
		argType := self externalType: descriptorClass.
		argType == nil ifTrue:[^self expected:'argument'].
		argType isVoid & argType isPointerType not ifFalse:[args nextPut: argType].
	].
	(self matchToken: #')') ifFalse:[^self expected:')'].
	(self matchToken: 'module:') ifTrue:[
		module := here.
		(self match: #string) ifFalse:[^self expected: 'String'].
		module := module asSymbol].
	Smalltalk at: #ExternalLibraryFunction ifPresent:[:xfn|
		fn := xfn name: externalName 
				module: module 
				callType: callType
				returnType: retType
				argumentTypes: args contents.
		self allocateLiteral: fn.
	].
	(self matchToken: 'error:')
		ifTrue:
			[| errorCodeVariable |
			 errorCodeVariable := here.
			(hereType == #string
			 or: [hereType == #word]) ifFalse:[^self expected: 'error code (a variable or string)'].
			 self advance.
			 self addPragma: (Pragma keyword: #primitive:error: arguments: (Array with: 120 with: errorCodeVariable)).
			 fn ifNotNil: [fn setErrorCodeName: errorCodeVariable]]
		ifFalse:
			[self addPragma: (Pragma keyword: #primitive: arguments: #(120))].
	^true
! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:19' prior: 58306169!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:22' prior: 58137223!
notify: string at: location
	cue requestor isNil
		ifTrue: [(encoder == self or: [encoder isNil]) ifTrue: [^ self fail  "failure setting up syntax error"].
				SyntaxErrorNotification
					inClass: encoder classEncoding
					category: cue category
					withCode: 
						(source contents asText
							copyReplaceFrom: location
							to: location - 1
							with: ((string , ' ->') asText allBold 
								addAttribute: TextColor red; yourself))
					doitFlag: doitFlag
					errorMessage: string
					location: location]
		ifFalse: [cue requestor
					notify: string , ' ->'
					at: location
					in: source].
	^self fail! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:17' prior: 34177108!
pasteTempAtMethodLevel: name
	| insertion delta theTextString characterBeforeMark |

	theTextString := cue requestor text string.
	characterBeforeMark := theTextString at: tempsMark-1 ifAbsent: [$ ].
	(theTextString at: tempsMark) = $| ifTrue: [
  		"Paste it before the second vertical bar"
		insertion := name, ' '.
		characterBeforeMark isSeparator ifFalse: [insertion := ' ', insertion].
		delta := 0.
	] ifFalse: [
		"No bars - insert some with CR, tab"
		insertion := '| ' , name , ' |',String cr.
		delta := 2.	"the bar and CR"
		characterBeforeMark = Character tab ifTrue: [
			insertion := insertion , String tab.
			delta := delta + 1.	"the tab"
		].
	].
	tempsMark := tempsMark +
		(self substituteWord: insertion
			wordInterval: (tempsMark to: tempsMark-1)
			offset: 0) - delta! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:16' prior: 52095305!
queryUndefined
	| varStart varName | 
	varName := parseNode key.
	varStart := self endOfLastToken + requestorOffset - varName size + 1.
	cue requestor selectFrom: varStart to: varStart + varName size - 1; select.
	(UndefinedVariable name: varName) ifFalse: [^ self fail]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38599341!
removeEmptyTempDeclarationsFrom: methodNode

	| sourceCode madeChanges tempsMarkHolder |
	sourceCode := cue requestor text asString.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	madeChanges := false.
	tempsMarkHolder do: [ :currentBlock | | tempsMarkChar0 tempsMarkChar1 tempsMarkChar2 end start |
		tempsMarkChar0 := (sourceCode at: currentBlock tempsMark).
		tempsMarkChar1 := (sourceCode at: currentBlock tempsMark - 1).
		tempsMarkChar2 := (sourceCode at: currentBlock tempsMark - 2).
		tempsMarkChar0 = $| & tempsMarkChar1 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 1].
		tempsMarkChar0 = $| & tempsMarkChar1 = $  & tempsMarkChar2 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 2].
		
		start notNil & end notNil ifTrue: [
			| lineStart lineEnd |
			lineStart := 1 + (sourceCode 
				lastIndexOf: Character cr 
				startingAt: start - 1
				ifAbsent: [ 0 ]).
			lineEnd := sourceCode 
				indexOf: Character cr
				startingAt: end + 1
				ifAbsent: [ sourceCode size ].
			((sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: lineStart) >= start 
				and: [ (sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: end + 1) > lineEnd ]) ifTrue: [
					start := lineStart.
					end := lineEnd ].
			cue requestor correctFrom: start to: end with: ''.
			madeChanges := true.
			currentBlock tempsMark: nil ] ].
	madeChanges ifTrue: [ReparseAfterSourceEditing signal]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38561281!
removeUnusedTemporaryNamed: temp from: str lookingAt: currentBlock movingTempMarksOf: someBlocks

	| start end |
	end := currentBlock tempsMark - 1.
	["Beginning at right temp marker..."
	start := end - temp size + 1.
	end < temp size or: [ (str at: start) = $| ]
		or: [ temp = (str copyFrom: start to: end) 
			and: [ ((str at: start - 1) = $| | (str at: start - 1) isSeparator) 
				& ((str at: end + 1) = $| | (str at: end + 1) isSeparator) ] ]]
		whileFalse: [ 
			"Search left for the unused temp"
			end := cue requestor nextTokenFrom: end direction: -1 ].
	(end < temp size or: [ (str at: start) = $| ])
		ifFalse: 
			[(str at: start - 1) = $ 
				ifTrue: [ start := start - 1 ].
			cue requestor correctFrom: start to: end with: ''.
			someBlocks do: [ :aBlock | aBlock tempsMark: aBlock tempsMark - (end - start + 1)].
			^true ].
	^false! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 38562194!
removeUnusedTemps: methodNode
	"Scan for unused temp names, and prompt the user about the prospect of removing each one found"

	| madeChanges tempsMarkHolder unusedTempNames tempMarkHoldersToChange |
	madeChanges := false.
	tempMarkHoldersToChange := OrderedCollection new.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	unusedTempNames := encoder unusedTempNames select: 
		[ :temp | (encoder lookupVariable: temp ifAbsent: [ ]) isUndefTemp 
				and: [ self queriedUnusedTemporaries at: temp ifAbsentPut: [UnusedVariable name: temp] ]].
	tempsMarkHolder do: [ :currentBlock | 
		tempMarkHoldersToChange add: currentBlock.
		unusedTempNames do: 
			[ :temp |
			(self 
				removeUnusedTemporaryNamed: temp 
				from: cue requestor text asString 
				lookingAt: currentBlock
				movingTempMarksOf: tempMarkHoldersToChange) ifTrue: [ madeChanges := true ]]].
	madeChanges
		ifTrue: [ self removeEmptyTempDeclarationsFrom: methodNode.
			ReparseAfterSourceEditing signal ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 34179326!
substituteWord: correctWord wordInterval: spot offset: o
	"Substitute the correctSelector into the (presumed interactive) receiver.
	 Update requestorOffset based on the delta size and answer the updated offset."

	cue requestor correctFrom: spot first + o to: spot last + o with: correctWord.
	requestorOffset := requestorOffset + correctWord size - spot size.
	^o + correctWord size - spot size! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34179807!
temporaries
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'
! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34180638!
temporariesIn: methodSelector
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance in: methodSelector)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:11' prior: 53971863!
compiledMethodFor: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code
	 into a method, and answers it.  If receiver is not nil, then the text can
	 refer to instance variables of that receiver (the Inspector uses this).
	 If aContext is not nil, the text can refer to temporaries in that context
	 (the Debugger uses this). If aRequestor is not nil, then it will receive a 
	 notify:at: message before the attempt to evaluate is aborted."

	| methodNode method theClass |
	theClass := (aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class.
	self from: textOrStream class: theClass context: aContext notifying: aRequestor.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].
	method := self interactive ifTrue: [ 	methodNode generateWithTempNames ] 
		ifFalse: [methodNode generate].
		
	logFlag ifTrue:
		[SystemChangeNotifier uniqueInstance evaluated: sourceStream contents context: aContext].
	^method! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:33' prior: 34363593!
format: aStream noPattern: noPattern ifFail: failBlock
	^(self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]) preen! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 10:08' prior: 58306325!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Compiler methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:10' prior: 50779387!
notify: aString at: location
	"Refer to the comment in Object|notify:."

	^ cue requestor == nil
		ifTrue: [SyntaxErrorNotification
					inClass: cue getClass
					category: cue category
					withCode: 
						(sourceStream contents
							copyReplaceFrom: location
							to: location - 1
							with: aString)
					doitFlag: false
					errorMessage: aString
					location: location]
		ifFalse: [cue requestor
					notify: aString
					at: location
					in: sourceStream]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:34' prior: 50777201!
parse: textOrStream in: aClass notifying: req
	"Compile the argument, textOrStream, with respect to the class, aClass, and
	 answer the MethodNode that is the root of the resulting parse tree.  Notify the
	 argument, req, if an error occurs. The failBlock is defaulted to an empty block."

	self from: textOrStream class: aClass context: nil notifying: req.
	^self parser
		parse: sourceStream
		cue: cue
		noPattern: false
		ifFail: []! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:09' prior: 36332471!
parser

	parser ifNil: [parser := (cue getClass ifNil: [self class]) newParser].
	^parser! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:37' prior: 50780779!
translate: aStream noPattern: noPattern ifFail: failBlock
	^self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:37' prior: 19124095!
translate: aStream noPattern: noPattern ifFail: failBlock parser: parser
	| tree |
	tree := parser
			parse: aStream
			cue: cue 
			noPattern: noPattern
			ifFail: [^ failBlock value].
	^ tree! !
!Encoder methodsFor: 'results' stamp: 'cwp 12/27/2012 10:26' prior: 50999892!
associationForClass
	| assoc |
	assoc := self environment associationAt: cue getClass name ifAbsent: [nil].
	^assoc value == cue getClass
		ifTrue: [assoc]
		ifFalse: [Association new value: cue getClass]! !
!Encoder methodsFor: 'temps' stamp: 'cwp 12/27/2012 10:25' prior: 20148386!
bindTemp: name in: methodSelector
	"Declare a temporary; error not if a field or class variable."
	scopeTable at: name ifPresent:[:node|
		"When non-interactive raise the error only if its a duplicate"
		(node isTemp or:[requestor interactive])
			ifTrue:[^self notify:'Name is already defined']
			ifFalse:[Transcript 
				show: '(', name, ' is shadowed in "' , cue getClass printString , '>>' , methodSelector printString , '")']].
	^self reallyBind: name! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:25' prior: 20149084!
classEncoding
	"This is a hack so that the parser may findout what class it was parsing for when it wants to create a syntax error view."
	^ cue getClass! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:39' prior: 20138819!
encodeLiteral: object

	^self
		name: object
		key: (cue  literalScannedAs: object notifying: self)
		class: LiteralNode
		type: LdLitType
		set: litSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 20139010!
encodeSelector: aSelector

	^self
		name: aSelector
		key: aSelector
		class: SelectorNode
		type: SendType
		set: selectorSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 58545123!
environment
	"Answer the environment of the current compilation context,
	 be it in a class or global (e.g. a workspace)"
	^cue environment! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 11:41' prior: 50994497!
lookupInPools: varName ifFound: assocBlock

	^Symbol
		hasInterned: varName
		ifTrue:
			[:sym|
			(cue bindingOf: sym)
				ifNil: [^false]
				ifNotNil: [:assoc| assocBlock value: assoc]]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 51004306!
possibleNamesFor: proposedName
	| results |
	results := cue getClass 
		possibleVariablesFor: proposedName 
		continuedFrom: nil.
	^ proposedName correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 50995012!
possibleVariablesFor: proposedVariable

	| results |
	results := proposedVariable correctAgainstDictionary: scopeTable
								continuedFrom: nil.
	proposedVariable first canBeGlobalVarInitial ifTrue:
		[ results := cue getClass possibleVariablesFor: proposedVariable
						continuedFrom: results ].
	^ proposedVariable correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:42' prior: 51002830!
undeclared: name
	| sym |
	requestor interactive ifTrue:
		[requestor requestor == #error: ifTrue:
			[requestor error: 'Undeclared'].
		 ^self notify: 'Undeclared'].
	"Allow knowlegeable clients to squash the undeclared warning if they want (e.g.
	 Diffing pretty printers that are simply formatting text).  As this breaks
	 compilation it should only be used by clients that want to discard the result
	 of the compilation.  To squash the warning use e.g.
		[Compiler format: code in: class notifying: nil decorated: false]
			on: UndeclaredVariableWarning
			do: [:ex| ex resume: false]"
	sym := name asSymbol.
	^(UndeclaredVariableWarning new name: name selector: selector class: cue getClass) signal
		ifTrue:
			[| undeclared |
			undeclared := cue environment undeclared.
			undeclared at: sym put: nil.
			self global: (undeclared associationAt: sym) name: sym]
		ifFalse:
			[self global: (Association key: sym) name: sym]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:23' prior: 51006007!
warnAboutShadowed: name
	requestor addWarning: name,' is shadowed'.
	selector ifNotNil:
		[Transcript cr; show: cue getClass name,'>>', selector, '(', name,' is shadowed)']! !

"Compiler"!
!SmalltalkImage methodsFor: 'housekeeping' stamp: 'cwp 6/22/2012 15:56' prior: 58497062!
cleanOutUndeclared 
	globals undeclared removeUnreferencedKeys! !
!SmalltalkImage methodsFor: 'special objects' stamp: 'cwp 6/22/2012 09:01' prior: 40515090!
recreateSpecialObjectsArray
	"Smalltalk recreateSpecialObjectsArray"
	
	"To external package developers:
	**** DO NOT OVERRIDE THIS METHOD.  *****
	If you are writing a plugin and need additional special object(s) for your own use, 
	use addGCRoot() function and use own, separate special objects registry "
	
	"The Special Objects Array is an array of objects used by the Squeak virtual machine.
	 Its contents are critical and accesses to it by the VM are unchecked, so don't even
	 think of playing here unless you know what you are doing."
	| newArray |
	newArray := Array new: 56.
	"Nil false and true get used throughout the interpreter"
	newArray at: 1 put: nil.
	newArray at: 2 put: false.
	newArray at: 3 put: true.
	"This association holds the active process (a ProcessScheduler)"
	newArray at: 4 put: (self bindingOf: #Processor).
	"Numerous classes below used for type checking and instantiation"
	newArray at: 5 put: Bitmap.
	newArray at: 6 put: SmallInteger.
	newArray at: 7 put: ByteString.
	newArray at: 8 put: Array.
	newArray at: 9 put: Smalltalk.
	newArray at: 10 put: Float.
	newArray at: 11 put: MethodContext.
	newArray at: 12 put: BlockContext.
	newArray at: 13 put: Point.
	newArray at: 14 put: LargePositiveInteger.
	newArray at: 15 put: Display.
	newArray at: 16 put: Message.
	newArray at: 17 put: CompiledMethod.
	newArray at: 18 put: (self specialObjectsArray at: 18).
	"(low space Semaphore)"
	newArray at: 19 put: Semaphore.
	newArray at: 20 put: Character.
	newArray at: 21 put: #doesNotUnderstand:.
	newArray at: 22 put: #cannotReturn:.
	newArray at: 23 put: nil. "This is the process signalling low space."
	"An array of the 32 selectors that are compiled as special bytecodes,
	 paired alternately with the number of arguments each takes."
	newArray at: 24 put: #(	#+ 1 #- 1 #< 1 #> 1 #<= 1 #>= 1 #= 1 #~= 1
							#* 1 #/ 1 #\\ 1 #@ 1 #bitShift: 1 #// 1 #bitAnd: 1 #bitOr: 1
							#at: 1 #at:put: 2 #size 0 #next 0 #nextPut: 1 #atEnd 0 #== 1 #class 0
							#blockCopy: 1 #value 0 #value: 1 #do: 1 #new 0 #new: 1 #x 0 #y 0 ).
	"An array of the 255 Characters in ascii order.
	 Cog inlines table into machine code at: prim so do not regenerate it."
	newArray at: 25 put: (self specialObjectsArray at: 25).
	newArray at: 26 put: #mustBeBoolean.
	newArray at: 27 put: ByteArray.
	newArray at: 28 put: Process.
	"An array of up to 31 classes whose instances will have compact headers"
	newArray at: 29 put: self compactClassesArray.
	newArray at: 30 put: (self specialObjectsArray at: 30). "(delay Semaphore)"
	newArray at: 31 put: (self specialObjectsArray at: 31). "(user interrupt Semaphore)"
	"Entries 32 - 34 unreferenced. Previously these contained prototype instances to be copied for fast initialization"
	newArray at: 32 put: nil. "was (Float new: 2)"
	newArray at: 33 put: nil. "was (LargePositiveInteger new: 4)"
	newArray at: 34 put: nil. "was Point new"
	newArray at: 35 put: #cannotInterpret:.
	"Note: This must be fixed once we start using context prototypes (yeah, right)"
	"(MethodContext new: CompiledMethod fullFrameSize)."
	newArray at: 36 put: (self specialObjectsArray at: 36). "Is the prototype MethodContext (unused by the VM)"
	newArray at: 37 put: BlockClosure.
	"(BlockContext new: CompiledMethod fullFrameSize)."
	newArray at: 38 put: (self specialObjectsArray at: 38). "Is the prototype BlockContext (unused by the VM)"
	"array of objects referred to by external code"
	newArray at: 39 put: (self specialObjectsArray at: 39).	"preserve external semaphores"
	newArray at: 40 put: nil. "Reserved for Mutex in Cog VMs"
	newArray at: 41 put: nil. "Reserved for a LinkedList instance for overlapped calls in CogMT"
	"finalization Semaphore"
	newArray at: 42 put: ((self specialObjectsArray at: 42) ifNil: [Semaphore new]).
	newArray at: 43 put: LargeNegativeInteger.
	"External objects for callout.
	 Note: Written so that one can actually completely remove the FFI."
	newArray at: 44 put: (self at: #ExternalAddress ifAbsent: []).
	newArray at: 45 put: (self at: #ExternalStructure ifAbsent: []).
	newArray at: 46 put: (self at: #ExternalData ifAbsent: []).
	newArray at: 47 put: (self at: #ExternalFunction ifAbsent: []).
	newArray at: 48 put: (self at: #ExternalLibrary ifAbsent: []).
	newArray at: 49 put: #aboutToReturn:through:.
	newArray at: 50 put: #run:with:in:.
	"51 reserved for immutability message"
	"newArray at: 51 put: #attemptToAssign:withIndex:."
	newArray at: 52 put: #(nil "nil => generic error" #'bad receiver'
							#'bad argument' #'bad index'
							#'bad number of arguments'
							#'inappropriate operation'  #'unsupported operation'
							#'no modification' #'insufficient object memory'
							#'insufficient C memory' #'not found' #'bad method'
							#'internal error in named primitive machinery'
							#'object may move').
	"53 to 55 are for Alien"
	newArray at: 53 put: (self at: #Alien ifAbsent: []).
	newArray at: 54 put: #invokeCallback:stack:registers:jmpbuf:.
	newArray at: 55 put: (self at: #UnsafeAlien ifAbsent: []).

	"Weak reference finalization"
	newArray at: 56 put: (self at: #WeakFinalizationList ifAbsent: []).

	"Now replace the interpreter's reference in one atomic operation"
	self specialObjectsArray becomeForward: newArray
	! !
!SmalltalkImage methodsFor: 'shrinking' stamp: 'cwp 6/22/2012 15:57' prior: 37288071!
unloadAllKnownPackages
	"Unload all packages we know how to unload and reload"

	"Prepare unloading"
	Smalltalk zapMVCprojects.
	Flaps disableGlobalFlaps: false.
	StandardScriptingSystem removeUnreferencedPlayers.
	Project removeAllButCurrent.
	#('Morphic-UserObjects' 'EToy-UserObjects' 'Morphic-Imported' )
		do: [:each | SystemOrganization removeSystemCategory: each].
	Smalltalk at: #ServiceRegistry ifPresent:[:aClass|
		SystemChangeNotifier uniqueInstance
			noMoreNotificationsFor: aClass.
	].
	World removeAllMorphs.

	"Go unloading"
	#(	'ReleaseBuilder' 'ScriptLoader'
		'311Deprecated' '39Deprecated'
		'Universes' 'SMLoader' 'SMBase' 'Installer-Core'
		'VersionNumberTests' 'VersionNumber'
		'Services-Base' 'PreferenceBrowser' 'Nebraska'
		'ToolBuilder-MVC' 'ST80'
		'CollectionsTests' 'GraphicsTests' 'KernelTests'  'MorphicTests' 
		'MultilingualTests' 'NetworkTests' 'ToolsTests' 'TraitsTests'
		'SystemChangeNotification-Tests' 'FlexibleVocabularies' 
		'EToys' 'Protocols' 'XML-Parser' 'Tests' 'SUnitGUI'
		'Help-Squeak' 'HelpSystem' 'SystemReporter'
	) do: [:pkgName| 
			(MCPackage named: pkgName) unload.
			MCMcmUpdater disableUpdatesOfPackage: pkgName.
			].
	"Traits use custom unload"
	Smalltalk at: #Trait ifPresent:[:aClass| aClass unloadTraits].

	"Post-unload cleanup"
	MCWorkingCopy flushObsoletePackageInfos.
	SystemOrganization removeSystemCategory: 'UserObjects'.
	Presenter defaultPresenterClass: nil.
	World dumpPresenter.
	ScheduledControllers := nil.
	Preferences removePreference: #allowEtoyUserCustomEvents.
	SystemOrganization removeEmptyCategories.
	ChangeSet removeChangeSetsNamedSuchThat:[:cs | (cs == ChangeSet current) not].
	globals undeclared removeUnreferencedKeys.
	StandardScriptingSystem initialize.
	MCFileBasedRepository flushAllCaches.
	MCDefinition clearInstances.
	Behavior flushObsoleteSubclasses.
	ChangeSet current clear.
	ChangeSet current name: 'Unnamed1'.
	Smalltalk flushClassNameCache.
	Smalltalk at: #Browser ifPresent:[:br| br initialize].
	DebuggerMethodMap voidMapCache.
	DataStream initialize.
	AppRegistry removeObsolete.
	FileServices removeObsolete.
	Preferences removeObsolete.
	TheWorldMenu removeObsolete.
	Smalltalk garbageCollect.
	Symbol compactSymbolTable.
	TheWorldMainDockingBar updateInstances.
	MorphicProject defaultFill: (Color gray: 0.9).
	World color: (Color gray: 0.9).
! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34' prior: 40472775!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:27' prior: 40496770!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!SystemDictionary methodsFor: 'dictionary access' stamp: 'cwp 6/22/2012 15:58' prior: 30574136!
at: aKey put: anObject 
	"Override from Dictionary to check Undeclared and fix up
	references to undeclared variables."
	| index element |
	(self includesKey: aKey) ifFalse: 
		[self declare: aKey from: (self at: #Undeclared).
		self flushClassNameCache].
	super at: aKey put: anObject.
	^ anObject! !

"System"!


From noreply at buildbot.pypy.org  Wed Jun 26 21:56:36 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 26 Jun 2013 21:56:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal) revert a052f153e311 for now:
 python-dev's waffled on nan containment
Message-ID: <20130626195636.B89501C0325@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r65008:7065efc00da6
Date: 2013-06-26 12:11 -0700
http://bitbucket.org/pypy/pypy/changeset/7065efc00da6/

Log:	(fijal) revert a052f153e311 for now: python-dev's waffled on nan
	containment quite a bit but there's no strong consensus on it

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1222,7 +1222,8 @@
     def _safe_find(self, w_list, obj, start, stop):
         l = self.unerase(w_list.lstorage)
         for i in range(start, min(stop, len(l))):
-            if l[i] == obj:
+            val = l[i]
+            if val == obj:
                 return i
         raise ValueError
 
@@ -1542,18 +1543,6 @@
         if reverse:
             l.reverse()
 
-    def _safe_find(self, w_list, obj, start, stop):
-        from rpython.rlib.rfloat import isnan
-        if not isnan(obj):
-            return AbstractUnwrappedStrategy._safe_find(self, w_list, obj,
-                                                        start, stop)
-        # unwrapped nan != nan, finding it requires more effort
-        l = self.unerase(w_list.lstorage)
-        for i in range(start, min(stop, len(l))):
-            if isnan(l[i]):
-                return i
-        raise ValueError
-
 
 class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = None
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1314,10 +1314,6 @@
         non_list = NonList()
         assert [] != non_list
 
-    def test_nan_containment(self):
-        nan = float('nan')
-        assert nan in [nan]
-
 
 class AppTestForRangeLists(AppTestW_ListObject):
     spaceconfig = {"objspace.std.withrangelist": True}

From noreply at buildbot.pypy.org  Wed Jun 26 21:56:38 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 26 Jun 2013 21:56:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: support operationerrfmt sans fmt args
 (like plain OperationError) as a
Message-ID: <20130626195638.04DA21C0325@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r65009:43ce1aa7ddcd
Date: 2013-06-26 12:35 -0700
http://bitbucket.org/pypy/pypy/changeset/43ce1aa7ddcd/

Log:	support operationerrfmt sans fmt args (like plain OperationError) as
	a convenience API

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -29,12 +29,12 @@
     _application_traceback = None
 
     def __init__(self, w_type, w_value, tb=None):
-        assert w_type is not None
         self.setup(w_type)
         self._w_value = w_value
         self._application_traceback = tb
 
     def setup(self, w_type):
+        assert w_type is not None
         self.w_type = w_type
         if not we_are_translated():
             self.debug_excs = []
@@ -347,7 +347,6 @@
                 self.xstrings = strings
                 for i, _, attr in entries:
                     setattr(self, attr, args[i])
-                assert w_type is not None
 
             def _compute_value(self, space):
                 lst = [None] * (len(formats) + len(formats) + 1)
@@ -369,6 +368,18 @@
         _fmtcache2[formats] = OpErrFmt
     return OpErrFmt, strings
 
+class OpErrFmtNoArgs(OperationError):
+
+    def __init__(self, w_type, value):
+        self.setup(w_type)
+        self._value = value
+
+    def get_w_value(self, space):
+        w_value = self._w_value
+        if w_value is None:
+            self._w_value = w_value = space.wrap(self._value)
+        return w_value
+
 def get_operationerr_class(valuefmt):
     try:
         result = _fmtcache[valuefmt]
@@ -389,6 +400,8 @@
     %T - The result of space.type(w_arg).getname(space)
 
     """
+    if not len(args):
+        return OpErrFmtNoArgs(w_type, valuefmt)
     OpErrFmt, strings = get_operationerr_class(valuefmt)
     return OpErrFmt(w_type, strings, *args)
 operationerrfmt._annspecialcase_ = 'specialize:arg(1)'
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -33,6 +33,14 @@
     operr3 = operationerrfmt("w_type2", "a %s b %s c", "bar", "4b")
     assert operr3.__class__ is not operr.__class__
 
+def test_operationerrfmt_noargs(space):
+    operr = operationerrfmt(space.w_AttributeError, "no attribute 'foo'")
+    operr.normalize_exception(space)
+    val = operr.get_w_value(space)
+    assert space.isinstance_w(val, space.w_AttributeError)
+    w_repr = space.repr(val)
+    assert space.str_w(w_repr) == "AttributeError(\"no attribute 'foo'\",)"
+
 def test_operationerrfmt_T(space):
     operr = operationerrfmt(space.w_AttributeError,
                             "'%T' object has no attribute '%s'",
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -225,8 +225,8 @@
             space.raise_key_error(w_key)
 
     def descr_reversed(self, space):
-        raise OperationError(space.w_TypeError, space.wrap(
-                'argument to reversed() must be a sequence'))
+        raise operationerrfmt(space.w_TypeError,
+                              'argument to reversed() must be a sequence')
 
     def descr_copy(self, space):
         """D.copy() -> a shallow copy of D"""

From noreply at buildbot.pypy.org  Wed Jun 26 21:56:39 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 26 Jun 2013 21:56:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20130626195639.5A1851C0325@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r65010:f9806d05ed7c
Date: 2013-06-26 12:53 -0700
http://bitbucket.org/pypy/pypy/changeset/f9806d05ed7c/

Log:	merge default

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -30,12 +30,12 @@
     w_cause = None
 
     def __init__(self, w_type, w_value, tb=None, w_cause=None):
-        assert w_type is not None
         self.setup(w_type, w_value)
         self._application_traceback = tb
         self.w_cause = w_cause
 
     def setup(self, w_type, w_value=None):
+        assert w_type is not None
         from pypy.objspace.std.typeobject import W_TypeObject
         self.w_type = w_type
         self._w_value = w_value
@@ -395,7 +395,6 @@
                 self.xstrings = strings
                 for i, _, attr in entries:
                     setattr(self, attr, args[i])
-                assert w_type is not None
                 self.setup(w_type)
 
             def _compute_value(self, space):
@@ -422,6 +421,18 @@
         _fmtcache2[formats] = OpErrFmt
     return OpErrFmt, strings
 
+class OpErrFmtNoArgs(OperationError):
+
+    def __init__(self, w_type, value):
+        self.setup(w_type)
+        self._value = value
+
+    def get_w_value(self, space):
+        w_value = self._w_value
+        if w_value is None:
+            self._w_value = w_value = space.wrap(self._value)
+        return w_value
+
 def get_operationerr_class(valuefmt):
     try:
         result = _fmtcache[valuefmt]
@@ -444,6 +455,8 @@
     %T - The result of space.type(w_arg).getname(space)
 
     """
+    if not len(args):
+        return OpErrFmtNoArgs(w_type, valuefmt)
     OpErrFmt, strings = get_operationerr_class(valuefmt)
     return OpErrFmt(w_type, strings, *args)
 operationerrfmt._annspecialcase_ = 'specialize:arg(1)'
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -37,6 +37,14 @@
     operr3 = operationerrfmt("w_type2", "a %s b %s c", "bar", "4b")
     assert operr3.__class__ is not operr.__class__
 
+def test_operationerrfmt_noargs(space):
+    operr = operationerrfmt(space.w_AttributeError, "no attribute 'foo'")
+    operr.normalize_exception(space)
+    val = operr.get_w_value(space)
+    assert space.isinstance_w(val, space.w_AttributeError)
+    w_repr = space.repr(val)
+    assert space.str_w(w_repr) == "AttributeError(\"no attribute 'foo'\",)"
+
 def test_operationerrfmt_T(space):
     operr = operationerrfmt(space.w_AttributeError,
                             "'%T' object has no attribute '%s'",
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -191,8 +191,8 @@
             space.raise_key_error(w_key)
 
     def descr_reversed(self, space):
-        raise OperationError(space.w_TypeError, space.wrap(
-                'argument to reversed() must be a sequence'))
+        raise operationerrfmt(space.w_TypeError,
+                              'argument to reversed() must be a sequence')
 
     def descr_copy(self, space):
         """D.copy() -> a shallow copy of D"""
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1201,7 +1201,8 @@
     def _safe_find(self, w_list, obj, start, stop):
         l = self.unerase(w_list.lstorage)
         for i in range(start, min(stop, len(l))):
-            if l[i] == obj:
+            val = l[i]
+            if val == obj:
                 return i
         raise ValueError
 
@@ -1521,18 +1522,6 @@
         if reverse:
             l.reverse()
 
-    def _safe_find(self, w_list, obj, start, stop):
-        from rpython.rlib.rfloat import isnan
-        if not isnan(obj):
-            return AbstractUnwrappedStrategy._safe_find(self, w_list, obj,
-                                                        start, stop)
-        # unwrapped nan != nan, finding it requires more effort
-        l = self.unerase(w_list.lstorage)
-        for i in range(start, min(stop, len(l))):
-            if isnan(l[i]):
-                return i
-        raise ValueError
-
 
 class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = None
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1274,10 +1274,6 @@
         non_list = NonList()
         assert [] != non_list
 
-    def test_nan_containment(self):
-        nan = float('nan')
-        assert nan in [nan]
-
     def test_issue1266(self):
         l = list(range(1))
         l.pop()
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -351,7 +351,8 @@
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
         self.producer = {}
-        self.pendingfields = []
+        self.pendingfields = None # set temporarily to a list, normally by
+                                  # heap.py, as we're about to generate a guard
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.replaces_guard = {}
@@ -546,12 +547,14 @@
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
+            pendingfields = self.pendingfields
+            self.pendingfields = None
             if self.replaces_guard and op in self.replaces_guard:
                 self.replace_op(self.replaces_guard[op], op)
                 del self.replaces_guard[op]
                 return
             else:
-                op = self.store_final_boxes_in_guard(op)
+                op = self.store_final_boxes_in_guard(op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
         if op.result:
@@ -571,12 +574,13 @@
         else:
             assert False
 
-    def store_final_boxes_in_guard(self, op):
+    def store_final_boxes_in_guard(self, op, pendingfields):
+        assert pendingfields is not None
         descr = op.getdescr()
         assert isinstance(descr, compile.ResumeGuardDescr)
         modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
         try:
-            newboxes = modifier.finish(self, self.pendingfields)
+            newboxes = modifier.finish(self, pendingfields)
             if len(newboxes) > self.metainterp_sd.options.failargs_limit:
                 raise resume.TagOverflow
         except resume.TagOverflow:
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -7,7 +7,13 @@
     def __init__(self, unroll):
         self.last_label_descr = None
         self.unroll = unroll
-        
+
+    def emit_operation(self, op):
+        if op.is_guard():
+            if self.optimizer.pendingfields is None:
+                self.optimizer.pendingfields = []
+        Optimization.emit_operation(self, op)
+
     def optimize_CALL_PURE(self, op):
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -27,7 +27,7 @@
     snapshot0 = resume.Snapshot(None, [b0])
     fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
     #
-    opt.store_final_boxes_in_guard(op)
+    opt.store_final_boxes_in_guard(op, [])
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]

From noreply at buildbot.pypy.org  Wed Jun 26 21:56:40 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 26 Jun 2013 21:56:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: consider nan containment as an impl detail
Message-ID: <20130626195640.8EEC41C0325@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r65011:c3ca88e6e231
Date: 2013-06-26 12:55 -0700
http://bitbucket.org/pypy/pypy/changeset/c3ca88e6e231/

Log:	consider nan containment as an impl detail

diff --git a/lib-python/3/test/test_float.py b/lib-python/3/test/test_float.py
--- a/lib-python/3/test/test_float.py
+++ b/lib-python/3/test/test_float.py
@@ -170,7 +170,8 @@
     def test_float_containment(self):
         floats = (INF, -INF, 0.0, 1.0, NAN)
         for f in floats:
-            self.assertIn(f, [f])
+            if f is NAN and support.check_impl_detail(pypy=False):
+                self.assertIn(f, [f])
             self.assertIn(f, (f,))
             self.assertIn(f, {f})
             self.assertIn(f, {f: None})

From noreply at buildbot.pypy.org  Wed Jun 26 22:17:48 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 22:17:48 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Divide the nursery into sections that
 are cleared incrementally, 
Message-ID: <20130626201748.609871C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r293:9bb735eaca1a
Date: 2013-06-26 22:17 +0200
http://bitbucket.org/pypy/stmgc/changeset/9bb735eaca1a/

Log:	Divide the nursery into sections that are cleared incrementally,
	like in minimark.py. This gives a code reorganization that makes it
	easy to implement the real goal: stm_allocate() of objects that are
	very large (here defined as "more than one section large").

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -586,10 +586,7 @@
       assert(is_private(W));
 
       if (W->h_tid & GCFLAG_OLD)
-        {
-          W->h_tid |= GCFLAG_WRITE_BARRIER;
-          record_write_barrier(W);
-        }
+        gcptrlist_insert(&d->old_objects_to_trace, W);
       else
         gcptrlist_insert(&d->public_with_young_copy, R);
     }
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -24,11 +24,13 @@
 {
     struct tx_descriptor *d = thread_descriptor;
 
+    assert(GC_NURSERY % GC_NURSERY_SECTION == 0);
+
     assert(d->nursery_base == NULL);
-    d->nursery_base = stm_malloc(GC_NURSERY);
-    memset(d->nursery_base, 0, GC_NURSERY);
-    d->nursery_end = d->nursery_base + GC_NURSERY;
-    d->nursery_current = d->nursery_base;
+    d->nursery_base = stm_malloc(GC_NURSERY);       /* start of nursery */
+    d->nursery_end = d->nursery_base + GC_NURSERY;  /* end of nursery */
+    d->nursery_current = d->nursery_base;           /* current position */
+    d->nursery_nextlimit = d->nursery_base;         /* next section limit */
 
     dprintf(("minor: nursery is at [%p to %p]\n", d->nursery_base,
              d->nursery_end));
@@ -50,32 +52,31 @@
     d->nursery_current = d->nursery_end;
 }
 
-static char *collect_and_allocate_size(size_t size);  /* forward */
+static gcptr allocate_next_section(size_t size, revision_t tid);  /* forward */
 
-inline static char *allocate_nursery(size_t size, int can_collect)
+inline static gcptr allocate_nursery(size_t size, revision_t tid)
 {
+    /* if 'tid == -1', we must not collect */
     struct tx_descriptor *d = thread_descriptor;
+    gcptr P;
     char *cur = d->nursery_current;
     char *end = cur + size;
     d->nursery_current = end;
-    if (end > d->nursery_end) {
-        if (can_collect) {
-            cur = collect_and_allocate_size(size);
-        }
-        else {
-            d->nursery_current = cur;
-            cur = NULL;
-        }
+    if (end > d->nursery_nextlimit) {
+        P = allocate_next_section(size, tid);
     }
-    return cur;
+    else {
+        P = (gcptr)cur;
+        P->h_tid = tid;
+    }
+    return P;
 }
 
 gcptr stm_allocate(size_t size, unsigned long tid)
 {
     /* XXX inline the fast path */
-    gcptr P = (gcptr)allocate_nursery(size, 1);
     assert(tid == (tid & STM_USER_TID_MASK));
-    P->h_tid = tid;
+    gcptr P = allocate_nursery(size, tid);
     P->h_revision = stm_private_rev_num;
     P->h_original = 0;
     return P;
@@ -84,7 +85,7 @@
 gcptr stmgc_duplicate(gcptr P)
 {
     size_t size = stmgc_size(P);
-    gcptr L = (gcptr)allocate_nursery(size, 0);
+    gcptr L = allocate_nursery(size, -1);
     if (L == NULL)
         return stmgc_duplicate_old(P);
 
@@ -531,19 +532,19 @@
     */
     teardown_minor_collect(d);
 
-    /* clear the nursery */
+    /* if in debugging mode, we allocate a different nursery and make
+       the old one inaccessible
+    */
 #if defined(_GC_DEBUG) && _GC_DEBUG >= 2
     stm_free(d->nursery_base, GC_NURSERY);
     d->nursery_base = stm_malloc(GC_NURSERY);
-    memset(d->nursery_base, 0, GC_NURSERY);
     d->nursery_end = d->nursery_base + GC_NURSERY;
     dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
              d->nursery_end));
-#else
-    memset(d->nursery_base, 0,
-           d->nursery_current - d->nursery_base);
 #endif
+
     d->nursery_current = d->nursery_base;
+    d->nursery_nextlimit = d->nursery_base;
 
     assert(!stmgc_minor_collect_anything_to_do(d));
 }
@@ -584,17 +585,66 @@
     }
 }
 
-static char *collect_and_allocate_size(size_t allocate_size)
+static gcptr allocate_next_section(size_t allocate_size, revision_t tid)
 {
-    stmgc_minor_collect();
-    //stmgcpage_possibly_major_collect(0);
+    /* This is called when the next allocation request hits
+       'nursery_nextlimit', which points to the next multiple of
+       GC_NURSERY_SECTION bytes in the nursery.
 
+       'tid' is the value to store in the h_tid of the result,
+       or if it's equal to -1, it means we must not collect.
+
+       First fix 'nursery_current', left to a bogus value by the caller.
+    */
     struct tx_descriptor *d = thread_descriptor;
-    assert(d->nursery_current == d->nursery_base);
+    d->nursery_current -= allocate_size;
 
-    //_debug_roots(d->shadowstack, *d->shadowstack_end_ref);
+    /* Are we asking for a "reasonable" number of bytes, i.e. a value
+       at most equal to one section?
+    */
+    if (allocate_size > GC_NURSERY_SECTION) {
+        /* No */
+        if (tid == -1)
+            return NULL;    /* cannot collect */
 
-    d->nursery_current = d->nursery_base + allocate_size;
-    assert(d->nursery_current <= d->nursery_end);  /* XXX object too big */
-    return d->nursery_base;
+        /* Allocate it externally, and make it old */
+        gcptr P = stmgcpage_malloc(allocate_size);
+        P->h_tid = tid | GCFLAG_OLD;
+        gcptrlist_insert(&d->old_objects_to_trace, P);
+        return P;
+    }
+
+    revision_t clear_section_count = GC_NURSERY_SECTION;
+
+    /* Are we at the end of the nursery? */
+    if (d->nursery_nextlimit == d->nursery_end) {
+        /* Yes */
+        if (tid == -1)
+            return NULL;    /* cannot collect */
+
+        /* Start a minor collection
+         */
+        if (d->nursery_current - d->nursery_base < clear_section_count) {
+            /* help cases where we do a large amount of minor collections */
+            clear_section_count = d->nursery_current - d->nursery_base;
+        }
+
+        stmgc_minor_collect();
+        stmgcpage_possibly_major_collect(0);
+
+        assert(d->nursery_current == d->nursery_base);
+        assert(d->nursery_nextlimit == d->nursery_base);
+    }
+
+    /* Clear the next section */
+    memset(d->nursery_nextlimit, 0, clear_section_count);
+    d->nursery_nextlimit += GC_NURSERY_SECTION;
+
+    /* Return the object from there */
+    gcptr P = (gcptr)d->nursery_current;
+    d->nursery_current += allocate_size;
+    assert(d->nursery_current <= d->nursery_nextlimit);
+
+    P->h_tid = tid;
+    return P;
 }
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -6,10 +6,19 @@
 //#define GC_NURSERY        (1<<20)    /* 1 MB */
 #endif
 
+#ifndef GC_NURSERY_SECTION
+# if GC_NURSERY >= 2 * 135168
+#  define GC_NURSERY_SECTION    135168
+# else
+#  define GC_NURSERY_SECTION    (GC_NURSERY / 2)
+# endif
+#endif
+
 
 #define NURSERY_FIELDS_DECL                                             \
     /* the nursery */                                                   \
     char *nursery_current;                                              \
+    char *nursery_nextlimit;                                            \
     char *nursery_end;                                                  \
     char *nursery_base;                                                 \
                                                                         \
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -1,5 +1,10 @@
 import os, cffi, thread, sys
 
+if sys.maxint > 2**32:
+    WORD = 8
+else:
+    WORD = 4
+
 # ----------
 
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -103,6 +108,7 @@
     /* some constants normally private that are useful in the tests */
     #define WORD                     ...
     #define GC_PAGE_SIZE             ...
+    #define GC_NURSERY_SECTION       ...
     #define GCFLAG_OLD               ...
     #define GCFLAG_VISITED           ...
     #define GCFLAG_PUBLIC            ...
@@ -276,7 +282,7 @@
     }
 '''.lstrip(),    include_dirs=[parent_dir],
                  undef_macros=['NDEBUG'],
-                 define_macros=[('GC_NURSERY', '(16 * sizeof(void *))'),
+                 define_macros=[('GC_NURSERY', str(16 * WORD)),
                                 ('_GC_DEBUG', '2'),
                                 ('GC_PAGE_SIZE', '1000'),
                                 ('GC_MIN', '200000'),
@@ -288,7 +294,7 @@
                  extra_compile_args=['-g', '-O0'])
 
 HDR = ffi.sizeof("struct stm_object_s")
-WORD = lib.WORD
+assert WORD == lib.WORD
 PAGE_ROOM = lib.GC_PAGE_SIZE - ffi.sizeof("page_header_t")
 for name in lib.__dict__:
     if name.startswith('GCFLAG_') or name.startswith('PREBUILT_'):
@@ -470,7 +476,10 @@
 def nalloc(size):
     "Allocate a fresh object from the nursery"
     p = lib.stm_allocate(size, 42 + size)
-    assert p.h_tid == 42 + size     # no GC flags
+    if size <= lib.GC_NURSERY_SECTION:
+        assert p.h_tid == 42 + size     # no GC flags
+    else:
+        assert p.h_tid == 42 + size + GCFLAG_OLD
     assert p.h_revision == lib.get_private_rev_num()
     return p
 
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -296,3 +296,8 @@
     assert not lib.in_nursery(p1)
     p2 = lib.getptr(p1, 0)
     assert lib.getlong(p2, 0) == 389719
+
+def test_nalloc_large_object():
+    for words in range(80):
+        p1 = nalloc(HDR + words * WORD)
+    # assert did not crash

From noreply at buildbot.pypy.org  Wed Jun 26 22:37:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 22:37:59 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix this test
Message-ID: <20130626203759.DE9A11C34D1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r295:1cdc5217622b
Date: 2013-06-26 22:33 +0200
http://bitbucket.org/pypy/stmgc/changeset/1cdc5217622b/

Log:	Fix this test

diff --git a/duhton/test/test_misc.py b/duhton/test/test_misc.py
--- a/duhton/test/test_misc.py
+++ b/duhton/test/test_misc.py
@@ -2,12 +2,12 @@
 
 
 def test_type():
-    assert run("(print (type 42))") == "<type 'int'>\n"
-    assert run("(print (type (+ 40 2)))") == "<type 'int'>\n"
+    assert run("(print (type 42))") == "'int'\n"
+    assert run("(print (type (+ 40 2)))") == "'int'\n"
 
 def test_quote():
     assert run("(print (quote (+ 40 2)))") == "( '+' 40 2 )\n"
-    assert run("(print (type (quote (+ 40 2))))") == "<type 'cons'>\n"
+    assert run("(print (type (quote (+ 40 2))))") == "'cons'\n"
 
 def test_if():
     assert evaluate("(if 5 6 7)") == 6

From noreply at buildbot.pypy.org  Wed Jun 26 22:37:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 22:37:58 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Start to fix the tests
Message-ID: <20130626203758.BD8FE1C34CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r294:d99155033db1
Date: 2013-06-26 22:31 +0200
http://bitbucket.org/pypy/stmgc/changeset/d99155033db1/

Log:	Start to fix the tests

diff --git a/duhton/frame.c b/duhton/frame.c
--- a/duhton/frame.c
+++ b/duhton/frame.c
@@ -106,12 +106,14 @@
     struct dictentry *entries = ob->ob_items;
     revision_t search_id = stm_id(symbol);
 
+#if 0
 #ifdef _GC_DEBUG
     int j;
     for (j = 0; j < right; j++) {
         dprintf(("\t%p\n", (gcptr)entries[j].symbol_id));
     }
 #endif
+#endif
 
     while (right > left) {
         int middle = (left + right) / 2;
diff --git a/duhton/test/support.py b/duhton/test/support.py
--- a/duhton/test/support.py
+++ b/duhton/test/support.py
@@ -1,4 +1,4 @@
-import py
+import py, re
 import os, subprocess
 from cStringIO import StringIO
 
@@ -18,10 +18,13 @@
     if exitcode:
         raise OSError("%r failed (exit code %r)\n" % (cmdargs[0], exitcode) +
                       error.rstrip())
-    if error:
+    if filter_out_colored_output(error):
         raise OSError("%r got on stderr:\n" % (cmdargs[0],) + error.rstrip())
     return result
 
+def filter_out_colored_output(text):
+    return re.sub('\x1b\\[3.m[\w\W]*?\x1b\\[0m', '', text)
+
 
 def execute(argv, stdin=''):
     global _compiled

From noreply at buildbot.pypy.org  Wed Jun 26 22:38:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Jun 2013 22:38:01 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Duh,
	the default values need to be multiples of 135168.
Message-ID: <20130626203801.0351D1C34D2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r296:6f81132a79c5
Date: 2013-06-26 22:37 +0200
http://bitbucket.org/pypy/stmgc/changeset/6f81132a79c5/

Log:	Duh, the default values need to be multiples of 135168.

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -24,8 +24,6 @@
 {
     struct tx_descriptor *d = thread_descriptor;
 
-    assert(GC_NURSERY % GC_NURSERY_SECTION == 0);
-
     assert(d->nursery_base == NULL);
     d->nursery_base = stm_malloc(GC_NURSERY);       /* start of nursery */
     d->nursery_end = d->nursery_base + GC_NURSERY;  /* end of nursery */
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -2,7 +2,7 @@
 #define _SRCSTM_NURSERY_H
 
 #ifndef GC_NURSERY
-#define GC_NURSERY        4194304    /* 4 MB */
+#define GC_NURSERY        4190208    /* 4 MB - 4 kb */
 //#define GC_NURSERY        (1<<20)    /* 1 MB */
 #endif
 
@@ -14,6 +14,10 @@
 # endif
 #endif
 
+#if GC_NURSERY % GC_NURSERY_SECTION != 0
+# error "GC_NURSERY must be a multiple of GC_NURSERY_SECTION"
+#endif
+
 
 #define NURSERY_FIELDS_DECL                                             \
     /* the nursery */                                                   \

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:02 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:02 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: A branch to create two sets
	of AST classes:
Message-ID: <20130626212202.902971C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65012:8dfc9edcb17e
Date: 2013-06-20 22:43 +0200
http://bitbucket.org/pypy/pypy/changeset/8dfc9edcb17e/

Log:	A branch to create two sets of AST classes:
	- one set is interp-level, and used by the compiler.
	- another set is exposed in the _ast module, and more flexible.


From noreply at buildbot.pypy.org  Wed Jun 26 23:22:03 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:03 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Remove all Python-level code
 from the ast classes, 
Message-ID: <20130626212203.EF8D41C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65013:7356f3a68d37
Date: 2013-06-21 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/7356f3a68d37/

Log:	Remove all Python-level code from the ast classes, only keep the
	minimum for the compiler.

	The ast module is broken, of course, but the interpreter works.

diff too long, truncating to 2000 out of 6858 lines

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1,25 +1,9 @@
 # Generated by tools/asdl_py.py
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter import typedef
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError, operationerrfmt
-from rpython.rlib.unroll import unrolling_iterable
+from pypy.interpreter.error import OperationError
 from rpython.tool.pairtype import extendabletype
-from rpython.tool.sourcetools import func_with_new_name
 
 
-def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
-            space.isinstance_w(w_obj, space.w_unicode)):
-        raise OperationError(space.w_TypeError, space.wrap(
-                'AST string must be of type str or unicode'))
-    return w_obj
-
-
-class AST(W_Root):
-
-    w_dict = None
-
+class AST(object):
     __metaclass__ = extendabletype
 
     def walkabout(self, visitor):
@@ -28,100 +12,10 @@
     def mutate_over(self, visitor):
         raise AssertionError("mutate_over() implementation not provided")
 
-    def sync_app_attrs(self, space):
-        raise NotImplementedError
-
-    def getdict(self, space):
-        if self.w_dict is None:
-            self.w_dict = space.newdict(instance=True)
-        return self.w_dict
-
-    def reduce_w(self, space):
-        w_dict = self.w_dict
-        if w_dict is None:
-            w_dict = space.newdict()
-        w_type = space.type(self)
-        w_fields = w_type.getdictvalue(space, "_fields")
-        for w_name in space.fixedview(w_fields):
-            space.setitem(w_dict, w_name,
-                          space.getattr(self, w_name))
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
-        if w_attrs:
-            for w_name in space.fixedview(w_attrs):
-                space.setitem(w_dict, w_name,
-                              space.getattr(self, w_name))
-        return space.newtuple([space.type(self),
-                               space.newtuple([]),
-                               w_dict])
-
-    def setstate_w(self, space, w_state):
-        for w_name in space.unpackiterable(w_state):
-            space.setattr(self, w_name,
-                          space.getitem(w_state, w_name))
-
-    def missing_field(self, space, required, host):
-        "Find which required field is missing."
-        state = self.initialization_state
-        for i in range(len(required)):
-            if (state >> i) & 1:
-                continue  # field is present
-            missing = required[i]
-            if missing is None:
-                continue  # field is optional
-            w_obj = self.getdictvalue(space, missing)
-            if w_obj is None:
-                err = "required field \"%s\" missing from %s"
-                raise operationerrfmt(space.w_TypeError, err, missing, host)
-            else:
-                err = "incorrect type for field \"%s\" in %s"
-                raise operationerrfmt(space.w_TypeError, err, missing, host)
-        raise AssertionError("should not reach here")
-
 
 class NodeVisitorNotImplemented(Exception):
     pass
 
-
-class _FieldsWrapper(W_Root):
-    "Hack around the fact we can't store tuples on a TypeDef."
-
-    def __init__(self, fields):
-        self.fields = fields
-
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
-
-
-def get_AST_new(node_class):
-    def generic_AST_new(space, w_type, __args__):
-        node = space.allocate_instance(node_class, w_type)
-        node.initialization_state = 0
-        return space.wrap(node)
-    return func_with_new_name(generic_AST_new, "new_%s" % node_class.__name__)
-
-def AST_init(space, w_self, __args__):
-    args_w, kwargs_w = __args__.unpack()
-    if args_w and len(args_w) != 0:
-        w_err = space.wrap("_ast.AST constructor takes 0 positional arguments")
-        raise OperationError(space.w_TypeError, w_err)
-    for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
-
-AST.typedef = typedef.TypeDef("AST",
-    _fields=_FieldsWrapper([]),
-    _attributes=_FieldsWrapper([]),
-    __module__='_ast',
-    __reduce__=interp2app(AST.reduce_w),
-    __setstate__=interp2app(AST.setstate_w),
-    __dict__ = typedef.GetSetProperty(typedef.descr_get_dict,
-                                      typedef.descr_set_dict, cls=AST),
-    __new__=interp2app(get_AST_new(AST)),
-    __init__=interp2app(AST_init),
-)
-
-
-
-
 class mod(AST):
     pass
 
@@ -129,8 +23,6 @@
 
     def __init__(self, body):
         self.body = body
-        self.w_body = None
-        self.initialization_state = 1
 
     def walkabout(self, visitor):
         visitor.visit_Module(self)
@@ -140,29 +32,11 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_Module(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 1:
-            self.missing_field(space, ['body'], 'Module')
-        else:
-            pass
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-
 
 class Interactive(mod):
 
     def __init__(self, body):
         self.body = body
-        self.w_body = None
-        self.initialization_state = 1
 
     def walkabout(self, visitor):
         visitor.visit_Interactive(self)
@@ -172,28 +46,11 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_Interactive(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 1:
-            self.missing_field(space, ['body'], 'Interactive')
-        else:
-            pass
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-
 
 class Expression(mod):
 
     def __init__(self, body):
         self.body = body
-        self.initialization_state = 1
 
     def walkabout(self, visitor):
         visitor.visit_Expression(self)
@@ -202,20 +59,11 @@
         self.body = self.body.mutate_over(visitor)
         return visitor.visit_Expression(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 1:
-            self.missing_field(space, ['body'], 'Expression')
-        else:
-            pass
-        self.body.sync_app_attrs(space)
-
 
 class Suite(mod):
 
     def __init__(self, body):
         self.body = body
-        self.w_body = None
-        self.initialization_state = 1
 
     def walkabout(self, visitor):
         visitor.visit_Suite(self)
@@ -225,22 +73,6 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_Suite(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 1:
-            self.missing_field(space, ['body'], 'Suite')
-        else:
-            pass
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-
 
 class stmt(AST):
 
@@ -254,11 +86,8 @@
         self.name = name
         self.args = args
         self.body = body
-        self.w_body = None
         self.decorator_list = decorator_list
-        self.w_decorator_list = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 63
 
     def walkabout(self, visitor):
         visitor.visit_FunctionDef(self)
@@ -271,46 +100,15 @@
             visitor._mutate_sequence(self.decorator_list)
         return visitor.visit_FunctionDef(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 63:
-            self.missing_field(space, ['lineno', 'col_offset', 'name', 'args', 'body', 'decorator_list'], 'FunctionDef')
-        else:
-            pass
-        self.args.sync_app_attrs(space)
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-        w_list = self.w_decorator_list
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.decorator_list = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.decorator_list = None
-        if self.decorator_list is not None:
-            for node in self.decorator_list:
-                node.sync_app_attrs(space)
-
 
 class ClassDef(stmt):
 
     def __init__(self, name, bases, body, decorator_list, lineno, col_offset):
         self.name = name
         self.bases = bases
-        self.w_bases = None
         self.body = body
-        self.w_body = None
         self.decorator_list = decorator_list
-        self.w_decorator_list = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 63
 
     def walkabout(self, visitor):
         visitor.visit_ClassDef(self)
@@ -324,49 +122,12 @@
             visitor._mutate_sequence(self.decorator_list)
         return visitor.visit_ClassDef(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 63:
-            self.missing_field(space, ['lineno', 'col_offset', 'name', 'bases', 'body', 'decorator_list'], 'ClassDef')
-        else:
-            pass
-        w_list = self.w_bases
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.bases = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.bases = None
-        if self.bases is not None:
-            for node in self.bases:
-                node.sync_app_attrs(space)
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-        w_list = self.w_decorator_list
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.decorator_list = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.decorator_list = None
-        if self.decorator_list is not None:
-            for node in self.decorator_list:
-                node.sync_app_attrs(space)
-
 
 class Return(stmt):
 
     def __init__(self, value, lineno, col_offset):
         self.value = value
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Return(self)
@@ -376,23 +137,12 @@
             self.value = self.value.mutate_over(visitor)
         return visitor.visit_Return(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~4) ^ 3:
-            self.missing_field(space, ['lineno', 'col_offset', None], 'Return')
-        else:
-            if not self.initialization_state & 4:
-                self.value = None
-        if self.value:
-            self.value.sync_app_attrs(space)
-
 
 class Delete(stmt):
 
     def __init__(self, targets, lineno, col_offset):
         self.targets = targets
-        self.w_targets = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Delete(self)
@@ -402,31 +152,13 @@
             visitor._mutate_sequence(self.targets)
         return visitor.visit_Delete(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'targets'], 'Delete')
-        else:
-            pass
-        w_list = self.w_targets
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.targets = None
-        if self.targets is not None:
-            for node in self.targets:
-                node.sync_app_attrs(space)
-
 
 class Assign(stmt):
 
     def __init__(self, targets, value, lineno, col_offset):
         self.targets = targets
-        self.w_targets = None
         self.value = value
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_Assign(self)
@@ -437,23 +169,6 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Assign(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'targets', 'value'], 'Assign')
-        else:
-            pass
-        w_list = self.w_targets
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.targets = None
-        if self.targets is not None:
-            for node in self.targets:
-                node.sync_app_attrs(space)
-        self.value.sync_app_attrs(space)
-
 
 class AugAssign(stmt):
 
@@ -462,7 +177,6 @@
         self.op = op
         self.value = value
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_AugAssign(self)
@@ -472,24 +186,14 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_AugAssign(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'target', 'op', 'value'], 'AugAssign')
-        else:
-            pass
-        self.target.sync_app_attrs(space)
-        self.value.sync_app_attrs(space)
-
 
 class Print(stmt):
 
     def __init__(self, dest, values, nl, lineno, col_offset):
         self.dest = dest
         self.values = values
-        self.w_values = None
         self.nl = nl
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_Print(self)
@@ -501,25 +205,6 @@
             visitor._mutate_sequence(self.values)
         return visitor.visit_Print(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~4) ^ 27:
-            self.missing_field(space, ['lineno', 'col_offset', None, 'values', 'nl'], 'Print')
-        else:
-            if not self.initialization_state & 4:
-                self.dest = None
-        if self.dest:
-            self.dest.sync_app_attrs(space)
-        w_list = self.w_values
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.values = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.values = None
-        if self.values is not None:
-            for node in self.values:
-                node.sync_app_attrs(space)
-
 
 class For(stmt):
 
@@ -527,11 +212,8 @@
         self.target = target
         self.iter = iter
         self.body = body
-        self.w_body = None
         self.orelse = orelse
-        self.w_orelse = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 63
 
     def walkabout(self, visitor):
         visitor.visit_For(self)
@@ -545,45 +227,14 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_For(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 63:
-            self.missing_field(space, ['lineno', 'col_offset', 'target', 'iter', 'body', 'orelse'], 'For')
-        else:
-            pass
-        self.target.sync_app_attrs(space)
-        self.iter.sync_app_attrs(space)
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-        w_list = self.w_orelse
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.orelse = None
-        if self.orelse is not None:
-            for node in self.orelse:
-                node.sync_app_attrs(space)
-
 
 class While(stmt):
 
     def __init__(self, test, body, orelse, lineno, col_offset):
         self.test = test
         self.body = body
-        self.w_body = None
         self.orelse = orelse
-        self.w_orelse = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_While(self)
@@ -596,44 +247,14 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_While(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'While')
-        else:
-            pass
-        self.test.sync_app_attrs(space)
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-        w_list = self.w_orelse
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.orelse = None
-        if self.orelse is not None:
-            for node in self.orelse:
-                node.sync_app_attrs(space)
-
 
 class If(stmt):
 
     def __init__(self, test, body, orelse, lineno, col_offset):
         self.test = test
         self.body = body
-        self.w_body = None
         self.orelse = orelse
-        self.w_orelse = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_If(self)
@@ -646,33 +267,6 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_If(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'If')
-        else:
-            pass
-        self.test.sync_app_attrs(space)
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-        w_list = self.w_orelse
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.orelse = None
-        if self.orelse is not None:
-            for node in self.orelse:
-                node.sync_app_attrs(space)
-
 
 class With(stmt):
 
@@ -680,9 +274,7 @@
         self.context_expr = context_expr
         self.optional_vars = optional_vars
         self.body = body
-        self.w_body = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_With(self)
@@ -695,26 +287,6 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_With(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~8) ^ 23:
-            self.missing_field(space, ['lineno', 'col_offset', 'context_expr', None, 'body'], 'With')
-        else:
-            if not self.initialization_state & 8:
-                self.optional_vars = None
-        self.context_expr.sync_app_attrs(space)
-        if self.optional_vars:
-            self.optional_vars.sync_app_attrs(space)
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-
 
 class Raise(stmt):
 
@@ -723,7 +295,6 @@
         self.inst = inst
         self.tback = tback
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_Raise(self)
@@ -737,35 +308,14 @@
             self.tback = self.tback.mutate_over(visitor)
         return visitor.visit_Raise(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~28) ^ 3:
-            self.missing_field(space, ['lineno', 'col_offset', None, None, None], 'Raise')
-        else:
-            if not self.initialization_state & 4:
-                self.type = None
-            if not self.initialization_state & 8:
-                self.inst = None
-            if not self.initialization_state & 16:
-                self.tback = None
-        if self.type:
-            self.type.sync_app_attrs(space)
-        if self.inst:
-            self.inst.sync_app_attrs(space)
-        if self.tback:
-            self.tback.sync_app_attrs(space)
-
 
 class TryExcept(stmt):
 
     def __init__(self, body, handlers, orelse, lineno, col_offset):
         self.body = body
-        self.w_body = None
         self.handlers = handlers
-        self.w_handlers = None
         self.orelse = orelse
-        self.w_orelse = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_TryExcept(self)
@@ -779,52 +329,13 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_TryExcept(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'body', 'handlers', 'orelse'], 'TryExcept')
-        else:
-            pass
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-        w_list = self.w_handlers
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.handlers = [space.interp_w(excepthandler, w_obj) for w_obj in list_w]
-            else:
-                self.handlers = None
-        if self.handlers is not None:
-            for node in self.handlers:
-                node.sync_app_attrs(space)
-        w_list = self.w_orelse
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.orelse = None
-        if self.orelse is not None:
-            for node in self.orelse:
-                node.sync_app_attrs(space)
-
 
 class TryFinally(stmt):
 
     def __init__(self, body, finalbody, lineno, col_offset):
         self.body = body
-        self.w_body = None
         self.finalbody = finalbody
-        self.w_finalbody = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_TryFinally(self)
@@ -836,32 +347,6 @@
             visitor._mutate_sequence(self.finalbody)
         return visitor.visit_TryFinally(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'body', 'finalbody'], 'TryFinally')
-        else:
-            pass
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-        w_list = self.w_finalbody
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.finalbody = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.finalbody = None
-        if self.finalbody is not None:
-            for node in self.finalbody:
-                node.sync_app_attrs(space)
-
 
 class Assert(stmt):
 
@@ -869,7 +354,6 @@
         self.test = test
         self.msg = msg
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_Assert(self)
@@ -880,24 +364,12 @@
             self.msg = self.msg.mutate_over(visitor)
         return visitor.visit_Assert(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~8) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'test', None], 'Assert')
-        else:
-            if not self.initialization_state & 8:
-                self.msg = None
-        self.test.sync_app_attrs(space)
-        if self.msg:
-            self.msg.sync_app_attrs(space)
-
 
 class Import(stmt):
 
     def __init__(self, names, lineno, col_offset):
         self.names = names
-        self.w_names = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Import(self)
@@ -907,32 +379,14 @@
             visitor._mutate_sequence(self.names)
         return visitor.visit_Import(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'names'], 'Import')
-        else:
-            pass
-        w_list = self.w_names
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.names = [space.interp_w(alias, w_obj) for w_obj in list_w]
-            else:
-                self.names = None
-        if self.names is not None:
-            for node in self.names:
-                node.sync_app_attrs(space)
-
 
 class ImportFrom(stmt):
 
     def __init__(self, module, names, level, lineno, col_offset):
         self.module = module
         self.names = names
-        self.w_names = None
         self.level = level
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_ImportFrom(self)
@@ -942,25 +396,6 @@
             visitor._mutate_sequence(self.names)
         return visitor.visit_ImportFrom(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~20) ^ 11:
-            self.missing_field(space, ['lineno', 'col_offset', None, 'names', None], 'ImportFrom')
-        else:
-            if not self.initialization_state & 4:
-                self.module = None
-            if not self.initialization_state & 16:
-                self.level = 0
-        w_list = self.w_names
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.names = [space.interp_w(alias, w_obj) for w_obj in list_w]
-            else:
-                self.names = None
-        if self.names is not None:
-            for node in self.names:
-                node.sync_app_attrs(space)
-
 
 class Exec(stmt):
 
@@ -969,7 +404,6 @@
         self.globals = globals
         self.locals = locals
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_Exec(self)
@@ -982,28 +416,12 @@
             self.locals = self.locals.mutate_over(visitor)
         return visitor.visit_Exec(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~24) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'body', None, None], 'Exec')
-        else:
-            if not self.initialization_state & 8:
-                self.globals = None
-            if not self.initialization_state & 16:
-                self.locals = None
-        self.body.sync_app_attrs(space)
-        if self.globals:
-            self.globals.sync_app_attrs(space)
-        if self.locals:
-            self.locals.sync_app_attrs(space)
-
 
 class Global(stmt):
 
     def __init__(self, names, lineno, col_offset):
         self.names = names
-        self.w_names = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Global(self)
@@ -1011,26 +429,12 @@
     def mutate_over(self, visitor):
         return visitor.visit_Global(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'names'], 'Global')
-        else:
-            pass
-        w_list = self.w_names
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.names = [space.realstr_w(w_obj) for w_obj in list_w]
-            else:
-                self.names = None
-
 
 class Expr(stmt):
 
     def __init__(self, value, lineno, col_offset):
         self.value = value
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Expr(self)
@@ -1039,19 +443,11 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Expr(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Expr')
-        else:
-            pass
-        self.value.sync_app_attrs(space)
-
 
 class Pass(stmt):
 
     def __init__(self, lineno, col_offset):
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 3
 
     def walkabout(self, visitor):
         visitor.visit_Pass(self)
@@ -1059,18 +455,11 @@
     def mutate_over(self, visitor):
         return visitor.visit_Pass(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 3:
-            self.missing_field(space, ['lineno', 'col_offset'], 'Pass')
-        else:
-            pass
-
 
 class Break(stmt):
 
     def __init__(self, lineno, col_offset):
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 3
 
     def walkabout(self, visitor):
         visitor.visit_Break(self)
@@ -1078,18 +467,11 @@
     def mutate_over(self, visitor):
         return visitor.visit_Break(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 3:
-            self.missing_field(space, ['lineno', 'col_offset'], 'Break')
-        else:
-            pass
-
 
 class Continue(stmt):
 
     def __init__(self, lineno, col_offset):
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 3
 
     def walkabout(self, visitor):
         visitor.visit_Continue(self)
@@ -1097,12 +479,6 @@
     def mutate_over(self, visitor):
         return visitor.visit_Continue(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 3:
-            self.missing_field(space, ['lineno', 'col_offset'], 'Continue')
-        else:
-            pass
-
 
 class expr(AST):
 
@@ -1115,9 +491,7 @@
     def __init__(self, op, values, lineno, col_offset):
         self.op = op
         self.values = values
-        self.w_values = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_BoolOp(self)
@@ -1127,22 +501,6 @@
             visitor._mutate_sequence(self.values)
         return visitor.visit_BoolOp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'op', 'values'], 'BoolOp')
-        else:
-            pass
-        w_list = self.w_values
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.values = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.values = None
-        if self.values is not None:
-            for node in self.values:
-                node.sync_app_attrs(space)
-
 
 class BinOp(expr):
 
@@ -1151,7 +509,6 @@
         self.op = op
         self.right = right
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_BinOp(self)
@@ -1161,14 +518,6 @@
         self.right = self.right.mutate_over(visitor)
         return visitor.visit_BinOp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'left', 'op', 'right'], 'BinOp')
-        else:
-            pass
-        self.left.sync_app_attrs(space)
-        self.right.sync_app_attrs(space)
-
 
 class UnaryOp(expr):
 
@@ -1176,7 +525,6 @@
         self.op = op
         self.operand = operand
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_UnaryOp(self)
@@ -1185,13 +533,6 @@
         self.operand = self.operand.mutate_over(visitor)
         return visitor.visit_UnaryOp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'op', 'operand'], 'UnaryOp')
-        else:
-            pass
-        self.operand.sync_app_attrs(space)
-
 
 class Lambda(expr):
 
@@ -1199,7 +540,6 @@
         self.args = args
         self.body = body
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_Lambda(self)
@@ -1209,14 +549,6 @@
         self.body = self.body.mutate_over(visitor)
         return visitor.visit_Lambda(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'args', 'body'], 'Lambda')
-        else:
-            pass
-        self.args.sync_app_attrs(space)
-        self.body.sync_app_attrs(space)
-
 
 class IfExp(expr):
 
@@ -1225,7 +557,6 @@
         self.body = body
         self.orelse = orelse
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_IfExp(self)
@@ -1236,25 +567,13 @@
         self.orelse = self.orelse.mutate_over(visitor)
         return visitor.visit_IfExp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'IfExp')
-        else:
-            pass
-        self.test.sync_app_attrs(space)
-        self.body.sync_app_attrs(space)
-        self.orelse.sync_app_attrs(space)
-
 
 class Dict(expr):
 
     def __init__(self, keys, values, lineno, col_offset):
         self.keys = keys
-        self.w_keys = None
         self.values = values
-        self.w_values = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_Dict(self)
@@ -1266,40 +585,12 @@
             visitor._mutate_sequence(self.values)
         return visitor.visit_Dict(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'keys', 'values'], 'Dict')
-        else:
-            pass
-        w_list = self.w_keys
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.keys = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.keys = None
-        if self.keys is not None:
-            for node in self.keys:
-                node.sync_app_attrs(space)
-        w_list = self.w_values
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.values = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.values = None
-        if self.values is not None:
-            for node in self.values:
-                node.sync_app_attrs(space)
-
 
 class Set(expr):
 
     def __init__(self, elts, lineno, col_offset):
         self.elts = elts
-        self.w_elts = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Set(self)
@@ -1309,31 +600,13 @@
             visitor._mutate_sequence(self.elts)
         return visitor.visit_Set(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'elts'], 'Set')
-        else:
-            pass
-        w_list = self.w_elts
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.elts = None
-        if self.elts is not None:
-            for node in self.elts:
-                node.sync_app_attrs(space)
-
 
 class ListComp(expr):
 
     def __init__(self, elt, generators, lineno, col_offset):
         self.elt = elt
         self.generators = generators
-        self.w_generators = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_ListComp(self)
@@ -1344,32 +617,13 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_ListComp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'ListComp')
-        else:
-            pass
-        self.elt.sync_app_attrs(space)
-        w_list = self.w_generators
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w]
-            else:
-                self.generators = None
-        if self.generators is not None:
-            for node in self.generators:
-                node.sync_app_attrs(space)
-
 
 class SetComp(expr):
 
     def __init__(self, elt, generators, lineno, col_offset):
         self.elt = elt
         self.generators = generators
-        self.w_generators = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_SetComp(self)
@@ -1380,23 +634,6 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_SetComp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'SetComp')
-        else:
-            pass
-        self.elt.sync_app_attrs(space)
-        w_list = self.w_generators
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w]
-            else:
-                self.generators = None
-        if self.generators is not None:
-            for node in self.generators:
-                node.sync_app_attrs(space)
-
 
 class DictComp(expr):
 
@@ -1404,9 +641,7 @@
         self.key = key
         self.value = value
         self.generators = generators
-        self.w_generators = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_DictComp(self)
@@ -1418,33 +653,13 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_DictComp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'key', 'value', 'generators'], 'DictComp')
-        else:
-            pass
-        self.key.sync_app_attrs(space)
-        self.value.sync_app_attrs(space)
-        w_list = self.w_generators
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w]
-            else:
-                self.generators = None
-        if self.generators is not None:
-            for node in self.generators:
-                node.sync_app_attrs(space)
-
 
 class GeneratorExp(expr):
 
     def __init__(self, elt, generators, lineno, col_offset):
         self.elt = elt
         self.generators = generators
-        self.w_generators = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_GeneratorExp(self)
@@ -1455,30 +670,12 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_GeneratorExp(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'GeneratorExp')
-        else:
-            pass
-        self.elt.sync_app_attrs(space)
-        w_list = self.w_generators
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w]
-            else:
-                self.generators = None
-        if self.generators is not None:
-            for node in self.generators:
-                node.sync_app_attrs(space)
-
 
 class Yield(expr):
 
     def __init__(self, value, lineno, col_offset):
         self.value = value
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Yield(self)
@@ -1488,26 +685,14 @@
             self.value = self.value.mutate_over(visitor)
         return visitor.visit_Yield(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~4) ^ 3:
-            self.missing_field(space, ['lineno', 'col_offset', None], 'Yield')
-        else:
-            if not self.initialization_state & 4:
-                self.value = None
-        if self.value:
-            self.value.sync_app_attrs(space)
-
 
 class Compare(expr):
 
     def __init__(self, left, ops, comparators, lineno, col_offset):
         self.left = left
         self.ops = ops
-        self.w_ops = None
         self.comparators = comparators
-        self.w_comparators = None
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_Compare(self)
@@ -1518,43 +703,16 @@
             visitor._mutate_sequence(self.comparators)
         return visitor.visit_Compare(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'left', 'ops', 'comparators'], 'Compare')
-        else:
-            pass
-        self.left.sync_app_attrs(space)
-        w_list = self.w_ops
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.ops = [space.interp_w(cmpop, w_obj).to_simple_int(space) for w_obj in list_w]
-            else:
-                self.ops = None
-        w_list = self.w_comparators
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.comparators = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.comparators = None
-        if self.comparators is not None:
-            for node in self.comparators:
-                node.sync_app_attrs(space)
-
 
 class Call(expr):
 
     def __init__(self, func, args, keywords, starargs, kwargs, lineno, col_offset):
         self.func = func
         self.args = args
-        self.w_args = None
         self.keywords = keywords
-        self.w_keywords = None
         self.starargs = starargs
         self.kwargs = kwargs
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 127
 
     def walkabout(self, visitor):
         visitor.visit_Call(self)
@@ -1571,47 +729,12 @@
             self.kwargs = self.kwargs.mutate_over(visitor)
         return visitor.visit_Call(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~96) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'func', 'args', 'keywords', None, None], 'Call')
-        else:
-            if not self.initialization_state & 32:
-                self.starargs = None
-            if not self.initialization_state & 64:
-                self.kwargs = None
-        self.func.sync_app_attrs(space)
-        w_list = self.w_args
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.args = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.args = None
-        if self.args is not None:
-            for node in self.args:
-                node.sync_app_attrs(space)
-        w_list = self.w_keywords
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.keywords = [space.interp_w(keyword, w_obj) for w_obj in list_w]
-            else:
-                self.keywords = None
-        if self.keywords is not None:
-            for node in self.keywords:
-                node.sync_app_attrs(space)
-        if self.starargs:
-            self.starargs.sync_app_attrs(space)
-        if self.kwargs:
-            self.kwargs.sync_app_attrs(space)
-
 
 class Repr(expr):
 
     def __init__(self, value, lineno, col_offset):
         self.value = value
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Repr(self)
@@ -1620,20 +743,12 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Repr(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Repr')
-        else:
-            pass
-        self.value.sync_app_attrs(space)
-
 
 class Num(expr):
 
     def __init__(self, n, lineno, col_offset):
         self.n = n
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Num(self)
@@ -1641,19 +756,12 @@
     def mutate_over(self, visitor):
         return visitor.visit_Num(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'n'], 'Num')
-        else:
-            pass
-
 
 class Str(expr):
 
     def __init__(self, s, lineno, col_offset):
         self.s = s
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Str(self)
@@ -1661,12 +769,6 @@
     def mutate_over(self, visitor):
         return visitor.visit_Str(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 's'], 'Str')
-        else:
-            pass
-
 
 class Attribute(expr):
 
@@ -1675,7 +777,6 @@
         self.attr = attr
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_Attribute(self)
@@ -1684,13 +785,6 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Attribute(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'value', 'attr', 'ctx'], 'Attribute')
-        else:
-            pass
-        self.value.sync_app_attrs(space)
-
 
 class Subscript(expr):
 
@@ -1699,7 +793,6 @@
         self.slice = slice
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_Subscript(self)
@@ -1709,14 +802,6 @@
         self.slice = self.slice.mutate_over(visitor)
         return visitor.visit_Subscript(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 31:
-            self.missing_field(space, ['lineno', 'col_offset', 'value', 'slice', 'ctx'], 'Subscript')
-        else:
-            pass
-        self.value.sync_app_attrs(space)
-        self.slice.sync_app_attrs(space)
-
 
 class Name(expr):
 
@@ -1724,7 +809,6 @@
         self.id = id
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_Name(self)
@@ -1732,21 +816,13 @@
     def mutate_over(self, visitor):
         return visitor.visit_Name(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'id', 'ctx'], 'Name')
-        else:
-            pass
-
 
 class List(expr):
 
     def __init__(self, elts, ctx, lineno, col_offset):
         self.elts = elts
-        self.w_elts = None
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_List(self)
@@ -1756,31 +832,13 @@
             visitor._mutate_sequence(self.elts)
         return visitor.visit_List(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'elts', 'ctx'], 'List')
-        else:
-            pass
-        w_list = self.w_elts
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.elts = None
-        if self.elts is not None:
-            for node in self.elts:
-                node.sync_app_attrs(space)
-
 
 class Tuple(expr):
 
     def __init__(self, elts, ctx, lineno, col_offset):
         self.elts = elts
-        self.w_elts = None
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_Tuple(self)
@@ -1790,29 +848,12 @@
             visitor._mutate_sequence(self.elts)
         return visitor.visit_Tuple(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 15:
-            self.missing_field(space, ['lineno', 'col_offset', 'elts', 'ctx'], 'Tuple')
-        else:
-            pass
-        w_list = self.w_elts
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.elts = None
-        if self.elts is not None:
-            for node in self.elts:
-                node.sync_app_attrs(space)
-
 
 class Const(expr):
 
     def __init__(self, value, lineno, col_offset):
         self.value = value
         expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Const(self)
@@ -1820,12 +861,6 @@
     def mutate_over(self, visitor):
         return visitor.visit_Const(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Const')
-        else:
-            pass
-
 
 class expr_context(AST):
 
@@ -1884,8 +919,6 @@
 
 class Ellipsis(slice):
 
-    def __init__(self):
-        self.initialization_state = 0
 
     def walkabout(self, visitor):
         visitor.visit_Ellipsis(self)
@@ -1893,12 +926,6 @@
     def mutate_over(self, visitor):
         return visitor.visit_Ellipsis(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 0:
-            self.missing_field(space, [], 'Ellipsis')
-        else:
-            pass
-
 
 class Slice(slice):
 
@@ -1906,7 +933,6 @@
         self.lower = lower
         self.upper = upper
         self.step = step
-        self.initialization_state = 7
 
     def walkabout(self, visitor):
         visitor.visit_Slice(self)
@@ -1920,30 +946,11 @@
             self.step = self.step.mutate_over(visitor)
         return visitor.visit_Slice(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~7) ^ 0:
-            self.missing_field(space, [None, None, None], 'Slice')
-        else:
-            if not self.initialization_state & 1:
-                self.lower = None
-            if not self.initialization_state & 2:
-                self.upper = None
-            if not self.initialization_state & 4:
-                self.step = None
-        if self.lower:
-            self.lower.sync_app_attrs(space)
-        if self.upper:
-            self.upper.sync_app_attrs(space)
-        if self.step:
-            self.step.sync_app_attrs(space)
-
 
 class ExtSlice(slice):
 
     def __init__(self, dims):
         self.dims = dims
-        self.w_dims = None
-        self.initialization_state = 1
 
     def walkabout(self, visitor):
         visitor.visit_ExtSlice(self)
@@ -1953,28 +960,11 @@
             visitor._mutate_sequence(self.dims)
         return visitor.visit_ExtSlice(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 1:
-            self.missing_field(space, ['dims'], 'ExtSlice')
-        else:
-            pass
-        w_list = self.w_dims
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.dims = [space.interp_w(slice, w_obj) for w_obj in list_w]
-            else:
-                self.dims = None
-        if self.dims is not None:
-            for node in self.dims:
-                node.sync_app_attrs(space)
-
 
 class Index(slice):
 
     def __init__(self, value):
         self.value = value
-        self.initialization_state = 1
 
     def walkabout(self, visitor):
         visitor.visit_Index(self)
@@ -1983,13 +973,6 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Index(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 1:
-            self.missing_field(space, ['value'], 'Index')
-        else:
-            pass
-        self.value.sync_app_attrs(space)
-
 
 class boolop(AST):
 
@@ -2233,8 +1216,6 @@
         self.target = target
         self.iter = iter
         self.ifs = ifs
-        self.w_ifs = None
-        self.initialization_state = 7
 
     def mutate_over(self, visitor):
         self.target = self.target.mutate_over(visitor)
@@ -2246,24 +1227,6 @@
     def walkabout(self, visitor):
         visitor.visit_comprehension(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            self.missing_field(space, ['target', 'iter', 'ifs'], 'comprehension')
-        else:
-            pass
-        self.target.sync_app_attrs(space)
-        self.iter.sync_app_attrs(space)
-        w_list = self.w_ifs
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.ifs = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.ifs = None
-        if self.ifs is not None:
-            for node in self.ifs:
-                node.sync_app_attrs(space)
-
 class excepthandler(AST):
 
     def __init__(self, lineno, col_offset):
@@ -2276,9 +1239,7 @@
         self.type = type
         self.name = name
         self.body = body
-        self.w_body = None
         excepthandler.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
 
     def walkabout(self, visitor):
         visitor.visit_ExceptHandler(self)
@@ -2292,40 +1253,14 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_ExceptHandler(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~12) ^ 19:
-            self.missing_field(space, ['lineno', 'col_offset', None, None, 'body'], 'ExceptHandler')
-        else:
-            if not self.initialization_state & 4:
-                self.type = None
-            if not self.initialization_state & 8:
-                self.name = None
-        if self.type:
-            self.type.sync_app_attrs(space)
-        if self.name:
-            self.name.sync_app_attrs(space)
-        w_list = self.w_body
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w]
-            else:
-                self.body = None
-        if self.body is not None:
-            for node in self.body:
-                node.sync_app_attrs(space)
-
 
 class arguments(AST):
 
     def __init__(self, args, vararg, kwarg, defaults):
         self.args = args
-        self.w_args = None
         self.vararg = vararg
         self.kwarg = kwarg
         self.defaults = defaults
-        self.w_defaults = None
-        self.initialization_state = 15
 
     def mutate_over(self, visitor):
         if self.args:
@@ -2337,41 +1272,11 @@
     def walkabout(self, visitor):
         visitor.visit_arguments(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~6) ^ 9:
-            self.missing_field(space, ['args', None, None, 'defaults'], 'arguments')
-        else:
-            if not self.initialization_state & 2:
-                self.vararg = None
-            if not self.initialization_state & 4:
-                self.kwarg = None
-        w_list = self.w_args
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.args = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.args = None
-        if self.args is not None:
-            for node in self.args:
-                node.sync_app_attrs(space)
-        w_list = self.w_defaults
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.defaults = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.defaults = None
-        if self.defaults is not None:
-            for node in self.defaults:
-                node.sync_app_attrs(space)
-
 class keyword(AST):
 
     def __init__(self, arg, value):
         self.arg = arg
         self.value = value
-        self.initialization_state = 3
 
     def mutate_over(self, visitor):
         self.value = self.value.mutate_over(visitor)
@@ -2380,19 +1285,11 @@
     def walkabout(self, visitor):
         visitor.visit_keyword(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 3:
-            self.missing_field(space, ['arg', 'value'], 'keyword')
-        else:
-            pass
-        self.value.sync_app_attrs(space)
-
 class alias(AST):
 
     def __init__(self, name, asname):
         self.name = name
         self.asname = asname
-        self.initialization_state = 3
 
     def mutate_over(self, visitor):
         return visitor.visit_alias(self)
@@ -2400,13 +1297,6 @@
     def walkabout(self, visitor):
         visitor.visit_alias(self)
 
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~2) ^ 1:
-            self.missing_field(space, ['name', None], 'alias')
-        else:
-            if not self.initialization_state & 2:
-                self.asname = None
-
 class ASTVisitor(object):
 
     def visit_sequence(self, seq):
@@ -2784,4412 +1674,3 @@
         pass
 
 
-mod.typedef = typedef.TypeDef("mod",
-    AST.typedef,
-    __module__='_ast',
-    _attributes=_FieldsWrapper([]),
-    __new__=interp2app(get_AST_new(mod)),
-)
-
-def Module_get_body(space, w_self):
-    if not w_self.initialization_state & 1:
-        raise operationerrfmt(space.w_AttributeError, "'%T' object has no attribute '%s'", w_self, 'body')
-    if w_self.w_body is None:
-        if w_self.body is None:
-            list_w = []
-        else:
-            list_w = [space.wrap(node) for node in w_self.body]
-        w_list = space.newlist(list_w)
-        w_self.w_body = w_list
-    return w_self.w_body
-
-def Module_set_body(space, w_self, w_new_value):
-    w_self.w_body = w_new_value
-    w_self.initialization_state |= 1
-
-_Module_field_unroller = unrolling_iterable(['body'])
-def Module_init(space, w_self, __args__):
-    w_self = space.descr_self_interp_w(Module, w_self)
-    w_self.w_body = None
-    args_w, kwargs_w = __args__.unpack()
-    if args_w:
-        if len(args_w) != 1:
-            w_err = space.wrap("Module constructor takes either 0 or 1 positional argument")
-            raise OperationError(space.w_TypeError, w_err)
-        i = 0
-        for field in _Module_field_unroller:
-            space.setattr(w_self, space.wrap(field), args_w[i])
-            i += 1
-    for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
-
-Module.typedef = typedef.TypeDef("Module",
-    mod.typedef,
-    __module__='_ast',
-    _fields=_FieldsWrapper(['body']),
-    body=typedef.GetSetProperty(Module_get_body, Module_set_body, cls=Module),
-    __new__=interp2app(get_AST_new(Module)),

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:05 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:05 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: in-progress: conversion of
 internal trees to Python-visible AST.
Message-ID: <20130626212205.3395B1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65014:e1dff3384dde
Date: 2013-06-23 13:57 +0200
http://bitbucket.org/pypy/pypy/changeset/e1dff3384dde/

Log:	in-progress: conversion of internal trees to Python-visible AST.

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1,6 +1,10 @@
 # Generated by tools/asdl_py.py
 from pypy.interpreter.error import OperationError
 from rpython.tool.pairtype import extendabletype
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter import typedef
+from pypy.interpreter.gateway import interp2app
+from rpython.tool.sourcetools import func_with_new_name
 
 
 class AST(object):
@@ -16,8 +20,101 @@
 class NodeVisitorNotImplemented(Exception):
     pass
 
+
+class _FieldsWrapper(W_Root):
+    "Hack around the fact we can't store tuples on a TypeDef."
+
+    def __init__(self, fields):
+        self.fields = fields
+
+    def __spacebind__(self, space):
+        return space.newtuple([space.wrap(field) for field in self.fields])
+
+
+class W_AST(W_Root):
+    w_dict = None
+
+    def getdict(self, space):
+        if self.w_dict is None:
+            self.w_dict = space.newdict(instance=True)
+        return self.w_dict
+
+    def reduce_w(self, space):
+        w_dict = self.w_dict
+        if w_dict is None:
+            w_dict = space.newdict()
+        w_type = space.type(self)
+        w_fields = w_type.getdictvalue(space, "_fields")
+        for w_name in space.fixedview(w_fields):
+            space.setitem(w_dict, w_name,
+                          space.getattr(self, w_name))
+        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        if w_attrs:
+            for w_name in space.fixedview(w_attrs):
+                space.setitem(w_dict, w_name,
+                              space.getattr(self, w_name))
+        return space.newtuple([space.type(self),
+                               space.newtuple([]),
+                               w_dict])
+
+    def setstate_w(self, space, w_state):
+        for w_name in space.unpackiterable(w_state):
+            space.setattr(self, w_name,
+                          space.getitem(w_state, w_name))
+
+def get_W_AST_new(node_class):
+    def generic_W_AST_new(space, w_type, __args__):
+        node = space.allocate_instance(node_class, w_type)
+        return space.wrap(node)
+    return func_with_new_name(generic_W_AST_new, "new_%s" % node_class.__name__)
+
+
+def W_AST_init(space, w_self, __args__):
+    args_w, kwargs_w = __args__.unpack()
+    if args_w and len(args_w) != 0:
+        w_err = space.wrap("_ast.AST constructor takes 0 positional arguments")
+        raise OperationError(space.w_TypeError, w_err)
+    for field, w_value in kwargs_w.iteritems():
+        space.setattr(w_self, space.wrap(field), w_value)
+
+
+W_AST.typedef = typedef.TypeDef("AST",
+    _fields=_FieldsWrapper([]),
+    _attributes=_FieldsWrapper([]),
+    __module__='_ast',
+    __reduce__=interp2app(W_AST.reduce_w),
+    __setstate__=interp2app(W_AST.setstate_w),
+    __dict__ = typedef.GetSetProperty(typedef.descr_get_dict,
+                                      typedef.descr_set_dict, cls=W_AST),
+    __new__=interp2app(get_W_AST_new(W_AST)),
+    __init__=interp2app(W_AST_init),
+)
+
+class State:
+    AST_TYPES = []
+
+    @classmethod
+    def ast_type(cls, name, base):
+        cls.AST_TYPES.append((name, base))
+
+    def __init__(self, space):
+        self.w_AST = space.gettypeobject(W_AST.typedef)
+        for (name, base) in self.AST_TYPES:
+            self.make_new_type(space, name, base)
+        
+    def make_new_type(self, space, name, base):
+        w_base = getattr(self, 'w_%s' % base)
+        w_type = space.call_function(
+            space.w_type, 
+            space.wrap(name), space.newtuple([w_base]), space.newdict())
+        setattr(self, 'w_%s' % name, w_type)
+
+def get(space):
+    return space.fromcache(State)
+
 class mod(AST):
     pass
+State.ast_type('mod', 'AST')
 
 class Module(mod):
 
@@ -32,6 +129,17 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_Module(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Module)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        return w_node
+State.ast_type('Module', 'mod')
+
 
 class Interactive(mod):
 
@@ -46,6 +154,17 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_Interactive(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Interactive)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        return w_node
+State.ast_type('Interactive', 'mod')
+
 
 class Expression(mod):
 
@@ -59,6 +178,13 @@
         self.body = self.body.mutate_over(visitor)
         return visitor.visit_Expression(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Expression)
+        w_body = self.body.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('body'), w_body)
+        return w_node
+State.ast_type('Expression', 'mod')
+
 
 class Suite(mod):
 
@@ -73,12 +199,24 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_Suite(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Suite)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        return w_node
+State.ast_type('Suite', 'mod')
+
 
 class stmt(AST):
 
     def __init__(self, lineno, col_offset):
         self.lineno = lineno
         self.col_offset = col_offset
+State.ast_type('stmt', 'AST')
 
 class FunctionDef(stmt):
 
@@ -100,6 +238,27 @@
             visitor._mutate_sequence(self.decorator_list)
         return visitor.visit_FunctionDef(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_FunctionDef)
+        w_name = space.wrap(self.name)
+        space.setattr(w_node, space.wrap('name'), w_name)
+        w_args = self.args.to_object(space)  # arguments
+        space.setattr(w_node, space.wrap('args'), w_args)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        if self.decorator_list is None:
+            decorator_list_w = []
+        else:
+            decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
+        w_decorator_list = space.newlist(decorator_list_w)
+        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        return w_node
+State.ast_type('FunctionDef', 'stmt')
+
 
 class ClassDef(stmt):
 
@@ -122,6 +281,31 @@
             visitor._mutate_sequence(self.decorator_list)
         return visitor.visit_ClassDef(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_ClassDef)
+        w_name = space.wrap(self.name)
+        space.setattr(w_node, space.wrap('name'), w_name)
+        if self.bases is None:
+            bases_w = []
+        else:
+            bases_w = [node.to_object(space) for node in self.bases] # expr
+        w_bases = space.newlist(bases_w)
+        space.setattr(w_node, space.wrap('bases'), w_bases)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        if self.decorator_list is None:
+            decorator_list_w = []
+        else:
+            decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
+        w_decorator_list = space.newlist(decorator_list_w)
+        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        return w_node
+State.ast_type('ClassDef', 'stmt')
+
 
 class Return(stmt):
 
@@ -137,6 +321,13 @@
             self.value = self.value.mutate_over(visitor)
         return visitor.visit_Return(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Return)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('Return', 'stmt')
+
 
 class Delete(stmt):
 
@@ -152,6 +343,17 @@
             visitor._mutate_sequence(self.targets)
         return visitor.visit_Delete(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Delete)
+        if self.targets is None:
+            targets_w = []
+        else:
+            targets_w = [node.to_object(space) for node in self.targets] # expr
+        w_targets = space.newlist(targets_w)
+        space.setattr(w_node, space.wrap('targets'), w_targets)
+        return w_node
+State.ast_type('Delete', 'stmt')
+
 
 class Assign(stmt):
 
@@ -169,6 +371,19 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Assign(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Assign)
+        if self.targets is None:
+            targets_w = []
+        else:
+            targets_w = [node.to_object(space) for node in self.targets] # expr
+        w_targets = space.newlist(targets_w)
+        space.setattr(w_node, space.wrap('targets'), w_targets)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('Assign', 'stmt')
+
 
 class AugAssign(stmt):
 
@@ -186,6 +401,17 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_AugAssign(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_AugAssign)
+        w_target = self.target.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('target'), w_target)
+        w_op = operator_to_class[self.op - 1]()
+        space.setattr(w_node, space.wrap('op'), w_op)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('AugAssign', 'stmt')
+
 
 class Print(stmt):
 
@@ -205,6 +431,21 @@
             visitor._mutate_sequence(self.values)
         return visitor.visit_Print(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Print)
+        w_dest = self.dest.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('dest'), w_dest)
+        if self.values is None:
+            values_w = []
+        else:
+            values_w = [node.to_object(space) for node in self.values] # expr
+        w_values = space.newlist(values_w)
+        space.setattr(w_node, space.wrap('values'), w_values)
+        w_nl = self.nl.to_object(space)  # bool
+        space.setattr(w_node, space.wrap('nl'), w_nl)
+        return w_node
+State.ast_type('Print', 'stmt')
+
 
 class For(stmt):
 
@@ -227,6 +468,27 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_For(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_For)
+        w_target = self.target.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('target'), w_target)
+        w_iter = self.iter.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('iter'), w_iter)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        if self.orelse is None:
+            orelse_w = []
+        else:
+            orelse_w = [node.to_object(space) for node in self.orelse] # stmt
+        w_orelse = space.newlist(orelse_w)
+        space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        return w_node
+State.ast_type('For', 'stmt')
+
 
 class While(stmt):
 
@@ -247,6 +509,25 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_While(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_While)
+        w_test = self.test.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('test'), w_test)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        if self.orelse is None:
+            orelse_w = []
+        else:
+            orelse_w = [node.to_object(space) for node in self.orelse] # stmt
+        w_orelse = space.newlist(orelse_w)
+        space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        return w_node
+State.ast_type('While', 'stmt')
+
 
 class If(stmt):
 
@@ -267,6 +548,25 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_If(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_If)
+        w_test = self.test.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('test'), w_test)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        if self.orelse is None:
+            orelse_w = []
+        else:
+            orelse_w = [node.to_object(space) for node in self.orelse] # stmt
+        w_orelse = space.newlist(orelse_w)
+        space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        return w_node
+State.ast_type('If', 'stmt')
+
 
 class With(stmt):
 
@@ -287,6 +587,21 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_With(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_With)
+        w_context_expr = self.context_expr.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
+        w_optional_vars = self.optional_vars.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        return w_node
+State.ast_type('With', 'stmt')
+
 
 class Raise(stmt):
 
@@ -308,6 +623,17 @@
             self.tback = self.tback.mutate_over(visitor)
         return visitor.visit_Raise(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Raise)
+        w_type = self.type.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('type'), w_type)
+        w_inst = self.inst.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('inst'), w_inst)
+        w_tback = self.tback.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('tback'), w_tback)
+        return w_node
+State.ast_type('Raise', 'stmt')
+
 
 class TryExcept(stmt):
 
@@ -329,6 +655,29 @@
             visitor._mutate_sequence(self.orelse)
         return visitor.visit_TryExcept(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_TryExcept)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        if self.handlers is None:
+            handlers_w = []
+        else:
+            handlers_w = [node.to_object(space) for node in self.handlers] # excepthandler
+        w_handlers = space.newlist(handlers_w)
+        space.setattr(w_node, space.wrap('handlers'), w_handlers)
+        if self.orelse is None:
+            orelse_w = []
+        else:
+            orelse_w = [node.to_object(space) for node in self.orelse] # stmt
+        w_orelse = space.newlist(orelse_w)
+        space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        return w_node
+State.ast_type('TryExcept', 'stmt')
+
 
 class TryFinally(stmt):
 
@@ -347,6 +696,23 @@
             visitor._mutate_sequence(self.finalbody)
         return visitor.visit_TryFinally(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_TryFinally)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        if self.finalbody is None:
+            finalbody_w = []
+        else:
+            finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt
+        w_finalbody = space.newlist(finalbody_w)
+        space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
+        return w_node
+State.ast_type('TryFinally', 'stmt')
+
 
 class Assert(stmt):
 
@@ -364,6 +730,15 @@
             self.msg = self.msg.mutate_over(visitor)
         return visitor.visit_Assert(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Assert)
+        w_test = self.test.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('test'), w_test)
+        w_msg = self.msg.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('msg'), w_msg)
+        return w_node
+State.ast_type('Assert', 'stmt')
+
 
 class Import(stmt):
 
@@ -379,6 +754,17 @@
             visitor._mutate_sequence(self.names)
         return visitor.visit_Import(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Import)
+        if self.names is None:
+            names_w = []
+        else:
+            names_w = [node.to_object(space) for node in self.names] # alias
+        w_names = space.newlist(names_w)
+        space.setattr(w_node, space.wrap('names'), w_names)
+        return w_node
+State.ast_type('Import', 'stmt')
+
 
 class ImportFrom(stmt):
 
@@ -396,6 +782,21 @@
             visitor._mutate_sequence(self.names)
         return visitor.visit_ImportFrom(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_ImportFrom)
+        w_module = space.wrap(self.module)
+        space.setattr(w_node, space.wrap('module'), w_module)
+        if self.names is None:
+            names_w = []
+        else:
+            names_w = [node.to_object(space) for node in self.names] # alias
+        w_names = space.newlist(names_w)
+        space.setattr(w_node, space.wrap('names'), w_names)
+        w_level = self.level.to_object(space)  # int
+        space.setattr(w_node, space.wrap('level'), w_level)
+        return w_node
+State.ast_type('ImportFrom', 'stmt')
+
 
 class Exec(stmt):
 
@@ -416,6 +817,17 @@
             self.locals = self.locals.mutate_over(visitor)
         return visitor.visit_Exec(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Exec)
+        w_body = self.body.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('body'), w_body)
+        w_globals = self.globals.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('globals'), w_globals)
+        w_locals = self.locals.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('locals'), w_locals)
+        return w_node
+State.ast_type('Exec', 'stmt')
+
 
 class Global(stmt):
 
@@ -429,6 +841,17 @@
     def mutate_over(self, visitor):
         return visitor.visit_Global(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Global)
+        if self.names is None:
+            names_w = []
+        else:
+            names_w = [node.to_object(space) for node in self.names] # identifier
+        w_names = space.newlist(names_w)
+        space.setattr(w_node, space.wrap('names'), w_names)
+        return w_node
+State.ast_type('Global', 'stmt')
+
 
 class Expr(stmt):
 
@@ -443,6 +866,13 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Expr(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Expr)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('Expr', 'stmt')
+
 
 class Pass(stmt):
 
@@ -455,6 +885,11 @@
     def mutate_over(self, visitor):
         return visitor.visit_Pass(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Pass)
+        return w_node
+State.ast_type('Pass', 'stmt')
+
 
 class Break(stmt):
 
@@ -467,6 +902,11 @@
     def mutate_over(self, visitor):
         return visitor.visit_Break(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Break)
+        return w_node
+State.ast_type('Break', 'stmt')
+
 
 class Continue(stmt):
 
@@ -479,12 +919,18 @@
     def mutate_over(self, visitor):
         return visitor.visit_Continue(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Continue)
+        return w_node
+State.ast_type('Continue', 'stmt')
+
 
 class expr(AST):
 
     def __init__(self, lineno, col_offset):
         self.lineno = lineno
         self.col_offset = col_offset
+State.ast_type('expr', 'AST')
 
 class BoolOp(expr):
 
@@ -501,6 +947,19 @@
             visitor._mutate_sequence(self.values)
         return visitor.visit_BoolOp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_BoolOp)
+        w_op = boolop_to_class[self.op - 1]()
+        space.setattr(w_node, space.wrap('op'), w_op)
+        if self.values is None:
+            values_w = []
+        else:
+            values_w = [node.to_object(space) for node in self.values] # expr
+        w_values = space.newlist(values_w)
+        space.setattr(w_node, space.wrap('values'), w_values)
+        return w_node
+State.ast_type('BoolOp', 'expr')
+
 
 class BinOp(expr):
 
@@ -518,6 +977,17 @@
         self.right = self.right.mutate_over(visitor)
         return visitor.visit_BinOp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_BinOp)
+        w_left = self.left.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('left'), w_left)
+        w_op = operator_to_class[self.op - 1]()
+        space.setattr(w_node, space.wrap('op'), w_op)
+        w_right = self.right.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('right'), w_right)
+        return w_node
+State.ast_type('BinOp', 'expr')
+
 
 class UnaryOp(expr):
 
@@ -533,6 +1003,15 @@
         self.operand = self.operand.mutate_over(visitor)
         return visitor.visit_UnaryOp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_UnaryOp)
+        w_op = unaryop_to_class[self.op - 1]()
+        space.setattr(w_node, space.wrap('op'), w_op)
+        w_operand = self.operand.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('operand'), w_operand)
+        return w_node
+State.ast_type('UnaryOp', 'expr')
+
 
 class Lambda(expr):
 
@@ -549,6 +1028,15 @@
         self.body = self.body.mutate_over(visitor)
         return visitor.visit_Lambda(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Lambda)
+        w_args = self.args.to_object(space)  # arguments
+        space.setattr(w_node, space.wrap('args'), w_args)
+        w_body = self.body.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('body'), w_body)
+        return w_node
+State.ast_type('Lambda', 'expr')
+
 
 class IfExp(expr):
 
@@ -567,6 +1055,17 @@
         self.orelse = self.orelse.mutate_over(visitor)
         return visitor.visit_IfExp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_IfExp)
+        w_test = self.test.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('test'), w_test)
+        w_body = self.body.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('body'), w_body)
+        w_orelse = self.orelse.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        return w_node
+State.ast_type('IfExp', 'expr')
+
 
 class Dict(expr):
 
@@ -585,6 +1084,23 @@
             visitor._mutate_sequence(self.values)
         return visitor.visit_Dict(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Dict)
+        if self.keys is None:
+            keys_w = []
+        else:
+            keys_w = [node.to_object(space) for node in self.keys] # expr
+        w_keys = space.newlist(keys_w)
+        space.setattr(w_node, space.wrap('keys'), w_keys)
+        if self.values is None:
+            values_w = []
+        else:
+            values_w = [node.to_object(space) for node in self.values] # expr
+        w_values = space.newlist(values_w)
+        space.setattr(w_node, space.wrap('values'), w_values)
+        return w_node
+State.ast_type('Dict', 'expr')
+
 
 class Set(expr):
 
@@ -600,6 +1116,17 @@
             visitor._mutate_sequence(self.elts)
         return visitor.visit_Set(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Set)
+        if self.elts is None:
+            elts_w = []
+        else:
+            elts_w = [node.to_object(space) for node in self.elts] # expr
+        w_elts = space.newlist(elts_w)
+        space.setattr(w_node, space.wrap('elts'), w_elts)
+        return w_node
+State.ast_type('Set', 'expr')
+
 
 class ListComp(expr):
 
@@ -617,6 +1144,19 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_ListComp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_ListComp)
+        w_elt = self.elt.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('elt'), w_elt)
+        if self.generators is None:
+            generators_w = []
+        else:
+            generators_w = [node.to_object(space) for node in self.generators] # comprehension
+        w_generators = space.newlist(generators_w)
+        space.setattr(w_node, space.wrap('generators'), w_generators)
+        return w_node
+State.ast_type('ListComp', 'expr')
+
 
 class SetComp(expr):
 
@@ -634,6 +1174,19 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_SetComp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_SetComp)
+        w_elt = self.elt.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('elt'), w_elt)
+        if self.generators is None:
+            generators_w = []
+        else:
+            generators_w = [node.to_object(space) for node in self.generators] # comprehension
+        w_generators = space.newlist(generators_w)
+        space.setattr(w_node, space.wrap('generators'), w_generators)
+        return w_node
+State.ast_type('SetComp', 'expr')
+
 
 class DictComp(expr):
 
@@ -653,6 +1206,21 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_DictComp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_DictComp)
+        w_key = self.key.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('key'), w_key)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        if self.generators is None:
+            generators_w = []
+        else:
+            generators_w = [node.to_object(space) for node in self.generators] # comprehension
+        w_generators = space.newlist(generators_w)
+        space.setattr(w_node, space.wrap('generators'), w_generators)
+        return w_node
+State.ast_type('DictComp', 'expr')
+
 
 class GeneratorExp(expr):
 
@@ -670,6 +1238,19 @@
             visitor._mutate_sequence(self.generators)
         return visitor.visit_GeneratorExp(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_GeneratorExp)
+        w_elt = self.elt.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('elt'), w_elt)
+        if self.generators is None:
+            generators_w = []
+        else:
+            generators_w = [node.to_object(space) for node in self.generators] # comprehension
+        w_generators = space.newlist(generators_w)
+        space.setattr(w_node, space.wrap('generators'), w_generators)
+        return w_node
+State.ast_type('GeneratorExp', 'expr')
+
 
 class Yield(expr):
 
@@ -685,6 +1266,13 @@
             self.value = self.value.mutate_over(visitor)
         return visitor.visit_Yield(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Yield)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('Yield', 'expr')
+
 
 class Compare(expr):
 
@@ -703,6 +1291,25 @@
             visitor._mutate_sequence(self.comparators)
         return visitor.visit_Compare(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Compare)
+        w_left = self.left.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('left'), w_left)
+        if self.ops is None:
+            ops_w = []
+        else:
+            ops_w = [cmpop_to_class[node - 1]() for node in self.ops] # cmpop
+        w_ops = space.newlist(ops_w)
+        space.setattr(w_node, space.wrap('ops'), w_ops)
+        if self.comparators is None:
+            comparators_w = []
+        else:
+            comparators_w = [node.to_object(space) for node in self.comparators] # expr
+        w_comparators = space.newlist(comparators_w)
+        space.setattr(w_node, space.wrap('comparators'), w_comparators)
+        return w_node
+State.ast_type('Compare', 'expr')
+
 
 class Call(expr):
 
@@ -729,6 +1336,29 @@
             self.kwargs = self.kwargs.mutate_over(visitor)
         return visitor.visit_Call(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Call)
+        w_func = self.func.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('func'), w_func)
+        if self.args is None:
+            args_w = []
+        else:
+            args_w = [node.to_object(space) for node in self.args] # expr
+        w_args = space.newlist(args_w)
+        space.setattr(w_node, space.wrap('args'), w_args)
+        if self.keywords is None:
+            keywords_w = []
+        else:
+            keywords_w = [node.to_object(space) for node in self.keywords] # keyword
+        w_keywords = space.newlist(keywords_w)
+        space.setattr(w_node, space.wrap('keywords'), w_keywords)
+        w_starargs = self.starargs.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('starargs'), w_starargs)
+        w_kwargs = self.kwargs.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
+        return w_node
+State.ast_type('Call', 'expr')
+
 
 class Repr(expr):
 
@@ -743,6 +1373,13 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Repr(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Repr)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('Repr', 'expr')
+
 
 class Num(expr):
 
@@ -756,6 +1393,13 @@
     def mutate_over(self, visitor):
         return visitor.visit_Num(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Num)
+        w_n = self.n
+        space.setattr(w_node, space.wrap('n'), w_n)
+        return w_node
+State.ast_type('Num', 'expr')
+
 
 class Str(expr):
 
@@ -769,6 +1413,13 @@
     def mutate_over(self, visitor):
         return visitor.visit_Str(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Str)
+        w_s = self.s
+        space.setattr(w_node, space.wrap('s'), w_s)
+        return w_node
+State.ast_type('Str', 'expr')
+
 
 class Attribute(expr):
 
@@ -785,6 +1436,17 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Attribute(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Attribute)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        w_attr = space.wrap(self.attr)
+        space.setattr(w_node, space.wrap('attr'), w_attr)
+        w_ctx = expr_context_to_class[self.ctx - 1]()
+        space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        return w_node
+State.ast_type('Attribute', 'expr')
+
 
 class Subscript(expr):
 
@@ -802,6 +1464,17 @@
         self.slice = self.slice.mutate_over(visitor)
         return visitor.visit_Subscript(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Subscript)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        w_slice = self.slice.to_object(space)  # slice
+        space.setattr(w_node, space.wrap('slice'), w_slice)
+        w_ctx = expr_context_to_class[self.ctx - 1]()
+        space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        return w_node
+State.ast_type('Subscript', 'expr')
+
 
 class Name(expr):
 
@@ -816,6 +1489,15 @@
     def mutate_over(self, visitor):
         return visitor.visit_Name(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Name)
+        w_id = space.wrap(self.id)
+        space.setattr(w_node, space.wrap('id'), w_id)
+        w_ctx = expr_context_to_class[self.ctx - 1]()
+        space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        return w_node
+State.ast_type('Name', 'expr')
+
 
 class List(expr):
 
@@ -832,6 +1514,19 @@
             visitor._mutate_sequence(self.elts)
         return visitor.visit_List(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_List)
+        if self.elts is None:
+            elts_w = []
+        else:
+            elts_w = [node.to_object(space) for node in self.elts] # expr
+        w_elts = space.newlist(elts_w)
+        space.setattr(w_node, space.wrap('elts'), w_elts)
+        w_ctx = expr_context_to_class[self.ctx - 1]()
+        space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        return w_node
+State.ast_type('List', 'expr')
+
 
 class Tuple(expr):
 
@@ -848,6 +1543,19 @@
             visitor._mutate_sequence(self.elts)
         return visitor.visit_Tuple(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Tuple)
+        if self.elts is None:
+            elts_w = []
+        else:
+            elts_w = [node.to_object(space) for node in self.elts] # expr
+        w_elts = space.newlist(elts_w)
+        space.setattr(w_node, space.wrap('elts'), w_elts)
+        w_ctx = expr_context_to_class[self.ctx - 1]()
+        space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        return w_node
+State.ast_type('Tuple', 'expr')
+
 
 class Const(expr):
 
@@ -861,42 +1569,56 @@
     def mutate_over(self, visitor):
         return visitor.visit_Const(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Const)
+        w_value = self.value
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('Const', 'expr')
+
 
 class expr_context(AST):
 
     def to_simple_int(self, space):
         w_msg = space.wrap("not a valid expr_context")
         raise OperationError(space.w_TypeError, w_msg)
+State.ast_type('expr_context', 'AST')
 
 class _Load(expr_context):
 
     def to_simple_int(self, space):
         return 1
+State.ast_type('Load', 'expr_context')
 
 class _Store(expr_context):
 
     def to_simple_int(self, space):
         return 2
+State.ast_type('Store', 'expr_context')
 
 class _Del(expr_context):
 
     def to_simple_int(self, space):
         return 3
+State.ast_type('Del', 'expr_context')
 
 class _AugLoad(expr_context):
 
     def to_simple_int(self, space):
         return 4
+State.ast_type('AugLoad', 'expr_context')
 
 class _AugStore(expr_context):
 
     def to_simple_int(self, space):
         return 5
+State.ast_type('AugStore', 'expr_context')
 
 class _Param(expr_context):
 
     def to_simple_int(self, space):
         return 6
+State.ast_type('Param', 'expr_context')
 
 Load = 1
 Store = 2
@@ -916,6 +1638,7 @@
 
 class slice(AST):
     pass
+State.ast_type('slice', 'AST')
 
 class Ellipsis(slice):
 
@@ -926,6 +1649,11 @@
     def mutate_over(self, visitor):
         return visitor.visit_Ellipsis(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Ellipsis)
+        return w_node
+State.ast_type('Ellipsis', 'slice')
+
 
 class Slice(slice):
 
@@ -946,6 +1674,17 @@
             self.step = self.step.mutate_over(visitor)
         return visitor.visit_Slice(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Slice)
+        w_lower = self.lower.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('lower'), w_lower)
+        w_upper = self.upper.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('upper'), w_upper)
+        w_step = self.step.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('step'), w_step)
+        return w_node
+State.ast_type('Slice', 'slice')
+
 
 class ExtSlice(slice):
 
@@ -960,6 +1699,17 @@
             visitor._mutate_sequence(self.dims)
         return visitor.visit_ExtSlice(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_ExtSlice)
+        if self.dims is None:
+            dims_w = []
+        else:
+            dims_w = [node.to_object(space) for node in self.dims] # slice
+        w_dims = space.newlist(dims_w)
+        space.setattr(w_node, space.wrap('dims'), w_dims)
+        return w_node
+State.ast_type('ExtSlice', 'slice')
+
 
 class Index(slice):
 
@@ -973,22 +1723,32 @@
         self.value = self.value.mutate_over(visitor)
         return visitor.visit_Index(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_Index)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('Index', 'slice')
+
 
 class boolop(AST):
 
     def to_simple_int(self, space):
         w_msg = space.wrap("not a valid boolop")
         raise OperationError(space.w_TypeError, w_msg)
+State.ast_type('boolop', 'AST')
 
 class _And(boolop):
 
     def to_simple_int(self, space):
         return 1
+State.ast_type('And', 'boolop')
 
 class _Or(boolop):
 
     def to_simple_int(self, space):
         return 2
+State.ast_type('Or', 'boolop')
 
 And = 1
 Or = 2
@@ -1003,66 +1763,79 @@
     def to_simple_int(self, space):
         w_msg = space.wrap("not a valid operator")
         raise OperationError(space.w_TypeError, w_msg)
+State.ast_type('operator', 'AST')
 
 class _Add(operator):
 
     def to_simple_int(self, space):
         return 1
+State.ast_type('Add', 'operator')
 
 class _Sub(operator):
 
     def to_simple_int(self, space):
         return 2
+State.ast_type('Sub', 'operator')
 
 class _Mult(operator):
 
     def to_simple_int(self, space):
         return 3
+State.ast_type('Mult', 'operator')
 
 class _Div(operator):
 
     def to_simple_int(self, space):
         return 4
+State.ast_type('Div', 'operator')
 
 class _Mod(operator):
 
     def to_simple_int(self, space):
         return 5
+State.ast_type('Mod', 'operator')
 
 class _Pow(operator):
 
     def to_simple_int(self, space):
         return 6
+State.ast_type('Pow', 'operator')
 
 class _LShift(operator):
 
     def to_simple_int(self, space):
         return 7
+State.ast_type('LShift', 'operator')
 
 class _RShift(operator):
 
     def to_simple_int(self, space):
         return 8
+State.ast_type('RShift', 'operator')
 
 class _BitOr(operator):
 
     def to_simple_int(self, space):
         return 9
+State.ast_type('BitOr', 'operator')
 
 class _BitXor(operator):
 
     def to_simple_int(self, space):
         return 10
+State.ast_type('BitXor', 'operator')
 
 class _BitAnd(operator):
 
     def to_simple_int(self, space):
         return 11
+State.ast_type('BitAnd', 'operator')
 
 class _FloorDiv(operator):
 
     def to_simple_int(self, space):
         return 12
+State.ast_type('FloorDiv', 'operator')
 
 Add = 1
 Sub = 2
@@ -1097,26 +1870,31 @@
     def to_simple_int(self, space):
         w_msg = space.wrap("not a valid unaryop")
         raise OperationError(space.w_TypeError, w_msg)
+State.ast_type('unaryop', 'AST')
 
 class _Invert(unaryop):
 
     def to_simple_int(self, space):
         return 1
+State.ast_type('Invert', 'unaryop')
 
 class _Not(unaryop):
 
     def to_simple_int(self, space):
         return 2
+State.ast_type('Not', 'unaryop')
 
 class _UAdd(unaryop):
 
     def to_simple_int(self, space):
         return 3
+State.ast_type('UAdd', 'unaryop')
 
 class _USub(unaryop):
 
     def to_simple_int(self, space):
         return 4
+State.ast_type('USub', 'unaryop')
 
 Invert = 1
 Not = 2
@@ -1135,56 +1913,67 @@
     def to_simple_int(self, space):
         w_msg = space.wrap("not a valid cmpop")
         raise OperationError(space.w_TypeError, w_msg)
+State.ast_type('cmpop', 'AST')
 
 class _Eq(cmpop):
 
     def to_simple_int(self, space):
         return 1
+State.ast_type('Eq', 'cmpop')
 
 class _NotEq(cmpop):
 
     def to_simple_int(self, space):
         return 2
+State.ast_type('NotEq', 'cmpop')
 
 class _Lt(cmpop):
 
     def to_simple_int(self, space):
         return 3
+State.ast_type('Lt', 'cmpop')
 
 class _LtE(cmpop):
 
     def to_simple_int(self, space):
         return 4
+State.ast_type('LtE', 'cmpop')
 
 class _Gt(cmpop):
 
     def to_simple_int(self, space):
         return 5
+State.ast_type('Gt', 'cmpop')
 
 class _GtE(cmpop):
 
     def to_simple_int(self, space):
         return 6
+State.ast_type('GtE', 'cmpop')
 
 class _Is(cmpop):
 
     def to_simple_int(self, space):
         return 7
+State.ast_type('Is', 'cmpop')
 
 class _IsNot(cmpop):
 
     def to_simple_int(self, space):
         return 8
+State.ast_type('IsNot', 'cmpop')
 
 class _In(cmpop):
 
     def to_simple_int(self, space):
         return 9
+State.ast_type('In', 'cmpop')
 
 class _NotIn(cmpop):
 
     def to_simple_int(self, space):
         return 10
+State.ast_type('NotIn', 'cmpop')
 
 Eq = 1
 NotEq = 2
@@ -1227,11 +2016,27 @@
     def walkabout(self, visitor):
         visitor.visit_comprehension(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_comprehension)
+        w_target = self.target.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('target'), w_target)
+        w_iter = self.iter.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('iter'), w_iter)
+        if self.ifs is None:
+            ifs_w = []
+        else:
+            ifs_w = [node.to_object(space) for node in self.ifs] # expr
+        w_ifs = space.newlist(ifs_w)
+        space.setattr(w_node, space.wrap('ifs'), w_ifs)
+        return w_node
+State.ast_type('comprehension', 'AST')
+
 class excepthandler(AST):
 
     def __init__(self, lineno, col_offset):
         self.lineno = lineno
         self.col_offset = col_offset
+State.ast_type('excepthandler', 'AST')
 
 class ExceptHandler(excepthandler):
 
@@ -1253,6 +2058,21 @@
             visitor._mutate_sequence(self.body)
         return visitor.visit_ExceptHandler(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_ExceptHandler)
+        w_type = self.type.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('type'), w_type)
+        w_name = self.name.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('name'), w_name)
+        if self.body is None:
+            body_w = []
+        else:
+            body_w = [node.to_object(space) for node in self.body] # stmt
+        w_body = space.newlist(body_w)
+        space.setattr(w_node, space.wrap('body'), w_body)
+        return w_node
+State.ast_type('ExceptHandler', 'excepthandler')
+
 
 class arguments(AST):
 
@@ -1272,6 +2092,27 @@
     def walkabout(self, visitor):
         visitor.visit_arguments(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_arguments)
+        if self.args is None:
+            args_w = []
+        else:
+            args_w = [node.to_object(space) for node in self.args] # expr
+        w_args = space.newlist(args_w)
+        space.setattr(w_node, space.wrap('args'), w_args)
+        w_vararg = space.wrap(self.vararg)
+        space.setattr(w_node, space.wrap('vararg'), w_vararg)
+        w_kwarg = space.wrap(self.kwarg)
+        space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
+        if self.defaults is None:
+            defaults_w = []
+        else:
+            defaults_w = [node.to_object(space) for node in self.defaults] # expr
+        w_defaults = space.newlist(defaults_w)
+        space.setattr(w_node, space.wrap('defaults'), w_defaults)
+        return w_node
+State.ast_type('arguments', 'AST')
+
 class keyword(AST):
 
     def __init__(self, arg, value):
@@ -1285,6 +2126,15 @@
     def walkabout(self, visitor):
         visitor.visit_keyword(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_keyword)
+        w_arg = space.wrap(self.arg)
+        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_value = self.value.to_object(space)  # expr
+        space.setattr(w_node, space.wrap('value'), w_value)
+        return w_node
+State.ast_type('keyword', 'AST')
+
 class alias(AST):
 
     def __init__(self, name, asname):
@@ -1297,6 +2147,15 @@
     def walkabout(self, visitor):
         visitor.visit_alias(self)
 
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_alias)
+        w_name = space.wrap(self.name)
+        space.setattr(w_node, space.wrap('name'), w_name)
+        w_asname = space.wrap(self.asname)
+        space.setattr(w_node, space.wrap('asname'), w_asname)
+        return w_node
+State.ast_type('alias', 'AST')
+
 class ASTVisitor(object):
 
     def visit_sequence(self, seq):
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -0,0 +1,30 @@
+from pypy.interpreter.astcompiler import ast
+class TestAstToObject:
+    def test_types(self, space):
+        assert space.is_true(space.issubtype(
+                ast.get(space).w_Module, ast.get(space).w_mod))
+                                  
+    def test_num(self, space):
+        value = space.wrap(42)
+        node = ast.Num(value, lineno=1, col_offset=1)
+        w_node = node.to_object(space)
+        assert space.getattr(w_node, space.wrap("n")) is value
+
+    def test_expr(self, space):
+        value = space.wrap(42)
+        node = ast.Num(value, lineno=1, col_offset=1)
+        expr = ast.Expr(node, lineno=1, col_offset=1)
+        w_node = expr.to_object(space)
+        # node.value.n
+        assert space.getattr(space.getattr(w_node, space.wrap("value")),
+                             space.wrap("n")) is value
+
+    def test_operation(self, space):
+        val1 = ast.Num(space.wrap(1), lineno=1, col_offset=1)
+        val2 = ast.Num(space.wrap(2), lineno=1, col_offset=1)
+        node = ast.BinOp(left=val1, right=val2, op=ast.Add,
+                         lineno=1, col_offset=1)
+        w_node = node.to_object(space)
+        w_op = space.getattr(w_node, space.wrap("op"))
+        assert space.is_true(space.issubtype(
+                ast.get(space).w_operator, w_op))
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -61,12 +61,14 @@
             self.emit("def to_simple_int(self, space):", 1)
             self.emit("w_msg = space.wrap(\"not a valid %s\")" % (base,), 2)
             self.emit("raise OperationError(space.w_TypeError, w_msg)", 2)
+            self.emit("State.ast_type('%s', 'AST')" % (base,))
             self.emit("")
             for i, cons in enumerate(sum.types):
                 self.emit("class _%s(%s):" % (cons.name, base))
                 self.emit("")
                 self.emit("def to_simple_int(self, space):", 1)
                 self.emit("return %i" % (i + 1,), 2)
+                self.emit("State.ast_type('%s', '%s')" % (cons.name, base))
                 self.emit("")
             for i, cons in enumerate(sum.types):
                 self.emit("%s = %i" % (cons.name, i + 1))
@@ -84,10 +86,10 @@
                 self.emit("def __init__(self, %s):" % (args,), 1)
                 for attr in sum.attributes:
                     self.visit(attr)
-                self.emit("")
             else:
                 self.emit("pass", 1)
-                self.emit("")
+            self.emit("State.ast_type('%r', 'AST')" % (base,))
+            self.emit("")
             for cons in sum.types:
                 self.visit(cons, base, sum.attributes)
                 self.emit("")
@@ -101,6 +103,47 @@
         self.emit("def walkabout(self, visitor):", 1)
         self.emit("visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
+        self.make_converters(product.fields, name)
+        self.emit("State.ast_type('%r', 'AST')" % (name,))
+        self.emit("")
+
+    def get_field_converter(self, field):
+        if field.seq:
+            lines = []
+            lines.append("if self.%s is None:" % field.name)
+            lines.append("    %s_w = []" % field.name)
+            lines.append("else:")
+            if field.type.value in self.data.simple_types:
+                wrapper = "%s_to_class[node - 1]()" % (field.type,)
+            elif field.type.value in ("object", "string"):
+                wrapper = "node"
+            else:
+                wrapper = "node.to_object(space)"
+            lines.append("    %s_w = [%s for node in self.%s] # %s" %
+                         (field.name, wrapper, field.name, field.type))
+            lines.append("w_%s = space.newlist(%s_w)" % (field.name, field.name))
+            return lines
+        elif field.type.value in self.data.simple_types:
+            return ["w_%s = %s_to_class[self.%s - 1]()" % 
+                    (field.name, field.type, field.name)]
+        elif field.type.value in ("object", "string"):
+            return ["w_%s = self.%s" % (field.name, field.name)]
+        elif field.type.value in ("identifier",):
+            return ["w_%s = space.wrap(self.%s)" % (field.name, field.name)]
+        else:
+            return ["w_%s = self.%s.to_object(space)  # %s" %
+                    (field.name, field.name, field.type)]
+
+    def make_converters(self, fields, name):
+        self.emit("def to_object(self, space):", 1)
+        self.emit("w_node = space.call_function(get(space).w_%s)" % name, 2)
+        for field in fields:
+            wrapping_code = self.get_field_converter(field)
+            for line in wrapping_code:
+                self.emit(line, 2)
+            self.emit("space.setattr(w_node, space.wrap(%r), w_%s)" % (
+                    str(field.name), field.name), 2)
+        self.emit("return w_node", 2)
 
     def make_constructor(self, fields, node, extras=None, base=None):
         if fields or extras:
@@ -142,6 +185,9 @@
         self.emit("visitor.visit_%s(self)" % (cons.name,), 2)
         self.emit("")
         self.make_mutate_over(cons, cons.name)
+        self.make_converters(cons.fields, cons.name)
+        self.emit("State.ast_type('%r', '%s')" % (cons.name, base))
+        self.emit("")
 
     def visitField(self, field):
         self.emit("self.%s = %s" % (field.name, field.name), 2)
@@ -212,8 +258,8 @@
         self.emit("")
 
     def visitField(self, field):
-        if field.type.value not in asdl.builtin_types and \
-                field.type.value not in self.data.simple_types:
+        if (field.type.value not in asdl.builtin_types and 
+            field.type.value not in self.data.simple_types):
             level = 2
             template = "node.%s.walkabout(self)"
             if field.seq:
@@ -276,6 +322,10 @@
 HEAD = """# Generated by tools/asdl_py.py
 from pypy.interpreter.error import OperationError
 from rpython.tool.pairtype import extendabletype
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter import typedef
+from pypy.interpreter.gateway import interp2app
+from rpython.tool.sourcetools import func_with_new_name
 
 
 class AST(object):
@@ -291,6 +341,98 @@
 class NodeVisitorNotImplemented(Exception):
     pass
 
+
+class _FieldsWrapper(W_Root):
+    "Hack around the fact we can't store tuples on a TypeDef."
+
+    def __init__(self, fields):
+        self.fields = fields
+
+    def __spacebind__(self, space):
+        return space.newtuple([space.wrap(field) for field in self.fields])
+
+
+class W_AST(W_Root):
+    w_dict = None
+
+    def getdict(self, space):
+        if self.w_dict is None:
+            self.w_dict = space.newdict(instance=True)
+        return self.w_dict
+
+    def reduce_w(self, space):
+        w_dict = self.w_dict
+        if w_dict is None:
+            w_dict = space.newdict()
+        w_type = space.type(self)
+        w_fields = w_type.getdictvalue(space, "_fields")
+        for w_name in space.fixedview(w_fields):
+            space.setitem(w_dict, w_name,
+                          space.getattr(self, w_name))
+        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        if w_attrs:
+            for w_name in space.fixedview(w_attrs):
+                space.setitem(w_dict, w_name,
+                              space.getattr(self, w_name))
+        return space.newtuple([space.type(self),
+                               space.newtuple([]),
+                               w_dict])
+
+    def setstate_w(self, space, w_state):
+        for w_name in space.unpackiterable(w_state):
+            space.setattr(self, w_name,
+                          space.getitem(w_state, w_name))
+
+def get_W_AST_new(node_class):
+    def generic_W_AST_new(space, w_type, __args__):
+        node = space.allocate_instance(node_class, w_type)
+        return space.wrap(node)
+    return func_with_new_name(generic_W_AST_new, "new_%s" % node_class.__name__)
+
+
+def W_AST_init(space, w_self, __args__):
+    args_w, kwargs_w = __args__.unpack()
+    if args_w and len(args_w) != 0:
+        w_err = space.wrap("_ast.AST constructor takes 0 positional arguments")
+        raise OperationError(space.w_TypeError, w_err)
+    for field, w_value in kwargs_w.iteritems():
+        space.setattr(w_self, space.wrap(field), w_value)
+
+
+W_AST.typedef = typedef.TypeDef("AST",
+    _fields=_FieldsWrapper([]),
+    _attributes=_FieldsWrapper([]),
+    __module__='_ast',
+    __reduce__=interp2app(W_AST.reduce_w),
+    __setstate__=interp2app(W_AST.setstate_w),
+    __dict__ = typedef.GetSetProperty(typedef.descr_get_dict,
+                                      typedef.descr_set_dict, cls=W_AST),
+    __new__=interp2app(get_W_AST_new(W_AST)),
+    __init__=interp2app(W_AST_init),
+)
+
+class State:
+    AST_TYPES = []
+
+    @classmethod
+    def ast_type(cls, name, base):
+        cls.AST_TYPES.append((name, base))
+
+    def __init__(self, space):
+        self.w_AST = space.gettypeobject(W_AST.typedef)
+        for (name, base) in self.AST_TYPES:
+            self.make_new_type(space, name, base)
+        
+    def make_new_type(self, space, name, base):
+        w_base = getattr(self, 'w_%s' % base)
+        w_type = space.call_function(
+            space.w_type, 
+            space.wrap(name), space.newtuple([w_base]), space.newdict())
+        setattr(self, 'w_%s' % name, w_type)
+
+def get(space):
+    return space.fromcache(State)
+
 """
 
 visitors = [ASTNodeVisitor, ASTVisitorVisitor, GenericASTVisitorVisitor]
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -25,7 +25,7 @@
 """
 
     ast_node = None
-    w_ast_type = space.gettypeobject(interp_ast.W_AST.typedef)
+    w_ast_type = space.gettypeobject(ast.W_AST.typedef)
     str_ = None
     if space.isinstance_w(w_source, w_ast_type):
         ast_node = space.interp_w(ast.mod, w_source)
@@ -57,7 +57,8 @@
     if ast_node is None:
         if flags & consts.PyCF_ONLY_AST:
             mod = ec.compiler.compile_to_ast(str_, filename, mode, flags)
-            return space.wrap(mod)
+            w_mod = mod.to_object(space)
+            return w_mod
         else:
             code = ec.compiler.compile(str_, filename, mode, flags)
     else:
diff --git a/pypy/module/_ast/__init__.py b/pypy/module/_ast/__init__.py
--- a/pypy/module/_ast/__init__.py
+++ b/pypy/module/_ast/__init__.py
@@ -7,14 +7,12 @@
     interpleveldefs = {
         "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST,
         "__version__"   : "space.wrap('82160')",  # from CPython's svn.
-        "AST": "interp_ast.W_AST",
         }
     appleveldefs = {}
 
 
 def _setup():
     defs = Module.interpleveldefs
-    for name, cls in ast.__dict__.iteritems():
-        if isinstance(cls, type) and issubclass(cls, ast.AST):
-            defs[name.lstrip("_")] = cls.__module__ + "." + name
+    for (name, base) in ast.State.AST_TYPES:
+        defs[name] = "pypy.interpreter.astcompiler.ast.get(space).w_" + name
 _setup()

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:06 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:06 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Progress.
Message-ID: <20130626212206.67FCC1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65015:c1e9b9cdc82a
Date: 2013-06-23 22:51 +0200
http://bitbucket.org/pypy/pypy/changeset/c1e9b9cdc82a/

Log:	Progress.

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -180,7 +180,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expression)
-        w_body = self.body.to_object(space)  # expr
+        w_body = self.body.to_object(space)
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
 State.ast_type('Expression', 'mod')
@@ -240,9 +240,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FunctionDef)
-        w_name = space.wrap(self.name)
+        w_name = self.name.to_object(space)
         space.setattr(w_node, space.wrap('name'), w_name)
-        w_args = self.args.to_object(space)  # arguments
+        w_args = self.args.to_object(space)
         space.setattr(w_node, space.wrap('args'), w_args)
         if self.body is None:
             body_w = []
@@ -283,7 +283,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ClassDef)
-        w_name = space.wrap(self.name)
+        w_name = self.name.to_object(space)
         space.setattr(w_node, space.wrap('name'), w_name)
         if self.bases is None:
             bases_w = []
@@ -323,7 +323,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Return', 'stmt')
@@ -379,7 +379,7 @@
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
         space.setattr(w_node, space.wrap('targets'), w_targets)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Assign', 'stmt')
@@ -403,11 +403,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AugAssign)
-        w_target = self.target.to_object(space)  # expr
+        w_target = self.target.to_object(space)
         space.setattr(w_node, space.wrap('target'), w_target)
-        w_op = operator_to_class[self.op - 1]()
+        w_op = operator_to_class[self.op - 1]().to_object(space)
         space.setattr(w_node, space.wrap('op'), w_op)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('AugAssign', 'stmt')
@@ -433,7 +433,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Print)
-        w_dest = self.dest.to_object(space)  # expr
+        w_dest = self.dest.to_object(space)
         space.setattr(w_node, space.wrap('dest'), w_dest)
         if self.values is None:
             values_w = []
@@ -441,7 +441,7 @@
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
         space.setattr(w_node, space.wrap('values'), w_values)
-        w_nl = self.nl.to_object(space)  # bool
+        w_nl = self.nl.to_object(space)
         space.setattr(w_node, space.wrap('nl'), w_nl)
         return w_node
 State.ast_type('Print', 'stmt')
@@ -470,9 +470,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_For)
-        w_target = self.target.to_object(space)  # expr
+        w_target = self.target.to_object(space)
         space.setattr(w_node, space.wrap('target'), w_target)
-        w_iter = self.iter.to_object(space)  # expr
+        w_iter = self.iter.to_object(space)
         space.setattr(w_node, space.wrap('iter'), w_iter)
         if self.body is None:
             body_w = []
@@ -511,7 +511,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_While)
-        w_test = self.test.to_object(space)  # expr
+        w_test = self.test.to_object(space)
         space.setattr(w_node, space.wrap('test'), w_test)
         if self.body is None:
             body_w = []
@@ -550,7 +550,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_If)
-        w_test = self.test.to_object(space)  # expr
+        w_test = self.test.to_object(space)
         space.setattr(w_node, space.wrap('test'), w_test)
         if self.body is None:
             body_w = []
@@ -589,9 +589,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_With)
-        w_context_expr = self.context_expr.to_object(space)  # expr
+        w_context_expr = self.context_expr.to_object(space)
         space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
-        w_optional_vars = self.optional_vars.to_object(space)  # expr
+        w_optional_vars = self.optional_vars.to_object(space)
         space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
         if self.body is None:
             body_w = []
@@ -625,11 +625,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
-        w_type = self.type.to_object(space)  # expr
+        w_type = self.type.to_object(space)
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_inst = self.inst.to_object(space)  # expr
+        w_inst = self.inst.to_object(space)
         space.setattr(w_node, space.wrap('inst'), w_inst)
-        w_tback = self.tback.to_object(space)  # expr
+        w_tback = self.tback.to_object(space)
         space.setattr(w_node, space.wrap('tback'), w_tback)
         return w_node
 State.ast_type('Raise', 'stmt')
@@ -732,9 +732,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Assert)
-        w_test = self.test.to_object(space)  # expr
+        w_test = self.test.to_object(space)
         space.setattr(w_node, space.wrap('test'), w_test)
-        w_msg = self.msg.to_object(space)  # expr
+        w_msg = self.msg.to_object(space)
         space.setattr(w_node, space.wrap('msg'), w_msg)
         return w_node
 State.ast_type('Assert', 'stmt')
@@ -784,7 +784,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = space.wrap(self.module)
+        w_module = self.module.to_object(space)
         space.setattr(w_node, space.wrap('module'), w_module)
         if self.names is None:
             names_w = []
@@ -792,7 +792,7 @@
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
         space.setattr(w_node, space.wrap('names'), w_names)
-        w_level = self.level.to_object(space)  # int
+        w_level = self.level.to_object(space)
         space.setattr(w_node, space.wrap('level'), w_level)
         return w_node
 State.ast_type('ImportFrom', 'stmt')
@@ -819,11 +819,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Exec)
-        w_body = self.body.to_object(space)  # expr
+        w_body = self.body.to_object(space)
         space.setattr(w_node, space.wrap('body'), w_body)
-        w_globals = self.globals.to_object(space)  # expr
+        w_globals = self.globals.to_object(space)
         space.setattr(w_node, space.wrap('globals'), w_globals)
-        w_locals = self.locals.to_object(space)  # expr
+        w_locals = self.locals.to_object(space)
         space.setattr(w_node, space.wrap('locals'), w_locals)
         return w_node
 State.ast_type('Exec', 'stmt')
@@ -868,7 +868,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expr)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Expr', 'stmt')
@@ -949,7 +949,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BoolOp)
-        w_op = boolop_to_class[self.op - 1]()
+        w_op = boolop_to_class[self.op - 1]().to_object(space)
         space.setattr(w_node, space.wrap('op'), w_op)
         if self.values is None:
             values_w = []
@@ -979,11 +979,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BinOp)
-        w_left = self.left.to_object(space)  # expr
+        w_left = self.left.to_object(space)
         space.setattr(w_node, space.wrap('left'), w_left)
-        w_op = operator_to_class[self.op - 1]()
+        w_op = operator_to_class[self.op - 1]().to_object(space)
         space.setattr(w_node, space.wrap('op'), w_op)
-        w_right = self.right.to_object(space)  # expr
+        w_right = self.right.to_object(space)
         space.setattr(w_node, space.wrap('right'), w_right)
         return w_node
 State.ast_type('BinOp', 'expr')
@@ -1005,9 +1005,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_UnaryOp)
-        w_op = unaryop_to_class[self.op - 1]()
+        w_op = unaryop_to_class[self.op - 1]().to_object(space)
         space.setattr(w_node, space.wrap('op'), w_op)
-        w_operand = self.operand.to_object(space)  # expr
+        w_operand = self.operand.to_object(space)
         space.setattr(w_node, space.wrap('operand'), w_operand)
         return w_node
 State.ast_type('UnaryOp', 'expr')
@@ -1030,9 +1030,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Lambda)
-        w_args = self.args.to_object(space)  # arguments
+        w_args = self.args.to_object(space)
         space.setattr(w_node, space.wrap('args'), w_args)
-        w_body = self.body.to_object(space)  # expr
+        w_body = self.body.to_object(space)
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
 State.ast_type('Lambda', 'expr')
@@ -1057,11 +1057,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_IfExp)
-        w_test = self.test.to_object(space)  # expr
+        w_test = self.test.to_object(space)
         space.setattr(w_node, space.wrap('test'), w_test)
-        w_body = self.body.to_object(space)  # expr
+        w_body = self.body.to_object(space)
         space.setattr(w_node, space.wrap('body'), w_body)
-        w_orelse = self.orelse.to_object(space)  # expr
+        w_orelse = self.orelse.to_object(space)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
         return w_node
 State.ast_type('IfExp', 'expr')
@@ -1146,7 +1146,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ListComp)
-        w_elt = self.elt.to_object(space)  # expr
+        w_elt = self.elt.to_object(space)
         space.setattr(w_node, space.wrap('elt'), w_elt)
         if self.generators is None:
             generators_w = []
@@ -1176,7 +1176,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_SetComp)
-        w_elt = self.elt.to_object(space)  # expr
+        w_elt = self.elt.to_object(space)
         space.setattr(w_node, space.wrap('elt'), w_elt)
         if self.generators is None:
             generators_w = []
@@ -1208,9 +1208,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_DictComp)
-        w_key = self.key.to_object(space)  # expr
+        w_key = self.key.to_object(space)
         space.setattr(w_node, space.wrap('key'), w_key)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         if self.generators is None:
             generators_w = []
@@ -1240,7 +1240,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_GeneratorExp)
-        w_elt = self.elt.to_object(space)  # expr
+        w_elt = self.elt.to_object(space)
         space.setattr(w_node, space.wrap('elt'), w_elt)
         if self.generators is None:
             generators_w = []
@@ -1268,7 +1268,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Yield', 'expr')
@@ -1293,12 +1293,12 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Compare)
-        w_left = self.left.to_object(space)  # expr
+        w_left = self.left.to_object(space)
         space.setattr(w_node, space.wrap('left'), w_left)
         if self.ops is None:
             ops_w = []
         else:
-            ops_w = [cmpop_to_class[node - 1]() for node in self.ops] # cmpop
+            ops_w = [cmpop_to_class[node - 1]().to_object(space) for node in self.ops] # cmpop
         w_ops = space.newlist(ops_w)
         space.setattr(w_node, space.wrap('ops'), w_ops)
         if self.comparators is None:
@@ -1338,7 +1338,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Call)
-        w_func = self.func.to_object(space)  # expr
+        w_func = self.func.to_object(space)
         space.setattr(w_node, space.wrap('func'), w_func)
         if self.args is None:
             args_w = []
@@ -1352,9 +1352,9 @@
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
         space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_starargs = self.starargs.to_object(space)  # expr
+        w_starargs = self.starargs.to_object(space)
         space.setattr(w_node, space.wrap('starargs'), w_starargs)
-        w_kwargs = self.kwargs.to_object(space)  # expr
+        w_kwargs = self.kwargs.to_object(space)
         space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
         return w_node
 State.ast_type('Call', 'expr')
@@ -1375,7 +1375,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Repr)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Repr', 'expr')
@@ -1438,11 +1438,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Attribute)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
-        w_attr = space.wrap(self.attr)
+        w_attr = self.attr.to_object(space)
         space.setattr(w_node, space.wrap('attr'), w_attr)
-        w_ctx = expr_context_to_class[self.ctx - 1]()
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Attribute', 'expr')
@@ -1466,11 +1466,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Subscript)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
-        w_slice = self.slice.to_object(space)  # slice
+        w_slice = self.slice.to_object(space)
         space.setattr(w_node, space.wrap('slice'), w_slice)
-        w_ctx = expr_context_to_class[self.ctx - 1]()
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Subscript', 'expr')
@@ -1491,9 +1491,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Name)
-        w_id = space.wrap(self.id)
+        w_id = self.id.to_object(space)
         space.setattr(w_node, space.wrap('id'), w_id)
-        w_ctx = expr_context_to_class[self.ctx - 1]()
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Name', 'expr')
@@ -1522,7 +1522,7 @@
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
         space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_ctx = expr_context_to_class[self.ctx - 1]()
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('List', 'expr')
@@ -1551,7 +1551,7 @@
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
         space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_ctx = expr_context_to_class[self.ctx - 1]()
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Tuple', 'expr')
@@ -1578,46 +1578,37 @@
 
 
 class expr_context(AST):
-
-    def to_simple_int(self, space):
-        w_msg = space.wrap("not a valid expr_context")
-        raise OperationError(space.w_TypeError, w_msg)
+    pass
 State.ast_type('expr_context', 'AST')
 
 class _Load(expr_context):
-
-    def to_simple_int(self, space):
-        return 1
+    def to_object(self, space):
+        return space.call_function(get(space).w_Load)
 State.ast_type('Load', 'expr_context')
 
 class _Store(expr_context):
-
-    def to_simple_int(self, space):
-        return 2
+    def to_object(self, space):
+        return space.call_function(get(space).w_Store)
 State.ast_type('Store', 'expr_context')
 
 class _Del(expr_context):
-
-    def to_simple_int(self, space):
-        return 3
+    def to_object(self, space):
+        return space.call_function(get(space).w_Del)
 State.ast_type('Del', 'expr_context')
 
 class _AugLoad(expr_context):
-
-    def to_simple_int(self, space):
-        return 4
+    def to_object(self, space):
+        return space.call_function(get(space).w_AugLoad)
 State.ast_type('AugLoad', 'expr_context')
 
 class _AugStore(expr_context):
-
-    def to_simple_int(self, space):
-        return 5
+    def to_object(self, space):
+        return space.call_function(get(space).w_AugStore)
 State.ast_type('AugStore', 'expr_context')
 
 class _Param(expr_context):
-
-    def to_simple_int(self, space):
-        return 6
+    def to_object(self, space):
+        return space.call_function(get(space).w_Param)
 State.ast_type('Param', 'expr_context')
 
 Load = 1
@@ -1676,11 +1667,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
-        w_lower = self.lower.to_object(space)  # expr
+        w_lower = self.lower.to_object(space)
         space.setattr(w_node, space.wrap('lower'), w_lower)
-        w_upper = self.upper.to_object(space)  # expr
+        w_upper = self.upper.to_object(space)
         space.setattr(w_node, space.wrap('upper'), w_upper)
-        w_step = self.step.to_object(space)  # expr
+        w_step = self.step.to_object(space)
         space.setattr(w_node, space.wrap('step'), w_step)
         return w_node
 State.ast_type('Slice', 'slice')
@@ -1725,29 +1716,24 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Index)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Index', 'slice')
 
 
 class boolop(AST):
-
-    def to_simple_int(self, space):
-        w_msg = space.wrap("not a valid boolop")
-        raise OperationError(space.w_TypeError, w_msg)
+    pass
 State.ast_type('boolop', 'AST')
 
 class _And(boolop):
-
-    def to_simple_int(self, space):
-        return 1
+    def to_object(self, space):
+        return space.call_function(get(space).w_And)
 State.ast_type('And', 'boolop')
 
 class _Or(boolop):
-
-    def to_simple_int(self, space):
-        return 2
+    def to_object(self, space):
+        return space.call_function(get(space).w_Or)
 State.ast_type('Or', 'boolop')
 
 And = 1
@@ -1759,82 +1745,67 @@
 ]
 
 class operator(AST):
-
-    def to_simple_int(self, space):
-        w_msg = space.wrap("not a valid operator")
-        raise OperationError(space.w_TypeError, w_msg)
+    pass
 State.ast_type('operator', 'AST')
 
 class _Add(operator):
-
-    def to_simple_int(self, space):
-        return 1
+    def to_object(self, space):
+        return space.call_function(get(space).w_Add)
 State.ast_type('Add', 'operator')
 
 class _Sub(operator):
-
-    def to_simple_int(self, space):
-        return 2
+    def to_object(self, space):
+        return space.call_function(get(space).w_Sub)
 State.ast_type('Sub', 'operator')
 
 class _Mult(operator):
-
-    def to_simple_int(self, space):
-        return 3
+    def to_object(self, space):
+        return space.call_function(get(space).w_Mult)
 State.ast_type('Mult', 'operator')
 
 class _Div(operator):
-
-    def to_simple_int(self, space):
-        return 4
+    def to_object(self, space):
+        return space.call_function(get(space).w_Div)
 State.ast_type('Div', 'operator')
 
 class _Mod(operator):
-
-    def to_simple_int(self, space):
-        return 5
+    def to_object(self, space):
+        return space.call_function(get(space).w_Mod)
 State.ast_type('Mod', 'operator')
 
 class _Pow(operator):
-
-    def to_simple_int(self, space):
-        return 6
+    def to_object(self, space):
+        return space.call_function(get(space).w_Pow)
 State.ast_type('Pow', 'operator')
 
 class _LShift(operator):
-
-    def to_simple_int(self, space):
-        return 7
+    def to_object(self, space):
+        return space.call_function(get(space).w_LShift)
 State.ast_type('LShift', 'operator')
 
 class _RShift(operator):
-
-    def to_simple_int(self, space):
-        return 8
+    def to_object(self, space):
+        return space.call_function(get(space).w_RShift)
 State.ast_type('RShift', 'operator')
 
 class _BitOr(operator):
-
-    def to_simple_int(self, space):
-        return 9
+    def to_object(self, space):
+        return space.call_function(get(space).w_BitOr)
 State.ast_type('BitOr', 'operator')
 
 class _BitXor(operator):
-
-    def to_simple_int(self, space):
-        return 10
+    def to_object(self, space):
+        return space.call_function(get(space).w_BitXor)
 State.ast_type('BitXor', 'operator')
 
 class _BitAnd(operator):
-
-    def to_simple_int(self, space):
-        return 11
+    def to_object(self, space):
+        return space.call_function(get(space).w_BitAnd)
 State.ast_type('BitAnd', 'operator')
 
 class _FloorDiv(operator):
-
-    def to_simple_int(self, space):
-        return 12
+    def to_object(self, space):
+        return space.call_function(get(space).w_FloorDiv)
 State.ast_type('FloorDiv', 'operator')
 
 Add = 1
@@ -1866,34 +1837,27 @@
 ]
 
 class unaryop(AST):
-
-    def to_simple_int(self, space):
-        w_msg = space.wrap("not a valid unaryop")
-        raise OperationError(space.w_TypeError, w_msg)
+    pass
 State.ast_type('unaryop', 'AST')
 
 class _Invert(unaryop):
-
-    def to_simple_int(self, space):
-        return 1
+    def to_object(self, space):
+        return space.call_function(get(space).w_Invert)
 State.ast_type('Invert', 'unaryop')
 
 class _Not(unaryop):
-
-    def to_simple_int(self, space):
-        return 2
+    def to_object(self, space):
+        return space.call_function(get(space).w_Not)
 State.ast_type('Not', 'unaryop')
 
 class _UAdd(unaryop):
-
-    def to_simple_int(self, space):
-        return 3
+    def to_object(self, space):
+        return space.call_function(get(space).w_UAdd)
 State.ast_type('UAdd', 'unaryop')
 
 class _USub(unaryop):
-
-    def to_simple_int(self, space):
-        return 4
+    def to_object(self, space):
+        return space.call_function(get(space).w_USub)
 State.ast_type('USub', 'unaryop')
 
 Invert = 1
@@ -1909,70 +1873,57 @@
 ]
 
 class cmpop(AST):
-
-    def to_simple_int(self, space):
-        w_msg = space.wrap("not a valid cmpop")
-        raise OperationError(space.w_TypeError, w_msg)
+    pass
 State.ast_type('cmpop', 'AST')
 
 class _Eq(cmpop):
-
-    def to_simple_int(self, space):
-        return 1
+    def to_object(self, space):
+        return space.call_function(get(space).w_Eq)
 State.ast_type('Eq', 'cmpop')
 
 class _NotEq(cmpop):
-
-    def to_simple_int(self, space):
-        return 2
+    def to_object(self, space):
+        return space.call_function(get(space).w_NotEq)
 State.ast_type('NotEq', 'cmpop')
 
 class _Lt(cmpop):
-
-    def to_simple_int(self, space):
-        return 3
+    def to_object(self, space):
+        return space.call_function(get(space).w_Lt)
 State.ast_type('Lt', 'cmpop')
 
 class _LtE(cmpop):
-
-    def to_simple_int(self, space):
-        return 4
+    def to_object(self, space):
+        return space.call_function(get(space).w_LtE)
 State.ast_type('LtE', 'cmpop')
 
 class _Gt(cmpop):
-
-    def to_simple_int(self, space):
-        return 5
+    def to_object(self, space):
+        return space.call_function(get(space).w_Gt)
 State.ast_type('Gt', 'cmpop')
 
 class _GtE(cmpop):
-
-    def to_simple_int(self, space):
-        return 6
+    def to_object(self, space):
+        return space.call_function(get(space).w_GtE)
 State.ast_type('GtE', 'cmpop')
 
 class _Is(cmpop):
-
-    def to_simple_int(self, space):
-        return 7
+    def to_object(self, space):
+        return space.call_function(get(space).w_Is)
 State.ast_type('Is', 'cmpop')
 
 class _IsNot(cmpop):
-
-    def to_simple_int(self, space):
-        return 8
+    def to_object(self, space):
+        return space.call_function(get(space).w_IsNot)
 State.ast_type('IsNot', 'cmpop')
 
 class _In(cmpop):
-
-    def to_simple_int(self, space):
-        return 9
+    def to_object(self, space):
+        return space.call_function(get(space).w_In)
 State.ast_type('In', 'cmpop')
 
 class _NotIn(cmpop):
-
-    def to_simple_int(self, space):
-        return 10
+    def to_object(self, space):
+        return space.call_function(get(space).w_NotIn)
 State.ast_type('NotIn', 'cmpop')
 
 Eq = 1
@@ -2018,9 +1969,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_comprehension)
-        w_target = self.target.to_object(space)  # expr
+        w_target = self.target.to_object(space)
         space.setattr(w_node, space.wrap('target'), w_target)
-        w_iter = self.iter.to_object(space)  # expr
+        w_iter = self.iter.to_object(space)
         space.setattr(w_node, space.wrap('iter'), w_iter)
         if self.ifs is None:
             ifs_w = []
@@ -2060,9 +2011,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
-        w_type = self.type.to_object(space)  # expr
+        w_type = self.type.to_object(space)
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_name = self.name.to_object(space)  # expr
+        w_name = self.name.to_object(space)
         space.setattr(w_node, space.wrap('name'), w_name)
         if self.body is None:
             body_w = []
@@ -2100,9 +2051,9 @@
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
         space.setattr(w_node, space.wrap('args'), w_args)
-        w_vararg = space.wrap(self.vararg)
+        w_vararg = self.vararg.to_object(space)
         space.setattr(w_node, space.wrap('vararg'), w_vararg)
-        w_kwarg = space.wrap(self.kwarg)
+        w_kwarg = self.kwarg.to_object(space)
         space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
         if self.defaults is None:
             defaults_w = []
@@ -2128,9 +2079,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.wrap(self.arg)
+        w_arg = self.arg.to_object(space)
         space.setattr(w_node, space.wrap('arg'), w_arg)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space)
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('keyword', 'AST')
@@ -2149,9 +2100,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_alias)
-        w_name = space.wrap(self.name)
+        w_name = self.name.to_object(space)
         space.setattr(w_node, space.wrap('name'), w_name)
-        w_asname = space.wrap(self.asname)
+        w_asname = self.asname.to_object(space)
         space.setattr(w_node, space.wrap('asname'), w_asname)
         return w_node
 State.ast_type('alias', 'AST')
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -26,5 +26,4 @@
                          lineno=1, col_offset=1)
         w_node = node.to_object(space)
         w_op = space.getattr(w_node, space.wrap("op"))
-        assert space.is_true(space.issubtype(
-                ast.get(space).w_operator, w_op))
+        assert space.isinstance_w(w_op, ast.get(space).w_operator)
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -57,17 +57,13 @@
     def visitSum(self, sum, base):
         if is_simple_sum(sum):
             self.emit("class %s(AST):" % (base,))
-            self.emit("")
-            self.emit("def to_simple_int(self, space):", 1)
-            self.emit("w_msg = space.wrap(\"not a valid %s\")" % (base,), 2)
-            self.emit("raise OperationError(space.w_TypeError, w_msg)", 2)
+            self.emit("pass", 1)
             self.emit("State.ast_type('%s', 'AST')" % (base,))
             self.emit("")
             for i, cons in enumerate(sum.types):
                 self.emit("class _%s(%s):" % (cons.name, base))
-                self.emit("")
-                self.emit("def to_simple_int(self, space):", 1)
-                self.emit("return %i" % (i + 1,), 2)
+                self.emit("def to_object(self, space):", 1)
+                self.emit("return space.call_function(get(space).w_%s)" % (cons.name,), 2)
                 self.emit("State.ast_type('%s', '%s')" % (cons.name, base))
                 self.emit("")
             for i, cons in enumerate(sum.types):
@@ -107,32 +103,28 @@
         self.emit("State.ast_type('%r', 'AST')" % (name,))
         self.emit("")
 
+    def get_value_converter(self, field, value):
+        if field.type.value in self.data.simple_types:
+            return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
+        elif field.type.value in ("object", "string"):
+            return value
+        else:
+            return "%s.to_object(space)" % (value,)
+
     def get_field_converter(self, field):
         if field.seq:
             lines = []
             lines.append("if self.%s is None:" % field.name)
             lines.append("    %s_w = []" % field.name)
             lines.append("else:")
-            if field.type.value in self.data.simple_types:
-                wrapper = "%s_to_class[node - 1]()" % (field.type,)
-            elif field.type.value in ("object", "string"):
-                wrapper = "node"
-            else:
-                wrapper = "node.to_object(space)"
+            wrapper = self.get_value_converter(field, "node")
             lines.append("    %s_w = [%s for node in self.%s] # %s" %
                          (field.name, wrapper, field.name, field.type))
             lines.append("w_%s = space.newlist(%s_w)" % (field.name, field.name))
             return lines
-        elif field.type.value in self.data.simple_types:
-            return ["w_%s = %s_to_class[self.%s - 1]()" % 
-                    (field.name, field.type, field.name)]
-        elif field.type.value in ("object", "string"):
-            return ["w_%s = self.%s" % (field.name, field.name)]
-        elif field.type.value in ("identifier",):
-            return ["w_%s = space.wrap(self.%s)" % (field.name, field.name)]
         else:
-            return ["w_%s = self.%s.to_object(space)  # %s" %
-                    (field.name, field.name, field.type)]
+            wrapper = self.get_value_converter(field, "self.%s" % field.name)
+            return ["w_%s = %s" % (field.name, wrapper)]
 
     def make_converters(self, fields, name):
         self.emit("def to_object(self, space):", 1)

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:07 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:07 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Fixes,
	enough for most ast tests to pass.
Message-ID: <20130626212207.9D5CA1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65016:cc256ebf2bc6
Date: 2013-06-24 20:53 +0200
http://bitbucket.org/pypy/pypy/changeset/cc256ebf2bc6/

Log:	Fixes, enough for most ast tests to pass.

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -180,7 +180,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expression)
-        w_body = self.body.to_object(space)
+        w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
 State.ast_type('Expression', 'mod')
@@ -240,9 +240,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FunctionDef)
-        w_name = self.name.to_object(space)
+        w_name = space.wrap(self.name)  # identifier
         space.setattr(w_node, space.wrap('name'), w_name)
-        w_args = self.args.to_object(space)
+        w_args = self.args.to_object(space)  # arguments
         space.setattr(w_node, space.wrap('args'), w_args)
         if self.body is None:
             body_w = []
@@ -283,7 +283,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ClassDef)
-        w_name = self.name.to_object(space)
+        w_name = space.wrap(self.name)  # identifier
         space.setattr(w_node, space.wrap('name'), w_name)
         if self.bases is None:
             bases_w = []
@@ -323,7 +323,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Return', 'stmt')
@@ -379,7 +379,7 @@
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
         space.setattr(w_node, space.wrap('targets'), w_targets)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Assign', 'stmt')
@@ -403,11 +403,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AugAssign)
-        w_target = self.target.to_object(space)
+        w_target = self.target.to_object(space)  # expr
         space.setattr(w_node, space.wrap('target'), w_target)
-        w_op = operator_to_class[self.op - 1]().to_object(space)
+        w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
         space.setattr(w_node, space.wrap('op'), w_op)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('AugAssign', 'stmt')
@@ -433,7 +433,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Print)
-        w_dest = self.dest.to_object(space)
+        w_dest = self.dest.to_object(space)  # expr
         space.setattr(w_node, space.wrap('dest'), w_dest)
         if self.values is None:
             values_w = []
@@ -441,7 +441,7 @@
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
         space.setattr(w_node, space.wrap('values'), w_values)
-        w_nl = self.nl.to_object(space)
+        w_nl = self.nl.to_object(space)  # bool
         space.setattr(w_node, space.wrap('nl'), w_nl)
         return w_node
 State.ast_type('Print', 'stmt')
@@ -470,9 +470,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_For)
-        w_target = self.target.to_object(space)
+        w_target = self.target.to_object(space)  # expr
         space.setattr(w_node, space.wrap('target'), w_target)
-        w_iter = self.iter.to_object(space)
+        w_iter = self.iter.to_object(space)  # expr
         space.setattr(w_node, space.wrap('iter'), w_iter)
         if self.body is None:
             body_w = []
@@ -511,7 +511,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_While)
-        w_test = self.test.to_object(space)
+        w_test = self.test.to_object(space)  # expr
         space.setattr(w_node, space.wrap('test'), w_test)
         if self.body is None:
             body_w = []
@@ -550,7 +550,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_If)
-        w_test = self.test.to_object(space)
+        w_test = self.test.to_object(space)  # expr
         space.setattr(w_node, space.wrap('test'), w_test)
         if self.body is None:
             body_w = []
@@ -589,9 +589,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_With)
-        w_context_expr = self.context_expr.to_object(space)
+        w_context_expr = self.context_expr.to_object(space)  # expr
         space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
-        w_optional_vars = self.optional_vars.to_object(space)
+        w_optional_vars = self.optional_vars.to_object(space)  # expr
         space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
         if self.body is None:
             body_w = []
@@ -625,11 +625,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
-        w_type = self.type.to_object(space)
+        w_type = self.type.to_object(space)  # expr
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_inst = self.inst.to_object(space)
+        w_inst = self.inst.to_object(space)  # expr
         space.setattr(w_node, space.wrap('inst'), w_inst)
-        w_tback = self.tback.to_object(space)
+        w_tback = self.tback.to_object(space)  # expr
         space.setattr(w_node, space.wrap('tback'), w_tback)
         return w_node
 State.ast_type('Raise', 'stmt')
@@ -732,9 +732,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Assert)
-        w_test = self.test.to_object(space)
+        w_test = self.test.to_object(space)  # expr
         space.setattr(w_node, space.wrap('test'), w_test)
-        w_msg = self.msg.to_object(space)
+        w_msg = self.msg.to_object(space)  # expr
         space.setattr(w_node, space.wrap('msg'), w_msg)
         return w_node
 State.ast_type('Assert', 'stmt')
@@ -784,7 +784,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = self.module.to_object(space)
+        w_module = space.wrap(self.module)  # identifier
         space.setattr(w_node, space.wrap('module'), w_module)
         if self.names is None:
             names_w = []
@@ -792,7 +792,7 @@
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
         space.setattr(w_node, space.wrap('names'), w_names)
-        w_level = self.level.to_object(space)
+        w_level = space.wrap(self.level)  # int
         space.setattr(w_node, space.wrap('level'), w_level)
         return w_node
 State.ast_type('ImportFrom', 'stmt')
@@ -819,11 +819,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Exec)
-        w_body = self.body.to_object(space)
+        w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
-        w_globals = self.globals.to_object(space)
+        w_globals = self.globals.to_object(space)  # expr
         space.setattr(w_node, space.wrap('globals'), w_globals)
-        w_locals = self.locals.to_object(space)
+        w_locals = self.locals.to_object(space)  # expr
         space.setattr(w_node, space.wrap('locals'), w_locals)
         return w_node
 State.ast_type('Exec', 'stmt')
@@ -846,7 +846,7 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [node.to_object(space) for node in self.names] # identifier
+            names_w = [space.wrap(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
         space.setattr(w_node, space.wrap('names'), w_names)
         return w_node
@@ -868,7 +868,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expr)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Expr', 'stmt')
@@ -949,7 +949,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BoolOp)
-        w_op = boolop_to_class[self.op - 1]().to_object(space)
+        w_op = boolop_to_class[self.op - 1]().to_object(space)  # boolop
         space.setattr(w_node, space.wrap('op'), w_op)
         if self.values is None:
             values_w = []
@@ -979,11 +979,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BinOp)
-        w_left = self.left.to_object(space)
+        w_left = self.left.to_object(space)  # expr
         space.setattr(w_node, space.wrap('left'), w_left)
-        w_op = operator_to_class[self.op - 1]().to_object(space)
+        w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
         space.setattr(w_node, space.wrap('op'), w_op)
-        w_right = self.right.to_object(space)
+        w_right = self.right.to_object(space)  # expr
         space.setattr(w_node, space.wrap('right'), w_right)
         return w_node
 State.ast_type('BinOp', 'expr')
@@ -1005,9 +1005,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_UnaryOp)
-        w_op = unaryop_to_class[self.op - 1]().to_object(space)
+        w_op = unaryop_to_class[self.op - 1]().to_object(space)  # unaryop
         space.setattr(w_node, space.wrap('op'), w_op)
-        w_operand = self.operand.to_object(space)
+        w_operand = self.operand.to_object(space)  # expr
         space.setattr(w_node, space.wrap('operand'), w_operand)
         return w_node
 State.ast_type('UnaryOp', 'expr')
@@ -1030,9 +1030,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Lambda)
-        w_args = self.args.to_object(space)
+        w_args = self.args.to_object(space)  # arguments
         space.setattr(w_node, space.wrap('args'), w_args)
-        w_body = self.body.to_object(space)
+        w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
 State.ast_type('Lambda', 'expr')
@@ -1057,11 +1057,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_IfExp)
-        w_test = self.test.to_object(space)
+        w_test = self.test.to_object(space)  # expr
         space.setattr(w_node, space.wrap('test'), w_test)
-        w_body = self.body.to_object(space)
+        w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
-        w_orelse = self.orelse.to_object(space)
+        w_orelse = self.orelse.to_object(space)  # expr
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
         return w_node
 State.ast_type('IfExp', 'expr')
@@ -1146,7 +1146,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ListComp)
-        w_elt = self.elt.to_object(space)
+        w_elt = self.elt.to_object(space)  # expr
         space.setattr(w_node, space.wrap('elt'), w_elt)
         if self.generators is None:
             generators_w = []
@@ -1176,7 +1176,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_SetComp)
-        w_elt = self.elt.to_object(space)
+        w_elt = self.elt.to_object(space)  # expr
         space.setattr(w_node, space.wrap('elt'), w_elt)
         if self.generators is None:
             generators_w = []
@@ -1208,9 +1208,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_DictComp)
-        w_key = self.key.to_object(space)
+        w_key = self.key.to_object(space)  # expr
         space.setattr(w_node, space.wrap('key'), w_key)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         if self.generators is None:
             generators_w = []
@@ -1240,7 +1240,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_GeneratorExp)
-        w_elt = self.elt.to_object(space)
+        w_elt = self.elt.to_object(space)  # expr
         space.setattr(w_node, space.wrap('elt'), w_elt)
         if self.generators is None:
             generators_w = []
@@ -1268,7 +1268,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Yield', 'expr')
@@ -1293,7 +1293,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Compare)
-        w_left = self.left.to_object(space)
+        w_left = self.left.to_object(space)  # expr
         space.setattr(w_node, space.wrap('left'), w_left)
         if self.ops is None:
             ops_w = []
@@ -1338,7 +1338,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Call)
-        w_func = self.func.to_object(space)
+        w_func = self.func.to_object(space)  # expr
         space.setattr(w_node, space.wrap('func'), w_func)
         if self.args is None:
             args_w = []
@@ -1352,9 +1352,9 @@
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
         space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_starargs = self.starargs.to_object(space)
+        w_starargs = self.starargs.to_object(space)  # expr
         space.setattr(w_node, space.wrap('starargs'), w_starargs)
-        w_kwargs = self.kwargs.to_object(space)
+        w_kwargs = self.kwargs.to_object(space)  # expr
         space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
         return w_node
 State.ast_type('Call', 'expr')
@@ -1375,7 +1375,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Repr)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Repr', 'expr')
@@ -1395,7 +1395,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Num)
-        w_n = self.n
+        w_n = self.n  # object
         space.setattr(w_node, space.wrap('n'), w_n)
         return w_node
 State.ast_type('Num', 'expr')
@@ -1415,7 +1415,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Str)
-        w_s = self.s
+        w_s = self.s  # string
         space.setattr(w_node, space.wrap('s'), w_s)
         return w_node
 State.ast_type('Str', 'expr')
@@ -1438,11 +1438,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Attribute)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
-        w_attr = self.attr.to_object(space)
+        w_attr = space.wrap(self.attr)  # identifier
         space.setattr(w_node, space.wrap('attr'), w_attr)
-        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Attribute', 'expr')
@@ -1466,11 +1466,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Subscript)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
-        w_slice = self.slice.to_object(space)
+        w_slice = self.slice.to_object(space)  # slice
         space.setattr(w_node, space.wrap('slice'), w_slice)
-        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Subscript', 'expr')
@@ -1491,9 +1491,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Name)
-        w_id = self.id.to_object(space)
+        w_id = space.wrap(self.id)  # identifier
         space.setattr(w_node, space.wrap('id'), w_id)
-        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Name', 'expr')
@@ -1522,7 +1522,7 @@
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
         space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('List', 'expr')
@@ -1551,7 +1551,7 @@
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
         space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)
+        w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
 State.ast_type('Tuple', 'expr')
@@ -1571,7 +1571,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Const)
-        w_value = self.value
+        w_value = self.value  # object
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Const', 'expr')
@@ -1667,11 +1667,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
-        w_lower = self.lower.to_object(space)
+        w_lower = self.lower.to_object(space)  # expr
         space.setattr(w_node, space.wrap('lower'), w_lower)
-        w_upper = self.upper.to_object(space)
+        w_upper = self.upper.to_object(space)  # expr
         space.setattr(w_node, space.wrap('upper'), w_upper)
-        w_step = self.step.to_object(space)
+        w_step = self.step.to_object(space)  # expr
         space.setattr(w_node, space.wrap('step'), w_step)
         return w_node
 State.ast_type('Slice', 'slice')
@@ -1716,7 +1716,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Index)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('Index', 'slice')
@@ -1969,9 +1969,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_comprehension)
-        w_target = self.target.to_object(space)
+        w_target = self.target.to_object(space)  # expr
         space.setattr(w_node, space.wrap('target'), w_target)
-        w_iter = self.iter.to_object(space)
+        w_iter = self.iter.to_object(space)  # expr
         space.setattr(w_node, space.wrap('iter'), w_iter)
         if self.ifs is None:
             ifs_w = []
@@ -2011,9 +2011,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
-        w_type = self.type.to_object(space)
+        w_type = self.type.to_object(space)  # expr
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_name = self.name.to_object(space)
+        w_name = self.name.to_object(space)  # expr
         space.setattr(w_node, space.wrap('name'), w_name)
         if self.body is None:
             body_w = []
@@ -2051,9 +2051,9 @@
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
         space.setattr(w_node, space.wrap('args'), w_args)
-        w_vararg = self.vararg.to_object(space)
+        w_vararg = space.wrap(self.vararg)  # identifier
         space.setattr(w_node, space.wrap('vararg'), w_vararg)
-        w_kwarg = self.kwarg.to_object(space)
+        w_kwarg = space.wrap(self.kwarg)  # identifier
         space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
         if self.defaults is None:
             defaults_w = []
@@ -2079,9 +2079,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = self.arg.to_object(space)
+        w_arg = space.wrap(self.arg)  # identifier
         space.setattr(w_node, space.wrap('arg'), w_arg)
-        w_value = self.value.to_object(space)
+        w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
 State.ast_type('keyword', 'AST')
@@ -2100,9 +2100,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_alias)
-        w_name = self.name.to_object(space)
+        w_name = space.wrap(self.name)  # identifier
         space.setattr(w_node, space.wrap('name'), w_name)
-        w_asname = self.asname.to_object(space)
+        w_asname = space.wrap(self.asname)  # identifier
         space.setattr(w_node, space.wrap('asname'), w_asname)
         return w_node
 State.ast_type('alias', 'AST')
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -108,6 +108,8 @@
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
         elif field.type.value in ("object", "string"):
             return value
+        elif field.type.value in ("identifier", "int"):
+            return "space.wrap(%s)" % (value,)
         else:
             return "%s.to_object(space)" % (value,)
 
@@ -124,7 +126,7 @@
             return lines
         else:
             wrapper = self.get_value_converter(field, "self.%s" % field.name)
-            return ["w_%s = %s" % (field.name, wrapper)]
+            return ["w_%s = %s  # %s" % (field.name, wrapper, field.type)]
 
     def make_converters(self, fields, name):
         self.emit("def to_object(self, space):", 1)

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:08 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:08 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Progress: obj2mod()
Message-ID: <20130626212208.C9C141C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65017:14092bb5545c
Date: 2013-06-24 21:34 +0200
http://bitbucket.org/pypy/pypy/changeset/14092bb5545c/

Log:	Progress: obj2mod()

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -34,6 +34,13 @@
 class W_AST(W_Root):
     w_dict = None
 
+    def obj2mod(self, space):
+        if space.isinstance_w(self, get(space).w_Module):
+            return Module.from_object(space, self)
+        else:
+            raise OperationError(space.w_TypeError, space.wrap(
+                    "Expected mod node"))
+
     def getdict(self, space):
         if self.w_dict is None:
             self.w_dict = space.newdict(instance=True)
@@ -138,6 +145,12 @@
         w_body = space.newlist(body_w)
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        body = space.getattr(w_node, space.wrap('body'))
+        return Module(body)
+
 State.ast_type('Module', 'mod')
 
 
@@ -163,6 +176,12 @@
         w_body = space.newlist(body_w)
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        body = space.getattr(w_node, space.wrap('body'))
+        return Interactive(body)
+
 State.ast_type('Interactive', 'mod')
 
 
@@ -183,6 +202,12 @@
         w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        body = space.getattr(w_node, space.wrap('body'))
+        return Expression(body)
+
 State.ast_type('Expression', 'mod')
 
 
@@ -208,6 +233,12 @@
         w_body = space.newlist(body_w)
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        body = space.getattr(w_node, space.wrap('body'))
+        return Suite(body)
+
 State.ast_type('Suite', 'mod')
 
 
@@ -257,6 +288,15 @@
         w_decorator_list = space.newlist(decorator_list_w)
         space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        name = space.getattr(w_node, space.wrap('name'))
+        args = space.getattr(w_node, space.wrap('args'))
+        body = space.getattr(w_node, space.wrap('body'))
+        decorator_list = space.getattr(w_node, space.wrap('decorator_list'))
+        return FunctionDef(name, args, body, decorator_list)
+
 State.ast_type('FunctionDef', 'stmt')
 
 
@@ -304,6 +344,15 @@
         w_decorator_list = space.newlist(decorator_list_w)
         space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        name = space.getattr(w_node, space.wrap('name'))
+        bases = space.getattr(w_node, space.wrap('bases'))
+        body = space.getattr(w_node, space.wrap('body'))
+        decorator_list = space.getattr(w_node, space.wrap('decorator_list'))
+        return ClassDef(name, bases, body, decorator_list)
+
 State.ast_type('ClassDef', 'stmt')
 
 
@@ -326,6 +375,12 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        return Return(value)
+
 State.ast_type('Return', 'stmt')
 
 
@@ -352,6 +407,12 @@
         w_targets = space.newlist(targets_w)
         space.setattr(w_node, space.wrap('targets'), w_targets)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        targets = space.getattr(w_node, space.wrap('targets'))
+        return Delete(targets)
+
 State.ast_type('Delete', 'stmt')
 
 
@@ -382,6 +443,13 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        targets = space.getattr(w_node, space.wrap('targets'))
+        value = space.getattr(w_node, space.wrap('value'))
+        return Assign(targets, value)
+
 State.ast_type('Assign', 'stmt')
 
 
@@ -410,6 +478,14 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        target = space.getattr(w_node, space.wrap('target'))
+        op = space.getattr(w_node, space.wrap('op'))
+        value = space.getattr(w_node, space.wrap('value'))
+        return AugAssign(target, op, value)
+
 State.ast_type('AugAssign', 'stmt')
 
 
@@ -444,6 +520,14 @@
         w_nl = self.nl.to_object(space)  # bool
         space.setattr(w_node, space.wrap('nl'), w_nl)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        dest = space.getattr(w_node, space.wrap('dest'))
+        values = space.getattr(w_node, space.wrap('values'))
+        nl = space.getattr(w_node, space.wrap('nl'))
+        return Print(dest, values, nl)
+
 State.ast_type('Print', 'stmt')
 
 
@@ -487,6 +571,15 @@
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        target = space.getattr(w_node, space.wrap('target'))
+        iter = space.getattr(w_node, space.wrap('iter'))
+        body = space.getattr(w_node, space.wrap('body'))
+        orelse = space.getattr(w_node, space.wrap('orelse'))
+        return For(target, iter, body, orelse)
+
 State.ast_type('For', 'stmt')
 
 
@@ -526,6 +619,14 @@
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        test = space.getattr(w_node, space.wrap('test'))
+        body = space.getattr(w_node, space.wrap('body'))
+        orelse = space.getattr(w_node, space.wrap('orelse'))
+        return While(test, body, orelse)
+
 State.ast_type('While', 'stmt')
 
 
@@ -565,6 +666,14 @@
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        test = space.getattr(w_node, space.wrap('test'))
+        body = space.getattr(w_node, space.wrap('body'))
+        orelse = space.getattr(w_node, space.wrap('orelse'))
+        return If(test, body, orelse)
+
 State.ast_type('If', 'stmt')
 
 
@@ -600,6 +709,14 @@
         w_body = space.newlist(body_w)
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        context_expr = space.getattr(w_node, space.wrap('context_expr'))
+        optional_vars = space.getattr(w_node, space.wrap('optional_vars'))
+        body = space.getattr(w_node, space.wrap('body'))
+        return With(context_expr, optional_vars, body)
+
 State.ast_type('With', 'stmt')
 
 
@@ -632,6 +749,14 @@
         w_tback = self.tback.to_object(space)  # expr
         space.setattr(w_node, space.wrap('tback'), w_tback)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        type = space.getattr(w_node, space.wrap('type'))
+        inst = space.getattr(w_node, space.wrap('inst'))
+        tback = space.getattr(w_node, space.wrap('tback'))
+        return Raise(type, inst, tback)
+
 State.ast_type('Raise', 'stmt')
 
 
@@ -676,6 +801,14 @@
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        body = space.getattr(w_node, space.wrap('body'))
+        handlers = space.getattr(w_node, space.wrap('handlers'))
+        orelse = space.getattr(w_node, space.wrap('orelse'))
+        return TryExcept(body, handlers, orelse)
+
 State.ast_type('TryExcept', 'stmt')
 
 
@@ -711,6 +844,13 @@
         w_finalbody = space.newlist(finalbody_w)
         space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        body = space.getattr(w_node, space.wrap('body'))
+        finalbody = space.getattr(w_node, space.wrap('finalbody'))
+        return TryFinally(body, finalbody)
+
 State.ast_type('TryFinally', 'stmt')
 
 
@@ -737,6 +877,13 @@
         w_msg = self.msg.to_object(space)  # expr
         space.setattr(w_node, space.wrap('msg'), w_msg)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        test = space.getattr(w_node, space.wrap('test'))
+        msg = space.getattr(w_node, space.wrap('msg'))
+        return Assert(test, msg)
+
 State.ast_type('Assert', 'stmt')
 
 
@@ -763,6 +910,12 @@
         w_names = space.newlist(names_w)
         space.setattr(w_node, space.wrap('names'), w_names)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        names = space.getattr(w_node, space.wrap('names'))
+        return Import(names)
+
 State.ast_type('Import', 'stmt')
 
 
@@ -795,6 +948,14 @@
         w_level = space.wrap(self.level)  # int
         space.setattr(w_node, space.wrap('level'), w_level)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        module = space.getattr(w_node, space.wrap('module'))
+        names = space.getattr(w_node, space.wrap('names'))
+        level = space.getattr(w_node, space.wrap('level'))
+        return ImportFrom(module, names, level)
+
 State.ast_type('ImportFrom', 'stmt')
 
 
@@ -826,6 +987,14 @@
         w_locals = self.locals.to_object(space)  # expr
         space.setattr(w_node, space.wrap('locals'), w_locals)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        body = space.getattr(w_node, space.wrap('body'))
+        globals = space.getattr(w_node, space.wrap('globals'))
+        locals = space.getattr(w_node, space.wrap('locals'))
+        return Exec(body, globals, locals)
+
 State.ast_type('Exec', 'stmt')
 
 
@@ -850,6 +1019,12 @@
         w_names = space.newlist(names_w)
         space.setattr(w_node, space.wrap('names'), w_names)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        names = space.getattr(w_node, space.wrap('names'))
+        return Global(names)
+
 State.ast_type('Global', 'stmt')
 
 
@@ -871,6 +1046,12 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        return Expr(value)
+
 State.ast_type('Expr', 'stmt')
 
 
@@ -888,6 +1069,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Pass)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        return Pass()
+
 State.ast_type('Pass', 'stmt')
 
 
@@ -905,6 +1091,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Break)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        return Break()
+
 State.ast_type('Break', 'stmt')
 
 
@@ -922,6 +1113,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Continue)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        return Continue()
+
 State.ast_type('Continue', 'stmt')
 
 
@@ -958,6 +1154,13 @@
         w_values = space.newlist(values_w)
         space.setattr(w_node, space.wrap('values'), w_values)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        op = space.getattr(w_node, space.wrap('op'))
+        values = space.getattr(w_node, space.wrap('values'))
+        return BoolOp(op, values)
+
 State.ast_type('BoolOp', 'expr')
 
 
@@ -986,6 +1189,14 @@
         w_right = self.right.to_object(space)  # expr
         space.setattr(w_node, space.wrap('right'), w_right)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        left = space.getattr(w_node, space.wrap('left'))
+        op = space.getattr(w_node, space.wrap('op'))
+        right = space.getattr(w_node, space.wrap('right'))
+        return BinOp(left, op, right)
+
 State.ast_type('BinOp', 'expr')
 
 
@@ -1010,6 +1221,13 @@
         w_operand = self.operand.to_object(space)  # expr
         space.setattr(w_node, space.wrap('operand'), w_operand)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        op = space.getattr(w_node, space.wrap('op'))
+        operand = space.getattr(w_node, space.wrap('operand'))
+        return UnaryOp(op, operand)
+
 State.ast_type('UnaryOp', 'expr')
 
 
@@ -1035,6 +1253,13 @@
         w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        args = space.getattr(w_node, space.wrap('args'))
+        body = space.getattr(w_node, space.wrap('body'))
+        return Lambda(args, body)
+
 State.ast_type('Lambda', 'expr')
 
 
@@ -1064,6 +1289,14 @@
         w_orelse = self.orelse.to_object(space)  # expr
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        test = space.getattr(w_node, space.wrap('test'))
+        body = space.getattr(w_node, space.wrap('body'))
+        orelse = space.getattr(w_node, space.wrap('orelse'))
+        return IfExp(test, body, orelse)
+
 State.ast_type('IfExp', 'expr')
 
 
@@ -1099,6 +1332,13 @@
         w_values = space.newlist(values_w)
         space.setattr(w_node, space.wrap('values'), w_values)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        keys = space.getattr(w_node, space.wrap('keys'))
+        values = space.getattr(w_node, space.wrap('values'))
+        return Dict(keys, values)
+
 State.ast_type('Dict', 'expr')
 
 
@@ -1125,6 +1365,12 @@
         w_elts = space.newlist(elts_w)
         space.setattr(w_node, space.wrap('elts'), w_elts)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        elts = space.getattr(w_node, space.wrap('elts'))
+        return Set(elts)
+
 State.ast_type('Set', 'expr')
 
 
@@ -1155,6 +1401,13 @@
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        elt = space.getattr(w_node, space.wrap('elt'))
+        generators = space.getattr(w_node, space.wrap('generators'))
+        return ListComp(elt, generators)
+
 State.ast_type('ListComp', 'expr')
 
 
@@ -1185,6 +1438,13 @@
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        elt = space.getattr(w_node, space.wrap('elt'))
+        generators = space.getattr(w_node, space.wrap('generators'))
+        return SetComp(elt, generators)
+
 State.ast_type('SetComp', 'expr')
 
 
@@ -1219,6 +1479,14 @@
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        key = space.getattr(w_node, space.wrap('key'))
+        value = space.getattr(w_node, space.wrap('value'))
+        generators = space.getattr(w_node, space.wrap('generators'))
+        return DictComp(key, value, generators)
+
 State.ast_type('DictComp', 'expr')
 
 
@@ -1249,6 +1517,13 @@
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        elt = space.getattr(w_node, space.wrap('elt'))
+        generators = space.getattr(w_node, space.wrap('generators'))
+        return GeneratorExp(elt, generators)
+
 State.ast_type('GeneratorExp', 'expr')
 
 
@@ -1271,6 +1546,12 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        return Yield(value)
+
 State.ast_type('Yield', 'expr')
 
 
@@ -1308,6 +1589,14 @@
         w_comparators = space.newlist(comparators_w)
         space.setattr(w_node, space.wrap('comparators'), w_comparators)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        left = space.getattr(w_node, space.wrap('left'))
+        ops = space.getattr(w_node, space.wrap('ops'))
+        comparators = space.getattr(w_node, space.wrap('comparators'))
+        return Compare(left, ops, comparators)
+
 State.ast_type('Compare', 'expr')
 
 
@@ -1357,6 +1646,16 @@
         w_kwargs = self.kwargs.to_object(space)  # expr
         space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        func = space.getattr(w_node, space.wrap('func'))
+        args = space.getattr(w_node, space.wrap('args'))
+        keywords = space.getattr(w_node, space.wrap('keywords'))
+        starargs = space.getattr(w_node, space.wrap('starargs'))
+        kwargs = space.getattr(w_node, space.wrap('kwargs'))
+        return Call(func, args, keywords, starargs, kwargs)
+
 State.ast_type('Call', 'expr')
 
 
@@ -1378,6 +1677,12 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        return Repr(value)
+
 State.ast_type('Repr', 'expr')
 
 
@@ -1398,6 +1703,12 @@
         w_n = self.n  # object
         space.setattr(w_node, space.wrap('n'), w_n)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        n = space.getattr(w_node, space.wrap('n'))
+        return Num(n)
+
 State.ast_type('Num', 'expr')
 
 
@@ -1418,6 +1729,12 @@
         w_s = self.s  # string
         space.setattr(w_node, space.wrap('s'), w_s)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        s = space.getattr(w_node, space.wrap('s'))
+        return Str(s)
+
 State.ast_type('Str', 'expr')
 
 
@@ -1445,6 +1762,14 @@
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        attr = space.getattr(w_node, space.wrap('attr'))
+        ctx = space.getattr(w_node, space.wrap('ctx'))
+        return Attribute(value, attr, ctx)
+
 State.ast_type('Attribute', 'expr')
 
 
@@ -1473,6 +1798,14 @@
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        slice = space.getattr(w_node, space.wrap('slice'))
+        ctx = space.getattr(w_node, space.wrap('ctx'))
+        return Subscript(value, slice, ctx)
+
 State.ast_type('Subscript', 'expr')
 
 
@@ -1496,6 +1829,13 @@
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        id = space.getattr(w_node, space.wrap('id'))
+        ctx = space.getattr(w_node, space.wrap('ctx'))
+        return Name(id, ctx)
+
 State.ast_type('Name', 'expr')
 
 
@@ -1525,6 +1865,13 @@
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        elts = space.getattr(w_node, space.wrap('elts'))
+        ctx = space.getattr(w_node, space.wrap('ctx'))
+        return List(elts, ctx)
+
 State.ast_type('List', 'expr')
 
 
@@ -1554,6 +1901,13 @@
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        elts = space.getattr(w_node, space.wrap('elts'))
+        ctx = space.getattr(w_node, space.wrap('ctx'))
+        return Tuple(elts, ctx)
+
 State.ast_type('Tuple', 'expr')
 
 
@@ -1574,6 +1928,12 @@
         w_value = self.value  # object
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        return Const(value)
+
 State.ast_type('Const', 'expr')
 
 
@@ -1643,6 +2003,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Ellipsis)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        return Ellipsis()
+
 State.ast_type('Ellipsis', 'slice')
 
 
@@ -1674,6 +2039,14 @@
         w_step = self.step.to_object(space)  # expr
         space.setattr(w_node, space.wrap('step'), w_step)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        lower = space.getattr(w_node, space.wrap('lower'))
+        upper = space.getattr(w_node, space.wrap('upper'))
+        step = space.getattr(w_node, space.wrap('step'))
+        return Slice(lower, upper, step)
+
 State.ast_type('Slice', 'slice')
 
 
@@ -1699,6 +2072,12 @@
         w_dims = space.newlist(dims_w)
         space.setattr(w_node, space.wrap('dims'), w_dims)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        dims = space.getattr(w_node, space.wrap('dims'))
+        return ExtSlice(dims)
+
 State.ast_type('ExtSlice', 'slice')
 
 
@@ -1719,6 +2098,12 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        value = space.getattr(w_node, space.wrap('value'))
+        return Index(value)
+
 State.ast_type('Index', 'slice')
 
 
@@ -1980,6 +2365,14 @@
         w_ifs = space.newlist(ifs_w)
         space.setattr(w_node, space.wrap('ifs'), w_ifs)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        target = space.getattr(w_node, space.wrap('target'))
+        iter = space.getattr(w_node, space.wrap('iter'))
+        ifs = space.getattr(w_node, space.wrap('ifs'))
+        return comprehension(target, iter, ifs)
+
 State.ast_type('comprehension', 'AST')
 
 class excepthandler(AST):
@@ -2022,6 +2415,14 @@
         w_body = space.newlist(body_w)
         space.setattr(w_node, space.wrap('body'), w_body)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        type = space.getattr(w_node, space.wrap('type'))
+        name = space.getattr(w_node, space.wrap('name'))
+        body = space.getattr(w_node, space.wrap('body'))
+        return ExceptHandler(type, name, body)
+
 State.ast_type('ExceptHandler', 'excepthandler')
 
 
@@ -2062,6 +2463,15 @@
         w_defaults = space.newlist(defaults_w)
         space.setattr(w_node, space.wrap('defaults'), w_defaults)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        args = space.getattr(w_node, space.wrap('args'))
+        vararg = space.getattr(w_node, space.wrap('vararg'))
+        kwarg = space.getattr(w_node, space.wrap('kwarg'))
+        defaults = space.getattr(w_node, space.wrap('defaults'))
+        return arguments(args, vararg, kwarg, defaults)
+
 State.ast_type('arguments', 'AST')
 
 class keyword(AST):
@@ -2084,6 +2494,13 @@
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        arg = space.getattr(w_node, space.wrap('arg'))
+        value = space.getattr(w_node, space.wrap('value'))
+        return keyword(arg, value)
+
 State.ast_type('keyword', 'AST')
 
 class alias(AST):
@@ -2105,6 +2522,13 @@
         w_asname = space.wrap(self.asname)  # identifier
         space.setattr(w_node, space.wrap('asname'), w_asname)
         return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        name = space.getattr(w_node, space.wrap('name'))
+        asname = space.getattr(w_node, space.wrap('asname'))
+        return alias(name, asname)
+
 State.ast_type('alias', 'AST')
 
 class ASTVisitor(object):
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -27,3 +27,11 @@
         w_node = node.to_object(space)
         w_op = space.getattr(w_node, space.wrap("op"))
         assert space.isinstance_w(w_op, ast.get(space).w_operator)
+
+    def from_object(self, space):
+        value = space.wrap(42)
+        w_node = space.call_function(ast.get(space).w_Num)
+        space.setattr(w_node, space.wrap('n'), value)
+        node = ast.Num.from_object(space, w_node)
+        assert node.n is value
+        
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -128,6 +128,10 @@
             wrapper = self.get_value_converter(field, "self.%s" % field.name)
             return ["w_%s = %s  # %s" % (field.name, wrapper, field.type)]
 
+    def get_field_extractor(self, field):
+        lines = ["%s = space.getattr(w_node, space.wrap('%s'))" % (field.name, field.name)]
+        return lines
+
     def make_converters(self, fields, name):
         self.emit("def to_object(self, space):", 1)
         self.emit("w_node = space.call_function(get(space).w_%s)" % name, 2)
@@ -138,6 +142,16 @@
             self.emit("space.setattr(w_node, space.wrap(%r), w_%s)" % (
                     str(field.name), field.name), 2)
         self.emit("return w_node", 2)
+        self.emit("")
+        self.emit("@staticmethod", 1)
+        self.emit("def from_object(space, w_node):", 1)
+        for field in fields:
+            unwrapping_code = self.get_field_extractor(field)
+            for line in unwrapping_code:
+                self.emit(line, 2)
+        self.emit("return %s(%s)" % (
+                name, ', '.join(str(field.name) for field in fields)), 2)
+        self.emit("")
 
     def make_constructor(self, fields, node, extras=None, base=None):
         if fields or extras:
@@ -349,6 +363,13 @@
 class W_AST(W_Root):
     w_dict = None
 
+    def obj2mod(self, space):
+        if space.isinstance_w(self, get(space).w_Module):
+            return Module.from_object(space, self)
+        else:
+            raise OperationError(space.w_TypeError, space.wrap(
+                    "Expected mod node"))
+
     def getdict(self, space):
         if self.w_dict is None:
             self.w_dict = space.newdict(instance=True)
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -28,8 +28,7 @@
     w_ast_type = space.gettypeobject(ast.W_AST.typedef)
     str_ = None
     if space.isinstance_w(w_source, w_ast_type):
-        ast_node = space.interp_w(ast.mod, w_source)
-        ast_node.sync_app_attrs(space)
+        ast_node = space.interp_w(ast.W_AST, w_source).obj2mod(space)
     elif space.isinstance_w(w_source, space.w_unicode):
         w_utf_8_source = space.call_method(w_source, "encode",
                                            space.wrap("utf-8"))

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:10 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:10 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Progress: Implement all
	from_object() methods
Message-ID: <20130626212210.103601C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65018:0354de7ac1fb
Date: 2013-06-24 22:23 +0200
http://bitbucket.org/pypy/pypy/changeset/0354de7ac1fb/

Log:	Progress: Implement all from_object() methods

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1,12 +1,27 @@
 # Generated by tools/asdl_py.py
-from pypy.interpreter.error import OperationError
-from rpython.tool.pairtype import extendabletype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.error import OperationError, operationerrfmt
+from rpython.tool.pairtype import extendabletype
 from rpython.tool.sourcetools import func_with_new_name
 
 
+def check_string(space, w_obj):
+    if not (space.isinstance_w(w_obj, space.w_str) or
+            space.isinstance_w(w_obj, space.w_unicode)):
+        raise OperationError(space.w_TypeError, space.wrap(
+                'AST string must be of type str or unicode'))
+    return w_obj
+
+def get_field(space, w_node, name, optional):
+    w_obj = w_node.getdictvalue(space, name)
+    if w_obj is None and not optional:
+        raise operationerrfmt(space.w_TypeError,
+                "required field \"%s\" missing from %T", name, w_node)
+    return w_obj
+
+
 class AST(object):
     __metaclass__ = extendabletype
 
@@ -34,13 +49,6 @@
 class W_AST(W_Root):
     w_dict = None
 
-    def obj2mod(self, space):
-        if space.isinstance_w(self, get(space).w_Module):
-            return Module.from_object(space, self)
-        else:
-            raise OperationError(space.w_TypeError, space.wrap(
-                    "Expected mod node"))
-
     def getdict(self, space):
         if self.w_dict is None:
             self.w_dict = space.newdict(instance=True)
@@ -120,7 +128,18 @@
     return space.fromcache(State)
 
 class mod(AST):
-    pass
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_Module):
+            return Module.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Interactive):
+            return Interactive.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Expression):
+            return Expression.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Suite):
+            return Suite.from_object(space, w_node)
+        raise operationerrfmt(space.w_TypeError,
+                "Expected mod node, got %T", w_node)
 State.ast_type('mod', 'AST')
 
 class Module(mod):
@@ -148,7 +167,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = space.getattr(w_node, space.wrap('body'))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Module(body)
 
 State.ast_type('Module', 'mod')
@@ -179,7 +200,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = space.getattr(w_node, space.wrap('body'))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Interactive(body)
 
 State.ast_type('Interactive', 'mod')
@@ -205,7 +228,7 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = space.getattr(w_node, space.wrap('body'))
+        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
         return Expression(body)
 
 State.ast_type('Expression', 'mod')
@@ -236,7 +259,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = space.getattr(w_node, space.wrap('body'))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Suite(body)
 
 State.ast_type('Suite', 'mod')
@@ -247,6 +272,57 @@
     def __init__(self, lineno, col_offset):
         self.lineno = lineno
         self.col_offset = col_offset
+
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_FunctionDef):
+            return FunctionDef.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_ClassDef):
+            return ClassDef.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Return):
+            return Return.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Delete):
+            return Delete.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Assign):
+            return Assign.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_AugAssign):
+            return AugAssign.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Print):
+            return Print.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_For):
+            return For.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_While):
+            return While.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_If):
+            return If.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_With):
+            return With.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Raise):
+            return Raise.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_TryExcept):
+            return TryExcept.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_TryFinally):
+            return TryFinally.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Assert):
+            return Assert.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Import):
+            return Import.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_ImportFrom):
+            return ImportFrom.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Exec):
+            return Exec.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Global):
+            return Global.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Expr):
+            return Expr.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Pass):
+            return Pass.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Break):
+            return Break.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Continue):
+            return Continue.from_object(space, w_node)
+        raise operationerrfmt(space.w_TypeError,
+                "Expected stmt node, got %T", w_node)
 State.ast_type('stmt', 'AST')
 
 class FunctionDef(stmt):
@@ -287,15 +363,25 @@
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
         space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.getattr(w_node, space.wrap('name'))
-        args = space.getattr(w_node, space.wrap('args'))
-        body = space.getattr(w_node, space.wrap('body'))
-        decorator_list = space.getattr(w_node, space.wrap('decorator_list'))
-        return FunctionDef(name, args, body, decorator_list)
+        name = space.str_w(space.getattr(w_node, space.wrap('name')))
+        args = arguments.from_object(space, space.getattr(w_node, space.wrap('args')))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        decorator_list_w = space.unpackiterable(
+                get_field(space, w_node, 'decorator_list', False))
+        decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return FunctionDef(name, args, body, decorator_list, lineno, col_offset)
 
 State.ast_type('FunctionDef', 'stmt')
 
@@ -343,15 +429,27 @@
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
         space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.getattr(w_node, space.wrap('name'))
-        bases = space.getattr(w_node, space.wrap('bases'))
-        body = space.getattr(w_node, space.wrap('body'))
-        decorator_list = space.getattr(w_node, space.wrap('decorator_list'))
-        return ClassDef(name, bases, body, decorator_list)
+        name = space.str_w(space.getattr(w_node, space.wrap('name')))
+        bases_w = space.unpackiterable(
+                get_field(space, w_node, 'bases', False))
+        bases = [expr.from_object(space, w_item) for w_item in bases_w]
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        decorator_list_w = space.unpackiterable(
+                get_field(space, w_node, 'decorator_list', False))
+        decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return ClassDef(name, bases, body, decorator_list, lineno, col_offset)
 
 State.ast_type('ClassDef', 'stmt')
 
@@ -374,12 +472,18 @@
         w_node = space.call_function(get(space).w_Return)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
-        return Return(value)
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Return(value, lineno, col_offset)
 
 State.ast_type('Return', 'stmt')
 
@@ -406,12 +510,20 @@
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
         space.setattr(w_node, space.wrap('targets'), w_targets)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        targets = space.getattr(w_node, space.wrap('targets'))
-        return Delete(targets)
+        targets_w = space.unpackiterable(
+                get_field(space, w_node, 'targets', False))
+        targets = [expr.from_object(space, w_item) for w_item in targets_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Delete(targets, lineno, col_offset)
 
 State.ast_type('Delete', 'stmt')
 
@@ -442,13 +554,21 @@
         space.setattr(w_node, space.wrap('targets'), w_targets)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        targets = space.getattr(w_node, space.wrap('targets'))
-        value = space.getattr(w_node, space.wrap('value'))
-        return Assign(targets, value)
+        targets_w = space.unpackiterable(
+                get_field(space, w_node, 'targets', False))
+        targets = [expr.from_object(space, w_item) for w_item in targets_w]
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Assign(targets, value, lineno, col_offset)
 
 State.ast_type('Assign', 'stmt')
 
@@ -477,14 +597,20 @@
         space.setattr(w_node, space.wrap('op'), w_op)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        target = space.getattr(w_node, space.wrap('target'))
-        op = space.getattr(w_node, space.wrap('op'))
-        value = space.getattr(w_node, space.wrap('value'))
-        return AugAssign(target, op, value)
+        target = expr.from_object(space, space.getattr(w_node, space.wrap('target')))
+        op = operator.from_object(space, space.getattr(w_node, space.wrap('op')))
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return AugAssign(target, op, value, lineno, col_offset)
 
 State.ast_type('AugAssign', 'stmt')
 
@@ -519,14 +645,22 @@
         space.setattr(w_node, space.wrap('values'), w_values)
         w_nl = self.nl.to_object(space)  # bool
         space.setattr(w_node, space.wrap('nl'), w_nl)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        dest = space.getattr(w_node, space.wrap('dest'))
-        values = space.getattr(w_node, space.wrap('values'))
-        nl = space.getattr(w_node, space.wrap('nl'))
-        return Print(dest, values, nl)
+        dest = expr.from_object(space, space.getattr(w_node, space.wrap('dest')))
+        values_w = space.unpackiterable(
+                get_field(space, w_node, 'values', False))
+        values = [expr.from_object(space, w_item) for w_item in values_w]
+        nl = bool.from_object(space, space.getattr(w_node, space.wrap('nl')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Print(dest, values, nl, lineno, col_offset)
 
 State.ast_type('Print', 'stmt')
 
@@ -570,15 +704,25 @@
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        target = space.getattr(w_node, space.wrap('target'))
-        iter = space.getattr(w_node, space.wrap('iter'))
-        body = space.getattr(w_node, space.wrap('body'))
-        orelse = space.getattr(w_node, space.wrap('orelse'))
-        return For(target, iter, body, orelse)
+        target = expr.from_object(space, space.getattr(w_node, space.wrap('target')))
+        iter = expr.from_object(space, space.getattr(w_node, space.wrap('iter')))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        orelse_w = space.unpackiterable(
+                get_field(space, w_node, 'orelse', False))
+        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return For(target, iter, body, orelse, lineno, col_offset)
 
 State.ast_type('For', 'stmt')
 
@@ -618,14 +762,24 @@
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        test = space.getattr(w_node, space.wrap('test'))
-        body = space.getattr(w_node, space.wrap('body'))
-        orelse = space.getattr(w_node, space.wrap('orelse'))
-        return While(test, body, orelse)
+        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        orelse_w = space.unpackiterable(
+                get_field(space, w_node, 'orelse', False))
+        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return While(test, body, orelse, lineno, col_offset)
 
 State.ast_type('While', 'stmt')
 
@@ -665,14 +819,24 @@
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        test = space.getattr(w_node, space.wrap('test'))
-        body = space.getattr(w_node, space.wrap('body'))
-        orelse = space.getattr(w_node, space.wrap('orelse'))
-        return If(test, body, orelse)
+        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        orelse_w = space.unpackiterable(
+                get_field(space, w_node, 'orelse', False))
+        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return If(test, body, orelse, lineno, col_offset)
 
 State.ast_type('If', 'stmt')
 
@@ -708,14 +872,22 @@
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
         space.setattr(w_node, space.wrap('body'), w_body)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        context_expr = space.getattr(w_node, space.wrap('context_expr'))
-        optional_vars = space.getattr(w_node, space.wrap('optional_vars'))
-        body = space.getattr(w_node, space.wrap('body'))
-        return With(context_expr, optional_vars, body)
+        context_expr = expr.from_object(space, space.getattr(w_node, space.wrap('context_expr')))
+        optional_vars = expr.from_object(space, space.getattr(w_node, space.wrap('optional_vars')))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return With(context_expr, optional_vars, body, lineno, col_offset)
 
 State.ast_type('With', 'stmt')
 
@@ -748,14 +920,20 @@
         space.setattr(w_node, space.wrap('inst'), w_inst)
         w_tback = self.tback.to_object(space)  # expr
         space.setattr(w_node, space.wrap('tback'), w_tback)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        type = space.getattr(w_node, space.wrap('type'))
-        inst = space.getattr(w_node, space.wrap('inst'))
-        tback = space.getattr(w_node, space.wrap('tback'))
-        return Raise(type, inst, tback)
+        type = expr.from_object(space, space.getattr(w_node, space.wrap('type')))
+        inst = expr.from_object(space, space.getattr(w_node, space.wrap('inst')))
+        tback = expr.from_object(space, space.getattr(w_node, space.wrap('tback')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Raise(type, inst, tback, lineno, col_offset)
 
 State.ast_type('Raise', 'stmt')
 
@@ -800,14 +978,26 @@
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        body = space.getattr(w_node, space.wrap('body'))
-        handlers = space.getattr(w_node, space.wrap('handlers'))
-        orelse = space.getattr(w_node, space.wrap('orelse'))
-        return TryExcept(body, handlers, orelse)
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        handlers_w = space.unpackiterable(
+                get_field(space, w_node, 'handlers', False))
+        handlers = [excepthandler.from_object(space, w_item) for w_item in handlers_w]
+        orelse_w = space.unpackiterable(
+                get_field(space, w_node, 'orelse', False))
+        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return TryExcept(body, handlers, orelse, lineno, col_offset)
 
 State.ast_type('TryExcept', 'stmt')
 
@@ -843,13 +1033,23 @@
             finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt
         w_finalbody = space.newlist(finalbody_w)
         space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        body = space.getattr(w_node, space.wrap('body'))
-        finalbody = space.getattr(w_node, space.wrap('finalbody'))
-        return TryFinally(body, finalbody)
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        finalbody_w = space.unpackiterable(
+                get_field(space, w_node, 'finalbody', False))
+        finalbody = [stmt.from_object(space, w_item) for w_item in finalbody_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return TryFinally(body, finalbody, lineno, col_offset)
 
 State.ast_type('TryFinally', 'stmt')
 
@@ -876,13 +1076,19 @@
         space.setattr(w_node, space.wrap('test'), w_test)
         w_msg = self.msg.to_object(space)  # expr
         space.setattr(w_node, space.wrap('msg'), w_msg)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        test = space.getattr(w_node, space.wrap('test'))
-        msg = space.getattr(w_node, space.wrap('msg'))
-        return Assert(test, msg)
+        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
+        msg = expr.from_object(space, space.getattr(w_node, space.wrap('msg')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Assert(test, msg, lineno, col_offset)
 
 State.ast_type('Assert', 'stmt')
 
@@ -909,12 +1115,20 @@
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
         space.setattr(w_node, space.wrap('names'), w_names)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        names = space.getattr(w_node, space.wrap('names'))
-        return Import(names)
+        names_w = space.unpackiterable(
+                get_field(space, w_node, 'names', False))
+        names = [alias.from_object(space, w_item) for w_item in names_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Import(names, lineno, col_offset)
 
 State.ast_type('Import', 'stmt')
 
@@ -947,14 +1161,22 @@
         space.setattr(w_node, space.wrap('names'), w_names)
         w_level = space.wrap(self.level)  # int
         space.setattr(w_node, space.wrap('level'), w_level)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        module = space.getattr(w_node, space.wrap('module'))
-        names = space.getattr(w_node, space.wrap('names'))
-        level = space.getattr(w_node, space.wrap('level'))
-        return ImportFrom(module, names, level)
+        module = space.str_w(space.getattr(w_node, space.wrap('module')))
+        names_w = space.unpackiterable(
+                get_field(space, w_node, 'names', False))
+        names = [alias.from_object(space, w_item) for w_item in names_w]
+        level = space.int_w(space.getattr(w_node, space.wrap('level')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return ImportFrom(module, names, level, lineno, col_offset)
 
 State.ast_type('ImportFrom', 'stmt')
 
@@ -986,14 +1208,20 @@
         space.setattr(w_node, space.wrap('globals'), w_globals)
         w_locals = self.locals.to_object(space)  # expr
         space.setattr(w_node, space.wrap('locals'), w_locals)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        body = space.getattr(w_node, space.wrap('body'))
-        globals = space.getattr(w_node, space.wrap('globals'))
-        locals = space.getattr(w_node, space.wrap('locals'))
-        return Exec(body, globals, locals)
+        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
+        globals = expr.from_object(space, space.getattr(w_node, space.wrap('globals')))
+        locals = expr.from_object(space, space.getattr(w_node, space.wrap('locals')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Exec(body, globals, locals, lineno, col_offset)
 
 State.ast_type('Exec', 'stmt')
 
@@ -1018,12 +1246,20 @@
             names_w = [space.wrap(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
         space.setattr(w_node, space.wrap('names'), w_names)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        names = space.getattr(w_node, space.wrap('names'))
-        return Global(names)
+        names_w = space.unpackiterable(
+                get_field(space, w_node, 'names', False))
+        names = [space.str_w(w_item) for w_item in names_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Global(names, lineno, col_offset)
 
 State.ast_type('Global', 'stmt')
 
@@ -1045,12 +1281,18 @@
         w_node = space.call_function(get(space).w_Expr)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
-        return Expr(value)
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Expr(value, lineno, col_offset)
 
 State.ast_type('Expr', 'stmt')
 
@@ -1068,11 +1310,17 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Pass)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        return Pass()
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Pass(lineno, col_offset)
 
 State.ast_type('Pass', 'stmt')
 
@@ -1090,11 +1338,17 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Break)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        return Break()
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Break(lineno, col_offset)
 
 State.ast_type('Break', 'stmt')
 
@@ -1112,11 +1366,17 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Continue)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        return Continue()
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Continue(lineno, col_offset)
 
 State.ast_type('Continue', 'stmt')
 
@@ -1126,6 +1386,57 @@
     def __init__(self, lineno, col_offset):
         self.lineno = lineno
         self.col_offset = col_offset
+
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_BoolOp):
+            return BoolOp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_BinOp):
+            return BinOp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_UnaryOp):
+            return UnaryOp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Lambda):
+            return Lambda.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_IfExp):
+            return IfExp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Dict):
+            return Dict.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Set):
+            return Set.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_ListComp):
+            return ListComp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_SetComp):
+            return SetComp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_DictComp):
+            return DictComp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_GeneratorExp):
+            return GeneratorExp.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Yield):
+            return Yield.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Compare):
+            return Compare.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Call):
+            return Call.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Repr):
+            return Repr.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Num):
+            return Num.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Str):
+            return Str.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Attribute):
+            return Attribute.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Subscript):
+            return Subscript.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Name):
+            return Name.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_List):
+            return List.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Tuple):
+            return Tuple.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Const):
+            return Const.from_object(space, w_node)
+        raise operationerrfmt(space.w_TypeError,
+                "Expected expr node, got %T", w_node)
 State.ast_type('expr', 'AST')
 
 class BoolOp(expr):
@@ -1153,13 +1464,21 @@
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
         space.setattr(w_node, space.wrap('values'), w_values)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        op = space.getattr(w_node, space.wrap('op'))
-        values = space.getattr(w_node, space.wrap('values'))
-        return BoolOp(op, values)
+        op = boolop.from_object(space, space.getattr(w_node, space.wrap('op')))
+        values_w = space.unpackiterable(
+                get_field(space, w_node, 'values', False))
+        values = [expr.from_object(space, w_item) for w_item in values_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return BoolOp(op, values, lineno, col_offset)
 
 State.ast_type('BoolOp', 'expr')
 
@@ -1188,14 +1507,20 @@
         space.setattr(w_node, space.wrap('op'), w_op)
         w_right = self.right.to_object(space)  # expr
         space.setattr(w_node, space.wrap('right'), w_right)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        left = space.getattr(w_node, space.wrap('left'))
-        op = space.getattr(w_node, space.wrap('op'))
-        right = space.getattr(w_node, space.wrap('right'))
-        return BinOp(left, op, right)
+        left = expr.from_object(space, space.getattr(w_node, space.wrap('left')))
+        op = operator.from_object(space, space.getattr(w_node, space.wrap('op')))
+        right = expr.from_object(space, space.getattr(w_node, space.wrap('right')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return BinOp(left, op, right, lineno, col_offset)
 
 State.ast_type('BinOp', 'expr')
 
@@ -1220,13 +1545,19 @@
         space.setattr(w_node, space.wrap('op'), w_op)
         w_operand = self.operand.to_object(space)  # expr
         space.setattr(w_node, space.wrap('operand'), w_operand)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        op = space.getattr(w_node, space.wrap('op'))
-        operand = space.getattr(w_node, space.wrap('operand'))
-        return UnaryOp(op, operand)
+        op = unaryop.from_object(space, space.getattr(w_node, space.wrap('op')))
+        operand = expr.from_object(space, space.getattr(w_node, space.wrap('operand')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return UnaryOp(op, operand, lineno, col_offset)
 
 State.ast_type('UnaryOp', 'expr')
 
@@ -1252,13 +1583,19 @@
         space.setattr(w_node, space.wrap('args'), w_args)
         w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        args = space.getattr(w_node, space.wrap('args'))
-        body = space.getattr(w_node, space.wrap('body'))
-        return Lambda(args, body)
+        args = arguments.from_object(space, space.getattr(w_node, space.wrap('args')))
+        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Lambda(args, body, lineno, col_offset)
 
 State.ast_type('Lambda', 'expr')
 
@@ -1288,14 +1625,20 @@
         space.setattr(w_node, space.wrap('body'), w_body)
         w_orelse = self.orelse.to_object(space)  # expr
         space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        test = space.getattr(w_node, space.wrap('test'))
-        body = space.getattr(w_node, space.wrap('body'))
-        orelse = space.getattr(w_node, space.wrap('orelse'))
-        return IfExp(test, body, orelse)
+        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
+        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
+        orelse = expr.from_object(space, space.getattr(w_node, space.wrap('orelse')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return IfExp(test, body, orelse, lineno, col_offset)
 
 State.ast_type('IfExp', 'expr')
 
@@ -1331,13 +1674,23 @@
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
         space.setattr(w_node, space.wrap('values'), w_values)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        keys = space.getattr(w_node, space.wrap('keys'))
-        values = space.getattr(w_node, space.wrap('values'))
-        return Dict(keys, values)
+        keys_w = space.unpackiterable(
+                get_field(space, w_node, 'keys', False))
+        keys = [expr.from_object(space, w_item) for w_item in keys_w]
+        values_w = space.unpackiterable(
+                get_field(space, w_node, 'values', False))
+        values = [expr.from_object(space, w_item) for w_item in values_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Dict(keys, values, lineno, col_offset)
 
 State.ast_type('Dict', 'expr')
 
@@ -1364,12 +1717,20 @@
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
         space.setattr(w_node, space.wrap('elts'), w_elts)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        elts = space.getattr(w_node, space.wrap('elts'))
-        return Set(elts)
+        elts_w = space.unpackiterable(
+                get_field(space, w_node, 'elts', False))
+        elts = [expr.from_object(space, w_item) for w_item in elts_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Set(elts, lineno, col_offset)
 
 State.ast_type('Set', 'expr')
 
@@ -1400,13 +1761,21 @@
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        elt = space.getattr(w_node, space.wrap('elt'))
-        generators = space.getattr(w_node, space.wrap('generators'))
-        return ListComp(elt, generators)
+        elt = expr.from_object(space, space.getattr(w_node, space.wrap('elt')))
+        generators_w = space.unpackiterable(
+                get_field(space, w_node, 'generators', False))
+        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return ListComp(elt, generators, lineno, col_offset)
 
 State.ast_type('ListComp', 'expr')
 
@@ -1437,13 +1806,21 @@
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        elt = space.getattr(w_node, space.wrap('elt'))
-        generators = space.getattr(w_node, space.wrap('generators'))
-        return SetComp(elt, generators)
+        elt = expr.from_object(space, space.getattr(w_node, space.wrap('elt')))
+        generators_w = space.unpackiterable(
+                get_field(space, w_node, 'generators', False))
+        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return SetComp(elt, generators, lineno, col_offset)
 
 State.ast_type('SetComp', 'expr')
 
@@ -1478,14 +1855,22 @@
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        key = space.getattr(w_node, space.wrap('key'))
-        value = space.getattr(w_node, space.wrap('value'))
-        generators = space.getattr(w_node, space.wrap('generators'))
-        return DictComp(key, value, generators)
+        key = expr.from_object(space, space.getattr(w_node, space.wrap('key')))
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        generators_w = space.unpackiterable(
+                get_field(space, w_node, 'generators', False))
+        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return DictComp(key, value, generators, lineno, col_offset)
 
 State.ast_type('DictComp', 'expr')
 
@@ -1516,13 +1901,21 @@
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
         space.setattr(w_node, space.wrap('generators'), w_generators)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        elt = space.getattr(w_node, space.wrap('elt'))
-        generators = space.getattr(w_node, space.wrap('generators'))
-        return GeneratorExp(elt, generators)
+        elt = expr.from_object(space, space.getattr(w_node, space.wrap('elt')))
+        generators_w = space.unpackiterable(
+                get_field(space, w_node, 'generators', False))
+        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return GeneratorExp(elt, generators, lineno, col_offset)
 
 State.ast_type('GeneratorExp', 'expr')
 
@@ -1545,12 +1938,18 @@
         w_node = space.call_function(get(space).w_Yield)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
-        return Yield(value)
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Yield(value, lineno, col_offset)
 
 State.ast_type('Yield', 'expr')
 
@@ -1588,14 +1987,24 @@
             comparators_w = [node.to_object(space) for node in self.comparators] # expr
         w_comparators = space.newlist(comparators_w)
         space.setattr(w_node, space.wrap('comparators'), w_comparators)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        left = space.getattr(w_node, space.wrap('left'))
-        ops = space.getattr(w_node, space.wrap('ops'))
-        comparators = space.getattr(w_node, space.wrap('comparators'))
-        return Compare(left, ops, comparators)
+        left = expr.from_object(space, space.getattr(w_node, space.wrap('left')))
+        ops_w = space.unpackiterable(
+                get_field(space, w_node, 'ops', False))
+        ops = [cmpop.from_object(space, w_item) for w_item in ops_w]
+        comparators_w = space.unpackiterable(
+                get_field(space, w_node, 'comparators', False))
+        comparators = [expr.from_object(space, w_item) for w_item in comparators_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Compare(left, ops, comparators, lineno, col_offset)
 
 State.ast_type('Compare', 'expr')
 
@@ -1645,16 +2054,26 @@
         space.setattr(w_node, space.wrap('starargs'), w_starargs)
         w_kwargs = self.kwargs.to_object(space)  # expr
         space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        func = space.getattr(w_node, space.wrap('func'))
-        args = space.getattr(w_node, space.wrap('args'))
-        keywords = space.getattr(w_node, space.wrap('keywords'))
-        starargs = space.getattr(w_node, space.wrap('starargs'))
-        kwargs = space.getattr(w_node, space.wrap('kwargs'))
-        return Call(func, args, keywords, starargs, kwargs)
+        func = expr.from_object(space, space.getattr(w_node, space.wrap('func')))
+        args_w = space.unpackiterable(
+                get_field(space, w_node, 'args', False))
+        args = [expr.from_object(space, w_item) for w_item in args_w]
+        keywords_w = space.unpackiterable(
+                get_field(space, w_node, 'keywords', False))
+        keywords = [keyword.from_object(space, w_item) for w_item in keywords_w]
+        starargs = expr.from_object(space, space.getattr(w_node, space.wrap('starargs')))
+        kwargs = expr.from_object(space, space.getattr(w_node, space.wrap('kwargs')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Call(func, args, keywords, starargs, kwargs, lineno, col_offset)
 
 State.ast_type('Call', 'expr')
 
@@ -1676,12 +2095,18 @@
         w_node = space.call_function(get(space).w_Repr)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
-        return Repr(value)
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Repr(value, lineno, col_offset)
 
 State.ast_type('Repr', 'expr')
 
@@ -1702,12 +2127,18 @@
         w_node = space.call_function(get(space).w_Num)
         w_n = self.n  # object
         space.setattr(w_node, space.wrap('n'), w_n)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         n = space.getattr(w_node, space.wrap('n'))
-        return Num(n)
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Num(n, lineno, col_offset)
 
 State.ast_type('Num', 'expr')
 
@@ -1728,12 +2159,18 @@
         w_node = space.call_function(get(space).w_Str)
         w_s = self.s  # string
         space.setattr(w_node, space.wrap('s'), w_s)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        s = space.getattr(w_node, space.wrap('s'))
-        return Str(s)
+        s = check_string(space, space.getattr(w_node, space.wrap('s')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Str(s, lineno, col_offset)
 
 State.ast_type('Str', 'expr')
 
@@ -1761,14 +2198,20 @@
         space.setattr(w_node, space.wrap('attr'), w_attr)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
-        attr = space.getattr(w_node, space.wrap('attr'))
-        ctx = space.getattr(w_node, space.wrap('ctx'))
-        return Attribute(value, attr, ctx)
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        attr = space.str_w(space.getattr(w_node, space.wrap('attr')))
+        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Attribute(value, attr, ctx, lineno, col_offset)
 
 State.ast_type('Attribute', 'expr')
 
@@ -1797,14 +2240,20 @@
         space.setattr(w_node, space.wrap('slice'), w_slice)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
-        slice = space.getattr(w_node, space.wrap('slice'))
-        ctx = space.getattr(w_node, space.wrap('ctx'))
-        return Subscript(value, slice, ctx)
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        slice = slice.from_object(space, space.getattr(w_node, space.wrap('slice')))
+        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Subscript(value, slice, ctx, lineno, col_offset)
 
 State.ast_type('Subscript', 'expr')
 
@@ -1828,13 +2277,19 @@
         space.setattr(w_node, space.wrap('id'), w_id)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        id = space.getattr(w_node, space.wrap('id'))
-        ctx = space.getattr(w_node, space.wrap('ctx'))
-        return Name(id, ctx)
+        id = space.str_w(space.getattr(w_node, space.wrap('id')))
+        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Name(id, ctx, lineno, col_offset)
 
 State.ast_type('Name', 'expr')
 
@@ -1864,13 +2319,21 @@
         space.setattr(w_node, space.wrap('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        elts = space.getattr(w_node, space.wrap('elts'))
-        ctx = space.getattr(w_node, space.wrap('ctx'))
-        return List(elts, ctx)
+        elts_w = space.unpackiterable(
+                get_field(space, w_node, 'elts', False))
+        elts = [expr.from_object(space, w_item) for w_item in elts_w]
+        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return List(elts, ctx, lineno, col_offset)
 
 State.ast_type('List', 'expr')
 
@@ -1900,13 +2363,21 @@
         space.setattr(w_node, space.wrap('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
         space.setattr(w_node, space.wrap('ctx'), w_ctx)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        elts = space.getattr(w_node, space.wrap('elts'))
-        ctx = space.getattr(w_node, space.wrap('ctx'))
-        return Tuple(elts, ctx)
+        elts_w = space.unpackiterable(
+                get_field(space, w_node, 'elts', False))
+        elts = [expr.from_object(space, w_item) for w_item in elts_w]
+        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Tuple(elts, ctx, lineno, col_offset)
 
 State.ast_type('Tuple', 'expr')
 
@@ -1927,18 +2398,39 @@
         w_node = space.call_function(get(space).w_Const)
         w_value = self.value  # object
         space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         value = space.getattr(w_node, space.wrap('value'))
-        return Const(value)
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return Const(value, lineno, col_offset)
 
 State.ast_type('Const', 'expr')
 
 
 class expr_context(AST):
-    pass
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_Load):
+            return 1
+        if space.isinstance_w(w_node, get(space).w_Store):
+            return 2
+        if space.isinstance_w(w_node, get(space).w_Del):
+            return 3
+        if space.isinstance_w(w_node, get(space).w_AugLoad):
+            return 4
+        if space.isinstance_w(w_node, get(space).w_AugStore):
+            return 5
+        if space.isinstance_w(w_node, get(space).w_Param):
+            return 6
+        raise operationerrfmt(space.w_TypeError,
+                "Expected expr_context node, got %T", w_node)
 State.ast_type('expr_context', 'AST')
 
 class _Load(expr_context):
@@ -1988,7 +2480,18 @@
 ]
 
 class slice(AST):
-    pass
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_Ellipsis):
+            return Ellipsis.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Slice):
+            return Slice.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_ExtSlice):
+            return ExtSlice.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_Index):
+            return Index.from_object(space, w_node)
+        raise operationerrfmt(space.w_TypeError,
+                "Expected slice node, got %T", w_node)
 State.ast_type('slice', 'AST')
 
 class Ellipsis(slice):
@@ -2042,9 +2545,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lower = space.getattr(w_node, space.wrap('lower'))
-        upper = space.getattr(w_node, space.wrap('upper'))
-        step = space.getattr(w_node, space.wrap('step'))
+        lower = expr.from_object(space, space.getattr(w_node, space.wrap('lower')))
+        upper = expr.from_object(space, space.getattr(w_node, space.wrap('upper')))
+        step = expr.from_object(space, space.getattr(w_node, space.wrap('step')))
         return Slice(lower, upper, step)
 
 State.ast_type('Slice', 'slice')
@@ -2075,7 +2578,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        dims = space.getattr(w_node, space.wrap('dims'))
+        dims_w = space.unpackiterable(
+                get_field(space, w_node, 'dims', False))
+        dims = [slice.from_object(space, w_item) for w_item in dims_w]
         return ExtSlice(dims)
 
 State.ast_type('ExtSlice', 'slice')
@@ -2101,14 +2606,21 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
         return Index(value)
 
 State.ast_type('Index', 'slice')
 
 
 class boolop(AST):
-    pass
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_And):
+            return 1
+        if space.isinstance_w(w_node, get(space).w_Or):
+            return 2
+        raise operationerrfmt(space.w_TypeError,
+                "Expected boolop node, got %T", w_node)
 State.ast_type('boolop', 'AST')
 
 class _And(boolop):
@@ -2130,7 +2642,34 @@
 ]
 
 class operator(AST):
-    pass
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_Add):
+            return 1
+        if space.isinstance_w(w_node, get(space).w_Sub):
+            return 2
+        if space.isinstance_w(w_node, get(space).w_Mult):
+            return 3
+        if space.isinstance_w(w_node, get(space).w_Div):
+            return 4
+        if space.isinstance_w(w_node, get(space).w_Mod):
+            return 5
+        if space.isinstance_w(w_node, get(space).w_Pow):
+            return 6
+        if space.isinstance_w(w_node, get(space).w_LShift):
+            return 7
+        if space.isinstance_w(w_node, get(space).w_RShift):
+            return 8
+        if space.isinstance_w(w_node, get(space).w_BitOr):
+            return 9
+        if space.isinstance_w(w_node, get(space).w_BitXor):
+            return 10
+        if space.isinstance_w(w_node, get(space).w_BitAnd):
+            return 11
+        if space.isinstance_w(w_node, get(space).w_FloorDiv):
+            return 12
+        raise operationerrfmt(space.w_TypeError,
+                "Expected operator node, got %T", w_node)
 State.ast_type('operator', 'AST')
 
 class _Add(operator):
@@ -2222,7 +2761,18 @@
 ]
 
 class unaryop(AST):
-    pass
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_Invert):
+            return 1
+        if space.isinstance_w(w_node, get(space).w_Not):
+            return 2
+        if space.isinstance_w(w_node, get(space).w_UAdd):
+            return 3
+        if space.isinstance_w(w_node, get(space).w_USub):
+            return 4
+        raise operationerrfmt(space.w_TypeError,
+                "Expected unaryop node, got %T", w_node)
 State.ast_type('unaryop', 'AST')
 
 class _Invert(unaryop):
@@ -2258,7 +2808,30 @@
 ]
 
 class cmpop(AST):
-    pass
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_Eq):
+            return 1
+        if space.isinstance_w(w_node, get(space).w_NotEq):
+            return 2
+        if space.isinstance_w(w_node, get(space).w_Lt):
+            return 3
+        if space.isinstance_w(w_node, get(space).w_LtE):
+            return 4
+        if space.isinstance_w(w_node, get(space).w_Gt):
+            return 5
+        if space.isinstance_w(w_node, get(space).w_GtE):
+            return 6
+        if space.isinstance_w(w_node, get(space).w_Is):
+            return 7
+        if space.isinstance_w(w_node, get(space).w_IsNot):
+            return 8
+        if space.isinstance_w(w_node, get(space).w_In):
+            return 9
+        if space.isinstance_w(w_node, get(space).w_NotIn):
+            return 10
+        raise operationerrfmt(space.w_TypeError,
+                "Expected cmpop node, got %T", w_node)
 State.ast_type('cmpop', 'AST')
 
 class _Eq(cmpop):
@@ -2368,9 +2941,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        target = space.getattr(w_node, space.wrap('target'))
-        iter = space.getattr(w_node, space.wrap('iter'))
-        ifs = space.getattr(w_node, space.wrap('ifs'))
+        target = expr.from_object(space, space.getattr(w_node, space.wrap('target')))
+        iter = expr.from_object(space, space.getattr(w_node, space.wrap('iter')))
+        ifs_w = space.unpackiterable(
+                get_field(space, w_node, 'ifs', False))
+        ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(target, iter, ifs)
 
 State.ast_type('comprehension', 'AST')
@@ -2380,6 +2955,13 @@
     def __init__(self, lineno, col_offset):
         self.lineno = lineno
         self.col_offset = col_offset
+
+    @staticmethod
+    def from_object(space, w_node):
+        if space.isinstance_w(w_node, get(space).w_ExceptHandler):
+            return ExceptHandler.from_object(space, w_node)
+        raise operationerrfmt(space.w_TypeError,
+                "Expected excepthandler node, got %T", w_node)
 State.ast_type('excepthandler', 'AST')
 
 class ExceptHandler(excepthandler):
@@ -2414,14 +2996,22 @@
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
         space.setattr(w_node, space.wrap('body'), w_body)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
-        type = space.getattr(w_node, space.wrap('type'))
-        name = space.getattr(w_node, space.wrap('name'))
-        body = space.getattr(w_node, space.wrap('body'))
-        return ExceptHandler(type, name, body)
+        type = expr.from_object(space, space.getattr(w_node, space.wrap('type')))
+        name = expr.from_object(space, space.getattr(w_node, space.wrap('name')))
+        body_w = space.unpackiterable(
+                get_field(space, w_node, 'body', False))
+        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
+        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        return ExceptHandler(type, name, body, lineno, col_offset)
 
 State.ast_type('ExceptHandler', 'excepthandler')
 
@@ -2466,10 +3056,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        args = space.getattr(w_node, space.wrap('args'))
-        vararg = space.getattr(w_node, space.wrap('vararg'))
-        kwarg = space.getattr(w_node, space.wrap('kwarg'))
-        defaults = space.getattr(w_node, space.wrap('defaults'))
+        args_w = space.unpackiterable(
+                get_field(space, w_node, 'args', False))
+        args = [expr.from_object(space, w_item) for w_item in args_w]
+        vararg = space.str_w(space.getattr(w_node, space.wrap('vararg')))
+        kwarg = space.str_w(space.getattr(w_node, space.wrap('kwarg')))
+        defaults_w = space.unpackiterable(
+                get_field(space, w_node, 'defaults', False))
+        defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(args, vararg, kwarg, defaults)
 
 State.ast_type('arguments', 'AST')
@@ -2497,8 +3091,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        arg = space.getattr(w_node, space.wrap('arg'))
-        value = space.getattr(w_node, space.wrap('value'))
+        arg = space.str_w(space.getattr(w_node, space.wrap('arg')))
+        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
         return keyword(arg, value)
 
 State.ast_type('keyword', 'AST')
@@ -2525,8 +3119,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.getattr(w_node, space.wrap('name'))
-        asname = space.getattr(w_node, space.wrap('asname'))
+        name = space.str_w(space.getattr(w_node, space.wrap('name')))
+        asname = space.str_w(space.getattr(w_node, space.wrap('asname')))
         return alias(name, asname)
 
 State.ast_type('alias', 'AST')
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -32,6 +32,6 @@
         value = space.wrap(42)
         w_node = space.call_function(ast.get(space).w_Num)
         space.setattr(w_node, space.wrap('n'), value)
-        node = ast.Num.from_object(space, w_node)
+        node = ast.Num.from_object(space, w_node, lineno=1, col_offset=1)
         assert node.n is value
         
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -57,7 +57,14 @@
     def visitSum(self, sum, base):
         if is_simple_sum(sum):
             self.emit("class %s(AST):" % (base,))
-            self.emit("pass", 1)
+            self.emit("@staticmethod", 1)
+            self.emit("def from_object(space, w_node):", 1)
+            for i, cons in enumerate(sum.types):
+                self.emit("if space.isinstance_w(w_node, get(space).w_%s):"
+                          % (cons.name,), 2)
+                self.emit("return %i" % (i+1,), 3)
+            self.emit("raise operationerrfmt(space.w_TypeError,", 2)
+            self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
             self.emit("State.ast_type('%s', 'AST')" % (base,))
             self.emit("")
             for i, cons in enumerate(sum.types):
@@ -82,8 +89,16 @@
                 self.emit("def __init__(self, %s):" % (args,), 1)
                 for attr in sum.attributes:
                     self.visit(attr)
-            else:
-                self.emit("pass", 1)
+                self.emit("")
+            self.emit("@staticmethod", 1)
+            self.emit("def from_object(space, w_node):", 1)
+            for typ in sum.types:
+                self.emit("if space.isinstance_w(w_node, get(space).w_%s):"
+                          % (typ.name,), 2)
+                self.emit("return %s.from_object(space, w_node)"
+                          % (typ.name,), 3)
+            self.emit("raise operationerrfmt(space.w_TypeError,", 2)
+            self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
             self.emit("State.ast_type('%r', 'AST')" % (base,))
             self.emit("")
             for cons in sum.types:
@@ -113,6 +128,20 @@
         else:
             return "%s.to_object(space)" % (value,)
 
+    def get_value_extractor(self, field, value):
+        if field.type.value in self.data.simple_types:
+            return "%s.from_object(space, %s)" % (field.type, value)
+        elif field.type.value in ("object",):
+            return value
+        elif field.type.value in ("string",):
+            return "check_string(space, %s)" % (value,)
+        elif field.type.value in ("identifier",):
+            return "space.str_w(%s)" % (value,)
+        elif field.type.value in ("int",):
+            return "space.int_w(%s)" % (value,)
+        else:
+            return "%s.from_object(space, %s)" % (field.type, value)
+
     def get_field_converter(self, field):
         if field.seq:
             lines = []
@@ -129,13 +158,26 @@
             return ["w_%s = %s  # %s" % (field.name, wrapper, field.type)]
 
     def get_field_extractor(self, field):
-        lines = ["%s = space.getattr(w_node, space.wrap('%s'))" % (field.name, field.name)]
+        if field.seq:
+            lines = []
+            lines.append("%s_w = space.unpackiterable(" % (field.name,))
+            lines.append("        get_field(space, w_node, '%s', %s))"
+                         % (field.name, field.opt))
+            value = self.get_value_extractor(field, "w_item")
+            lines.append("%s = [%s for w_item in %s_w]" %
+                         (field.name, value, field.name))
+        else:
+            value = self.get_value_extractor(
+                field, 
+                "space.getattr(w_node, space.wrap('%s'))" % (field.name,))
+            lines = ["%s = %s" % (field.name, value)]
         return lines
 
-    def make_converters(self, fields, name):
+    def make_converters(self, fields, name, extras=None):
         self.emit("def to_object(self, space):", 1)
         self.emit("w_node = space.call_function(get(space).w_%s)" % name, 2)
-        for field in fields:
+        all_fields = fields + extras if extras else fields
+        for field in all_fields:
             wrapping_code = self.get_field_converter(field)
             for line in wrapping_code:
                 self.emit(line, 2)
@@ -145,12 +187,12 @@
         self.emit("")
         self.emit("@staticmethod", 1)
         self.emit("def from_object(space, w_node):", 1)
-        for field in fields:
+        for field in all_fields:
             unwrapping_code = self.get_field_extractor(field)
             for line in unwrapping_code:
                 self.emit(line, 2)
         self.emit("return %s(%s)" % (
-                name, ', '.join(str(field.name) for field in fields)), 2)
+                name, ', '.join(str(field.name) for field in all_fields)), 2)
         self.emit("")
 
     def make_constructor(self, fields, node, extras=None, base=None):
@@ -193,7 +235,7 @@
         self.emit("visitor.visit_%s(self)" % (cons.name,), 2)
         self.emit("")
         self.make_mutate_over(cons, cons.name)
-        self.make_converters(cons.fields, cons.name)
+        self.make_converters(cons.fields, cons.name, extra_attributes)
         self.emit("State.ast_type('%r', '%s')" % (cons.name, base))
         self.emit("")
 
@@ -327,15 +369,30 @@
         self.optional_masks = optional_masks
 
 
-HEAD = """# Generated by tools/asdl_py.py
-from pypy.interpreter.error import OperationError
-from rpython.tool.pairtype import extendabletype
+HEAD = r"""# Generated by tools/asdl_py.py
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.error import OperationError, operationerrfmt
+from rpython.tool.pairtype import extendabletype
 from rpython.tool.sourcetools import func_with_new_name
 
 
+def check_string(space, w_obj):
+    if not (space.isinstance_w(w_obj, space.w_str) or
+            space.isinstance_w(w_obj, space.w_unicode)):
+        raise OperationError(space.w_TypeError, space.wrap(
+                'AST string must be of type str or unicode'))
+    return w_obj
+
+def get_field(space, w_node, name, optional):
+    w_obj = w_node.getdictvalue(space, name)
+    if w_obj is None and not optional:
+        raise operationerrfmt(space.w_TypeError,
+                "required field \"%s\" missing from %T", name, w_node)
+    return w_obj
+
+
 class AST(object):
     __metaclass__ = extendabletype
 
@@ -363,13 +420,6 @@
 class W_AST(W_Root):
     w_dict = None
 
-    def obj2mod(self, space):
-        if space.isinstance_w(self, get(space).w_Module):
-            return Module.from_object(space, self)
-        else:
-            raise OperationError(space.w_TypeError, space.wrap(
-                    "Expected mod node"))
-
     def getdict(self, space):
         if self.w_dict is None:
             self.w_dict = space.newdict(instance=True)
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -28,7 +28,7 @@
     w_ast_type = space.gettypeobject(ast.W_AST.typedef)
     str_ = None
     if space.isinstance_w(w_source, w_ast_type):
-        ast_node = space.interp_w(ast.W_AST, w_source).obj2mod(space)
+        ast_node = ast.mod.from_object(space, w_source)
     elif space.isinstance_w(w_source, space.w_unicode):
         w_utf_8_source = space.call_method(w_source, "encode",
                                            space.wrap("utf-8"))

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:11 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:11 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Progress: constructor,
	"_fields"
Message-ID: <20130626212211.441ED1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65019:0546fe3feeba
Date: 2013-06-24 23:07 +0200
http://bitbucket.org/pypy/pypy/changeset/0546fe3feeba/

Log:	Progress: constructor, "_fields"

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -77,18 +77,21 @@
             space.setattr(self, w_name,
                           space.getitem(w_state, w_name))
 
-def get_W_AST_new(node_class):
-    def generic_W_AST_new(space, w_type, __args__):
-        node = space.allocate_instance(node_class, w_type)
-        return space.wrap(node)
-    return func_with_new_name(generic_W_AST_new, "new_%s" % node_class.__name__)
-
+def W_AST_new(space, w_type, __args__):
+    node = space.allocate_instance(W_AST, w_type)
+    return space.wrap(node)
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
-    if args_w and len(args_w) != 0:
-        w_err = space.wrap("_ast.AST constructor takes 0 positional arguments")
-        raise OperationError(space.w_TypeError, w_err)
+    fields_w = space.unpackiterable(space.getattr(space.type(w_self),
+                                    space.wrap("_fields")))
+    num_fields = len(fields_w) if fields_w else 0
+    if args_w and len(args_w) != num_fields:
+        raise operationerrfmt(space.w_TypeError,
+                "_ast.%T constructor takes %s positional arguments", w_self, num_fields)
+    if args_w:
+        for i, w_field in enumerate(fields_w):
+            space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
         space.setattr(w_self, space.wrap(field), w_value)
 
@@ -101,7 +104,7 @@
     __setstate__=interp2app(W_AST.setstate_w),
     __dict__ = typedef.GetSetProperty(typedef.descr_get_dict,
                                       typedef.descr_set_dict, cls=W_AST),
-    __new__=interp2app(get_W_AST_new(W_AST)),
+    __new__=interp2app(W_AST_new),
     __init__=interp2app(W_AST_init),
 )
 
@@ -109,19 +112,23 @@
     AST_TYPES = []
 
     @classmethod
-    def ast_type(cls, name, base):
-        cls.AST_TYPES.append((name, base))
+    def ast_type(cls, name, base, fields):
+        cls.AST_TYPES.append((name, base, fields))
 
     def __init__(self, space):
         self.w_AST = space.gettypeobject(W_AST.typedef)
-        for (name, base) in self.AST_TYPES:
-            self.make_new_type(space, name, base)
+        for (name, base, fields) in self.AST_TYPES:
+            self.make_new_type(space, name, base, fields)
         
-    def make_new_type(self, space, name, base):
+    def make_new_type(self, space, name, base, fields):
         w_base = getattr(self, 'w_%s' % base)
+        w_dict = space.newdict()
+        if fields is not None:
+            space.setitem_str(w_dict, "_fields",
+                              space.newtuple([space.wrap(f) for f in fields]))
         w_type = space.call_function(
             space.w_type, 
-            space.wrap(name), space.newtuple([w_base]), space.newdict())
+            space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
@@ -140,7 +147,7 @@
             return Suite.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected mod node, got %T", w_node)
-State.ast_type('mod', 'AST')
+State.ast_type('mod', 'AST', None)
 
 class Module(mod):
 
@@ -172,7 +179,7 @@
         body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Module(body)
 
-State.ast_type('Module', 'mod')
+State.ast_type('Module', 'mod', ['body'])
 
 
 class Interactive(mod):
@@ -205,7 +212,7 @@
         body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Interactive(body)
 
-State.ast_type('Interactive', 'mod')
+State.ast_type('Interactive', 'mod', ['body'])
 
 
 class Expression(mod):
@@ -228,10 +235,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
+        body = expr.from_object(space, get_field(space, w_node, 'body', False))
         return Expression(body)
 
-State.ast_type('Expression', 'mod')
+State.ast_type('Expression', 'mod', ['body'])
 
 
 class Suite(mod):
@@ -264,7 +271,7 @@
         body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Suite(body)
 
-State.ast_type('Suite', 'mod')
+State.ast_type('Suite', 'mod', ['body'])
 
 
 class stmt(AST):
@@ -323,7 +330,7 @@
             return Continue.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected stmt node, got %T", w_node)
-State.ast_type('stmt', 'AST')
+State.ast_type('stmt', 'AST', None)
 
 class FunctionDef(stmt):
 
@@ -371,19 +378,19 @@
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.str_w(space.getattr(w_node, space.wrap('name')))
-        args = arguments.from_object(space, space.getattr(w_node, space.wrap('args')))
+        name = space.str_w(get_field(space, w_node, 'name', False))
+        args = arguments.from_object(space, get_field(space, w_node, 'args', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
         body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(
                 get_field(space, w_node, 'decorator_list', False))
         decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return FunctionDef(name, args, body, decorator_list, lineno, col_offset)
 
-State.ast_type('FunctionDef', 'stmt')
+State.ast_type('FunctionDef', 'stmt', ['name', 'args', 'body', 'decorator_list'])
 
 
 class ClassDef(stmt):
@@ -437,7 +444,7 @@
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.str_w(space.getattr(w_node, space.wrap('name')))
+        name = space.str_w(get_field(space, w_node, 'name', False))
         bases_w = space.unpackiterable(
                 get_field(space, w_node, 'bases', False))
         bases = [expr.from_object(space, w_item) for w_item in bases_w]
@@ -447,11 +454,11 @@
         decorator_list_w = space.unpackiterable(
                 get_field(space, w_node, 'decorator_list', False))
         decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return ClassDef(name, bases, body, decorator_list, lineno, col_offset)
 
-State.ast_type('ClassDef', 'stmt')
+State.ast_type('ClassDef', 'stmt', ['name', 'bases', 'body', 'decorator_list'])
 
 
 class Return(stmt):
@@ -480,12 +487,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', True))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Return(value, lineno, col_offset)
 
-State.ast_type('Return', 'stmt')
+State.ast_type('Return', 'stmt', ['value'])
 
 
 class Delete(stmt):
@@ -521,11 +528,11 @@
         targets_w = space.unpackiterable(
                 get_field(space, w_node, 'targets', False))
         targets = [expr.from_object(space, w_item) for w_item in targets_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Delete(targets, lineno, col_offset)
 
-State.ast_type('Delete', 'stmt')
+State.ast_type('Delete', 'stmt', ['targets'])
 
 
 class Assign(stmt):
@@ -565,12 +572,12 @@
         targets_w = space.unpackiterable(
                 get_field(space, w_node, 'targets', False))
         targets = [expr.from_object(space, w_item) for w_item in targets_w]
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Assign(targets, value, lineno, col_offset)
 
-State.ast_type('Assign', 'stmt')
+State.ast_type('Assign', 'stmt', ['targets', 'value'])
 
 
 class AugAssign(stmt):
@@ -605,14 +612,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        target = expr.from_object(space, space.getattr(w_node, space.wrap('target')))
-        op = operator.from_object(space, space.getattr(w_node, space.wrap('op')))
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        target = expr.from_object(space, get_field(space, w_node, 'target', False))
+        op = operator.from_object(space, get_field(space, w_node, 'op', False))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return AugAssign(target, op, value, lineno, col_offset)
 
-State.ast_type('AugAssign', 'stmt')
+State.ast_type('AugAssign', 'stmt', ['target', 'op', 'value'])
 
 
 class Print(stmt):
@@ -653,16 +660,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        dest = expr.from_object(space, space.getattr(w_node, space.wrap('dest')))
+        dest = expr.from_object(space, get_field(space, w_node, 'dest', True))
         values_w = space.unpackiterable(
                 get_field(space, w_node, 'values', False))
         values = [expr.from_object(space, w_item) for w_item in values_w]
-        nl = bool.from_object(space, space.getattr(w_node, space.wrap('nl')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        nl = bool.from_object(space, get_field(space, w_node, 'nl', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Print(dest, values, nl, lineno, col_offset)
 
-State.ast_type('Print', 'stmt')
+State.ast_type('Print', 'stmt', ['dest', 'values', 'nl'])
 
 
 class For(stmt):
@@ -712,19 +719,19 @@
 
     @staticmethod
     def from_object(space, w_node):
-        target = expr.from_object(space, space.getattr(w_node, space.wrap('target')))
-        iter = expr.from_object(space, space.getattr(w_node, space.wrap('iter')))
+        target = expr.from_object(space, get_field(space, w_node, 'target', False))
+        iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
         body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
         orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return For(target, iter, body, orelse, lineno, col_offset)
 
-State.ast_type('For', 'stmt')
+State.ast_type('For', 'stmt', ['target', 'iter', 'body', 'orelse'])
 
 
 class While(stmt):
@@ -770,18 +777,18 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
+        test = expr.from_object(space, get_field(space, w_node, 'test', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
         body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
         orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return While(test, body, orelse, lineno, col_offset)
 
-State.ast_type('While', 'stmt')
+State.ast_type('While', 'stmt', ['test', 'body', 'orelse'])
 
 
 class If(stmt):
@@ -827,18 +834,18 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
+        test = expr.from_object(space, get_field(space, w_node, 'test', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
         body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
         orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return If(test, body, orelse, lineno, col_offset)
 
-State.ast_type('If', 'stmt')
+State.ast_type('If', 'stmt', ['test', 'body', 'orelse'])
 
 
 class With(stmt):
@@ -880,16 +887,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        context_expr = expr.from_object(space, space.getattr(w_node, space.wrap('context_expr')))
-        optional_vars = expr.from_object(space, space.getattr(w_node, space.wrap('optional_vars')))
+        context_expr = expr.from_object(space, get_field(space, w_node, 'context_expr', False))
+        optional_vars = expr.from_object(space, get_field(space, w_node, 'optional_vars', True))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
         body = [stmt.from_object(space, w_item) for w_item in body_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return With(context_expr, optional_vars, body, lineno, col_offset)
 
-State.ast_type('With', 'stmt')
+State.ast_type('With', 'stmt', ['context_expr', 'optional_vars', 'body'])
 
 
 class Raise(stmt):
@@ -928,14 +935,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        type = expr.from_object(space, space.getattr(w_node, space.wrap('type')))
-        inst = expr.from_object(space, space.getattr(w_node, space.wrap('inst')))
-        tback = expr.from_object(space, space.getattr(w_node, space.wrap('tback')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        type = expr.from_object(space, get_field(space, w_node, 'type', True))
+        inst = expr.from_object(space, get_field(space, w_node, 'inst', True))
+        tback = expr.from_object(space, get_field(space, w_node, 'tback', True))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Raise(type, inst, tback, lineno, col_offset)
 
-State.ast_type('Raise', 'stmt')
+State.ast_type('Raise', 'stmt', ['type', 'inst', 'tback'])
 
 
 class TryExcept(stmt):
@@ -995,11 +1002,11 @@
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
         orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return TryExcept(body, handlers, orelse, lineno, col_offset)
 
-State.ast_type('TryExcept', 'stmt')
+State.ast_type('TryExcept', 'stmt', ['body', 'handlers', 'orelse'])
 
 
 class TryFinally(stmt):
@@ -1047,11 +1054,11 @@
         finalbody_w = space.unpackiterable(
                 get_field(space, w_node, 'finalbody', False))
         finalbody = [stmt.from_object(space, w_item) for w_item in finalbody_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return TryFinally(body, finalbody, lineno, col_offset)
 
-State.ast_type('TryFinally', 'stmt')
+State.ast_type('TryFinally', 'stmt', ['body', 'finalbody'])
 
 
 class Assert(stmt):
@@ -1084,13 +1091,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
-        msg = expr.from_object(space, space.getattr(w_node, space.wrap('msg')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        test = expr.from_object(space, get_field(space, w_node, 'test', False))
+        msg = expr.from_object(space, get_field(space, w_node, 'msg', True))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Assert(test, msg, lineno, col_offset)
 
-State.ast_type('Assert', 'stmt')
+State.ast_type('Assert', 'stmt', ['test', 'msg'])
 
 
 class Import(stmt):
@@ -1126,11 +1133,11 @@
         names_w = space.unpackiterable(
                 get_field(space, w_node, 'names', False))
         names = [alias.from_object(space, w_item) for w_item in names_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Import(names, lineno, col_offset)
 
-State.ast_type('Import', 'stmt')
+State.ast_type('Import', 'stmt', ['names'])
 
 
 class ImportFrom(stmt):
@@ -1169,16 +1176,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        module = space.str_w(space.getattr(w_node, space.wrap('module')))
+        module = space.str_w(get_field(space, w_node, 'module', True))
         names_w = space.unpackiterable(
                 get_field(space, w_node, 'names', False))
         names = [alias.from_object(space, w_item) for w_item in names_w]
-        level = space.int_w(space.getattr(w_node, space.wrap('level')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        level = space.int_w(get_field(space, w_node, 'level', True))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return ImportFrom(module, names, level, lineno, col_offset)
 
-State.ast_type('ImportFrom', 'stmt')
+State.ast_type('ImportFrom', 'stmt', ['module', 'names', 'level'])
 
 
 class Exec(stmt):
@@ -1216,14 +1223,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
-        globals = expr.from_object(space, space.getattr(w_node, space.wrap('globals')))
-        locals = expr.from_object(space, space.getattr(w_node, space.wrap('locals')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        globals = expr.from_object(space, get_field(space, w_node, 'globals', True))
+        locals = expr.from_object(space, get_field(space, w_node, 'locals', True))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Exec(body, globals, locals, lineno, col_offset)
 
-State.ast_type('Exec', 'stmt')
+State.ast_type('Exec', 'stmt', ['body', 'globals', 'locals'])
 
 
 class Global(stmt):
@@ -1257,11 +1264,11 @@
         names_w = space.unpackiterable(
                 get_field(space, w_node, 'names', False))
         names = [space.str_w(w_item) for w_item in names_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Global(names, lineno, col_offset)
 
-State.ast_type('Global', 'stmt')
+State.ast_type('Global', 'stmt', ['names'])
 
 
 class Expr(stmt):
@@ -1289,12 +1296,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Expr(value, lineno, col_offset)
 
-State.ast_type('Expr', 'stmt')
+State.ast_type('Expr', 'stmt', ['value'])
 
 
 class Pass(stmt):
@@ -1318,11 +1325,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Pass(lineno, col_offset)
 
-State.ast_type('Pass', 'stmt')
+State.ast_type('Pass', 'stmt', [])
 
 
 class Break(stmt):
@@ -1346,11 +1353,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Break(lineno, col_offset)
 
-State.ast_type('Break', 'stmt')
+State.ast_type('Break', 'stmt', [])
 
 
 class Continue(stmt):
@@ -1374,11 +1381,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Continue(lineno, col_offset)
 
-State.ast_type('Continue', 'stmt')
+State.ast_type('Continue', 'stmt', [])
 
 
 class expr(AST):
@@ -1437,7 +1444,7 @@
             return Const.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected expr node, got %T", w_node)
-State.ast_type('expr', 'AST')
+State.ast_type('expr', 'AST', None)
 
 class BoolOp(expr):
 
@@ -1472,15 +1479,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        op = boolop.from_object(space, space.getattr(w_node, space.wrap('op')))
+        op = boolop.from_object(space, get_field(space, w_node, 'op', False))
         values_w = space.unpackiterable(
                 get_field(space, w_node, 'values', False))
         values = [expr.from_object(space, w_item) for w_item in values_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return BoolOp(op, values, lineno, col_offset)
 
-State.ast_type('BoolOp', 'expr')
+State.ast_type('BoolOp', 'expr', ['op', 'values'])
 
 
 class BinOp(expr):
@@ -1515,14 +1522,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        left = expr.from_object(space, space.getattr(w_node, space.wrap('left')))
-        op = operator.from_object(space, space.getattr(w_node, space.wrap('op')))
-        right = expr.from_object(space, space.getattr(w_node, space.wrap('right')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        left = expr.from_object(space, get_field(space, w_node, 'left', False))
+        op = operator.from_object(space, get_field(space, w_node, 'op', False))
+        right = expr.from_object(space, get_field(space, w_node, 'right', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return BinOp(left, op, right, lineno, col_offset)
 
-State.ast_type('BinOp', 'expr')
+State.ast_type('BinOp', 'expr', ['left', 'op', 'right'])
 
 
 class UnaryOp(expr):
@@ -1553,13 +1560,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        op = unaryop.from_object(space, space.getattr(w_node, space.wrap('op')))
-        operand = expr.from_object(space, space.getattr(w_node, space.wrap('operand')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        op = unaryop.from_object(space, get_field(space, w_node, 'op', False))
+        operand = expr.from_object(space, get_field(space, w_node, 'operand', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return UnaryOp(op, operand, lineno, col_offset)
 
-State.ast_type('UnaryOp', 'expr')
+State.ast_type('UnaryOp', 'expr', ['op', 'operand'])
 
 
 class Lambda(expr):
@@ -1591,13 +1598,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        args = arguments.from_object(space, space.getattr(w_node, space.wrap('args')))
-        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        args = arguments.from_object(space, get_field(space, w_node, 'args', False))
+        body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Lambda(args, body, lineno, col_offset)
 
-State.ast_type('Lambda', 'expr')
+State.ast_type('Lambda', 'expr', ['args', 'body'])
 
 
 class IfExp(expr):
@@ -1633,14 +1640,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, space.getattr(w_node, space.wrap('test')))
-        body = expr.from_object(space, space.getattr(w_node, space.wrap('body')))
-        orelse = expr.from_object(space, space.getattr(w_node, space.wrap('orelse')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        test = expr.from_object(space, get_field(space, w_node, 'test', False))
+        body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        orelse = expr.from_object(space, get_field(space, w_node, 'orelse', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return IfExp(test, body, orelse, lineno, col_offset)
 
-State.ast_type('IfExp', 'expr')
+State.ast_type('IfExp', 'expr', ['test', 'body', 'orelse'])
 
 
 class Dict(expr):
@@ -1688,11 +1695,11 @@
         values_w = space.unpackiterable(
                 get_field(space, w_node, 'values', False))
         values = [expr.from_object(space, w_item) for w_item in values_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Dict(keys, values, lineno, col_offset)
 
-State.ast_type('Dict', 'expr')
+State.ast_type('Dict', 'expr', ['keys', 'values'])
 
 
 class Set(expr):
@@ -1728,11 +1735,11 @@
         elts_w = space.unpackiterable(
                 get_field(space, w_node, 'elts', False))
         elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Set(elts, lineno, col_offset)
 
-State.ast_type('Set', 'expr')
+State.ast_type('Set', 'expr', ['elts'])
 
 
 class ListComp(expr):
@@ -1769,15 +1776,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elt = expr.from_object(space, space.getattr(w_node, space.wrap('elt')))
+        elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
         generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return ListComp(elt, generators, lineno, col_offset)
 
-State.ast_type('ListComp', 'expr')
+State.ast_type('ListComp', 'expr', ['elt', 'generators'])
 
 
 class SetComp(expr):
@@ -1814,15 +1821,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elt = expr.from_object(space, space.getattr(w_node, space.wrap('elt')))
+        elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
         generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return SetComp(elt, generators, lineno, col_offset)
 
-State.ast_type('SetComp', 'expr')
+State.ast_type('SetComp', 'expr', ['elt', 'generators'])
 
 
 class DictComp(expr):
@@ -1863,16 +1870,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        key = expr.from_object(space, space.getattr(w_node, space.wrap('key')))
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        key = expr.from_object(space, get_field(space, w_node, 'key', False))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
         generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return DictComp(key, value, generators, lineno, col_offset)
 
-State.ast_type('DictComp', 'expr')
+State.ast_type('DictComp', 'expr', ['key', 'value', 'generators'])
 
 
 class GeneratorExp(expr):
@@ -1909,15 +1916,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elt = expr.from_object(space, space.getattr(w_node, space.wrap('elt')))
+        elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
         generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return GeneratorExp(elt, generators, lineno, col_offset)
 
-State.ast_type('GeneratorExp', 'expr')
+State.ast_type('GeneratorExp', 'expr', ['elt', 'generators'])
 
 
 class Yield(expr):
@@ -1946,12 +1953,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', True))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Yield(value, lineno, col_offset)
 
-State.ast_type('Yield', 'expr')
+State.ast_type('Yield', 'expr', ['value'])
 
 
 class Compare(expr):
@@ -1995,18 +2002,18 @@
 
     @staticmethod
     def from_object(space, w_node):
-        left = expr.from_object(space, space.getattr(w_node, space.wrap('left')))
+        left = expr.from_object(space, get_field(space, w_node, 'left', False))
         ops_w = space.unpackiterable(
                 get_field(space, w_node, 'ops', False))
         ops = [cmpop.from_object(space, w_item) for w_item in ops_w]
         comparators_w = space.unpackiterable(
                 get_field(space, w_node, 'comparators', False))
         comparators = [expr.from_object(space, w_item) for w_item in comparators_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Compare(left, ops, comparators, lineno, col_offset)
 
-State.ast_type('Compare', 'expr')
+State.ast_type('Compare', 'expr', ['left', 'ops', 'comparators'])
 
 
 class Call(expr):
@@ -2062,20 +2069,20 @@
 
     @staticmethod
     def from_object(space, w_node):
-        func = expr.from_object(space, space.getattr(w_node, space.wrap('func')))
+        func = expr.from_object(space, get_field(space, w_node, 'func', False))
         args_w = space.unpackiterable(
                 get_field(space, w_node, 'args', False))
         args = [expr.from_object(space, w_item) for w_item in args_w]
         keywords_w = space.unpackiterable(
                 get_field(space, w_node, 'keywords', False))
         keywords = [keyword.from_object(space, w_item) for w_item in keywords_w]
-        starargs = expr.from_object(space, space.getattr(w_node, space.wrap('starargs')))
-        kwargs = expr.from_object(space, space.getattr(w_node, space.wrap('kwargs')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        starargs = expr.from_object(space, get_field(space, w_node, 'starargs', True))
+        kwargs = expr.from_object(space, get_field(space, w_node, 'kwargs', True))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Call(func, args, keywords, starargs, kwargs, lineno, col_offset)
 
-State.ast_type('Call', 'expr')
+State.ast_type('Call', 'expr', ['func', 'args', 'keywords', 'starargs', 'kwargs'])
 
 
 class Repr(expr):
@@ -2103,12 +2110,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Repr(value, lineno, col_offset)
 
-State.ast_type('Repr', 'expr')
+State.ast_type('Repr', 'expr', ['value'])
 
 
 class Num(expr):
@@ -2135,12 +2142,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        n = space.getattr(w_node, space.wrap('n'))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        n = get_field(space, w_node, 'n', False)
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Num(n, lineno, col_offset)
 
-State.ast_type('Num', 'expr')
+State.ast_type('Num', 'expr', ['n'])
 
 
 class Str(expr):
@@ -2167,12 +2174,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        s = check_string(space, space.getattr(w_node, space.wrap('s')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        s = check_string(space, get_field(space, w_node, 's', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Str(s, lineno, col_offset)
 
-State.ast_type('Str', 'expr')
+State.ast_type('Str', 'expr', ['s'])
 
 
 class Attribute(expr):
@@ -2206,14 +2213,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        attr = space.str_w(space.getattr(w_node, space.wrap('attr')))
-        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        attr = space.str_w(get_field(space, w_node, 'attr', False))
+        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Attribute(value, attr, ctx, lineno, col_offset)
 
-State.ast_type('Attribute', 'expr')
+State.ast_type('Attribute', 'expr', ['value', 'attr', 'ctx'])
 
 
 class Subscript(expr):
@@ -2248,14 +2255,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
-        slice = slice.from_object(space, space.getattr(w_node, space.wrap('slice')))
-        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        slice = slice.from_object(space, get_field(space, w_node, 'slice', False))
+        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Subscript(value, slice, ctx, lineno, col_offset)
 
-State.ast_type('Subscript', 'expr')
+State.ast_type('Subscript', 'expr', ['value', 'slice', 'ctx'])
 
 
 class Name(expr):
@@ -2285,13 +2292,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        id = space.str_w(space.getattr(w_node, space.wrap('id')))
-        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        id = space.str_w(get_field(space, w_node, 'id', False))
+        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Name(id, ctx, lineno, col_offset)
 
-State.ast_type('Name', 'expr')
+State.ast_type('Name', 'expr', ['id', 'ctx'])
 
 
 class List(expr):
@@ -2330,12 +2337,12 @@
         elts_w = space.unpackiterable(
                 get_field(space, w_node, 'elts', False))
         elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return List(elts, ctx, lineno, col_offset)
 
-State.ast_type('List', 'expr')
+State.ast_type('List', 'expr', ['elts', 'ctx'])
 
 
 class Tuple(expr):
@@ -2374,12 +2381,12 @@
         elts_w = space.unpackiterable(
                 get_field(space, w_node, 'elts', False))
         elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        ctx = expr_context.from_object(space, space.getattr(w_node, space.wrap('ctx')))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Tuple(elts, ctx, lineno, col_offset)
 
-State.ast_type('Tuple', 'expr')
+State.ast_type('Tuple', 'expr', ['elts', 'ctx'])
 
 
 class Const(expr):
@@ -2406,12 +2413,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = space.getattr(w_node, space.wrap('value'))
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        value = get_field(space, w_node, 'value', False)
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return Const(value, lineno, col_offset)
 
-State.ast_type('Const', 'expr')
+State.ast_type('Const', 'expr', ['value'])
 
 
 class expr_context(AST):
@@ -2431,37 +2438,37 @@
             return 6
         raise operationerrfmt(space.w_TypeError,
                 "Expected expr_context node, got %T", w_node)
-State.ast_type('expr_context', 'AST')
+State.ast_type('expr_context', 'AST', None)
 
 class _Load(expr_context):
     def to_object(self, space):
         return space.call_function(get(space).w_Load)
-State.ast_type('Load', 'expr_context')
+State.ast_type('Load', 'expr_context', None)
 
 class _Store(expr_context):
     def to_object(self, space):
         return space.call_function(get(space).w_Store)
-State.ast_type('Store', 'expr_context')
+State.ast_type('Store', 'expr_context', None)
 
 class _Del(expr_context):
     def to_object(self, space):
         return space.call_function(get(space).w_Del)
-State.ast_type('Del', 'expr_context')
+State.ast_type('Del', 'expr_context', None)
 
 class _AugLoad(expr_context):
     def to_object(self, space):
         return space.call_function(get(space).w_AugLoad)
-State.ast_type('AugLoad', 'expr_context')
+State.ast_type('AugLoad', 'expr_context', None)
 
 class _AugStore(expr_context):
     def to_object(self, space):
         return space.call_function(get(space).w_AugStore)
-State.ast_type('AugStore', 'expr_context')
+State.ast_type('AugStore', 'expr_context', None)
 
 class _Param(expr_context):
     def to_object(self, space):
         return space.call_function(get(space).w_Param)
-State.ast_type('Param', 'expr_context')
+State.ast_type('Param', 'expr_context', None)
 
 Load = 1
 Store = 2
@@ -2492,7 +2499,7 @@
             return Index.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected slice node, got %T", w_node)
-State.ast_type('slice', 'AST')
+State.ast_type('slice', 'AST', None)
 
 class Ellipsis(slice):
 
@@ -2511,7 +2518,7 @@
     def from_object(space, w_node):
         return Ellipsis()
 
-State.ast_type('Ellipsis', 'slice')
+State.ast_type('Ellipsis', 'slice', [])
 
 
 class Slice(slice):
@@ -2545,12 +2552,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lower = expr.from_object(space, space.getattr(w_node, space.wrap('lower')))
-        upper = expr.from_object(space, space.getattr(w_node, space.wrap('upper')))
-        step = expr.from_object(space, space.getattr(w_node, space.wrap('step')))
+        lower = expr.from_object(space, get_field(space, w_node, 'lower', True))
+        upper = expr.from_object(space, get_field(space, w_node, 'upper', True))
+        step = expr.from_object(space, get_field(space, w_node, 'step', True))
         return Slice(lower, upper, step)
 
-State.ast_type('Slice', 'slice')
+State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
 
 
 class ExtSlice(slice):
@@ -2583,7 +2590,7 @@
         dims = [slice.from_object(space, w_item) for w_item in dims_w]
         return ExtSlice(dims)
 
-State.ast_type('ExtSlice', 'slice')
+State.ast_type('ExtSlice', 'slice', ['dims'])
 
 
 class Index(slice):
@@ -2606,10 +2613,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
         return Index(value)
 
-State.ast_type('Index', 'slice')
+State.ast_type('Index', 'slice', ['value'])
 
 
 class boolop(AST):
@@ -2621,17 +2628,17 @@
             return 2
         raise operationerrfmt(space.w_TypeError,
                 "Expected boolop node, got %T", w_node)
-State.ast_type('boolop', 'AST')
+State.ast_type('boolop', 'AST', None)
 
 class _And(boolop):
     def to_object(self, space):
         return space.call_function(get(space).w_And)
-State.ast_type('And', 'boolop')
+State.ast_type('And', 'boolop', None)
 
 class _Or(boolop):
     def to_object(self, space):
         return space.call_function(get(space).w_Or)
-State.ast_type('Or', 'boolop')
+State.ast_type('Or', 'boolop', None)
 
 And = 1
 Or = 2
@@ -2670,67 +2677,67 @@
             return 12
         raise operationerrfmt(space.w_TypeError,
                 "Expected operator node, got %T", w_node)
-State.ast_type('operator', 'AST')
+State.ast_type('operator', 'AST', None)
 
 class _Add(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_Add)
-State.ast_type('Add', 'operator')
+State.ast_type('Add', 'operator', None)
 
 class _Sub(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_Sub)
-State.ast_type('Sub', 'operator')
+State.ast_type('Sub', 'operator', None)
 
 class _Mult(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_Mult)
-State.ast_type('Mult', 'operator')
+State.ast_type('Mult', 'operator', None)
 
 class _Div(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_Div)
-State.ast_type('Div', 'operator')
+State.ast_type('Div', 'operator', None)
 
 class _Mod(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_Mod)
-State.ast_type('Mod', 'operator')
+State.ast_type('Mod', 'operator', None)
 
 class _Pow(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_Pow)
-State.ast_type('Pow', 'operator')
+State.ast_type('Pow', 'operator', None)
 
 class _LShift(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_LShift)
-State.ast_type('LShift', 'operator')
+State.ast_type('LShift', 'operator', None)
 
 class _RShift(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_RShift)
-State.ast_type('RShift', 'operator')
+State.ast_type('RShift', 'operator', None)
 
 class _BitOr(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_BitOr)
-State.ast_type('BitOr', 'operator')
+State.ast_type('BitOr', 'operator', None)
 
 class _BitXor(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_BitXor)
-State.ast_type('BitXor', 'operator')
+State.ast_type('BitXor', 'operator', None)
 
 class _BitAnd(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_BitAnd)
-State.ast_type('BitAnd', 'operator')
+State.ast_type('BitAnd', 'operator', None)
 
 class _FloorDiv(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_FloorDiv)
-State.ast_type('FloorDiv', 'operator')
+State.ast_type('FloorDiv', 'operator', None)
 
 Add = 1
 Sub = 2
@@ -2773,27 +2780,27 @@
             return 4
         raise operationerrfmt(space.w_TypeError,
                 "Expected unaryop node, got %T", w_node)
-State.ast_type('unaryop', 'AST')
+State.ast_type('unaryop', 'AST', None)
 
 class _Invert(unaryop):
     def to_object(self, space):
         return space.call_function(get(space).w_Invert)
-State.ast_type('Invert', 'unaryop')
+State.ast_type('Invert', 'unaryop', None)
 
 class _Not(unaryop):
     def to_object(self, space):
         return space.call_function(get(space).w_Not)
-State.ast_type('Not', 'unaryop')
+State.ast_type('Not', 'unaryop', None)
 
 class _UAdd(unaryop):
     def to_object(self, space):
         return space.call_function(get(space).w_UAdd)
-State.ast_type('UAdd', 'unaryop')
+State.ast_type('UAdd', 'unaryop', None)
 
 class _USub(unaryop):
     def to_object(self, space):
         return space.call_function(get(space).w_USub)
-State.ast_type('USub', 'unaryop')
+State.ast_type('USub', 'unaryop', None)
 
 Invert = 1
 Not = 2
@@ -2832,57 +2839,57 @@
             return 10
         raise operationerrfmt(space.w_TypeError,
                 "Expected cmpop node, got %T", w_node)
-State.ast_type('cmpop', 'AST')
+State.ast_type('cmpop', 'AST', None)
 
 class _Eq(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_Eq)
-State.ast_type('Eq', 'cmpop')
+State.ast_type('Eq', 'cmpop', None)
 
 class _NotEq(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_NotEq)
-State.ast_type('NotEq', 'cmpop')
+State.ast_type('NotEq', 'cmpop', None)
 
 class _Lt(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_Lt)
-State.ast_type('Lt', 'cmpop')
+State.ast_type('Lt', 'cmpop', None)
 
 class _LtE(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_LtE)
-State.ast_type('LtE', 'cmpop')
+State.ast_type('LtE', 'cmpop', None)
 
 class _Gt(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_Gt)
-State.ast_type('Gt', 'cmpop')
+State.ast_type('Gt', 'cmpop', None)
 
 class _GtE(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_GtE)
-State.ast_type('GtE', 'cmpop')
+State.ast_type('GtE', 'cmpop', None)
 
 class _Is(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_Is)
-State.ast_type('Is', 'cmpop')
+State.ast_type('Is', 'cmpop', None)
 
 class _IsNot(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_IsNot)
-State.ast_type('IsNot', 'cmpop')
+State.ast_type('IsNot', 'cmpop', None)
 
 class _In(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_In)
-State.ast_type('In', 'cmpop')
+State.ast_type('In', 'cmpop', None)
 
 class _NotIn(cmpop):
     def to_object(self, space):
         return space.call_function(get(space).w_NotIn)
-State.ast_type('NotIn', 'cmpop')
+State.ast_type('NotIn', 'cmpop', None)
 
 Eq = 1
 NotEq = 2
@@ -2941,14 +2948,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        target = expr.from_object(space, space.getattr(w_node, space.wrap('target')))
-        iter = expr.from_object(space, space.getattr(w_node, space.wrap('iter')))
+        target = expr.from_object(space, get_field(space, w_node, 'target', False))
+        iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
         ifs_w = space.unpackiterable(
                 get_field(space, w_node, 'ifs', False))
         ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(target, iter, ifs)
 
-State.ast_type('comprehension', 'AST')
+State.ast_type('comprehension', 'AST', None)
 
 class excepthandler(AST):
 
@@ -2962,7 +2969,7 @@
             return ExceptHandler.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected excepthandler node, got %T", w_node)
-State.ast_type('excepthandler', 'AST')
+State.ast_type('excepthandler', 'AST', None)
 
 class ExceptHandler(excepthandler):
 
@@ -3004,16 +3011,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        type = expr.from_object(space, space.getattr(w_node, space.wrap('type')))
-        name = expr.from_object(space, space.getattr(w_node, space.wrap('name')))
+        type = expr.from_object(space, get_field(space, w_node, 'type', True))
+        name = expr.from_object(space, get_field(space, w_node, 'name', True))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
         body = [stmt.from_object(space, w_item) for w_item in body_w]
-        lineno = space.int_w(space.getattr(w_node, space.wrap('lineno')))
-        col_offset = space.int_w(space.getattr(w_node, space.wrap('col_offset')))
+        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
         return ExceptHandler(type, name, body, lineno, col_offset)
 
-State.ast_type('ExceptHandler', 'excepthandler')
+State.ast_type('ExceptHandler', 'excepthandler', ['type', 'name', 'body'])
 
 
 class arguments(AST):
@@ -3059,14 +3066,14 @@
         args_w = space.unpackiterable(
                 get_field(space, w_node, 'args', False))
         args = [expr.from_object(space, w_item) for w_item in args_w]
-        vararg = space.str_w(space.getattr(w_node, space.wrap('vararg')))
-        kwarg = space.str_w(space.getattr(w_node, space.wrap('kwarg')))
+        vararg = space.str_w(get_field(space, w_node, 'vararg', True))
+        kwarg = space.str_w(get_field(space, w_node, 'kwarg', True))
         defaults_w = space.unpackiterable(
                 get_field(space, w_node, 'defaults', False))
         defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(args, vararg, kwarg, defaults)
 
-State.ast_type('arguments', 'AST')
+State.ast_type('arguments', 'AST', None)
 
 class keyword(AST):
 
@@ -3091,11 +3098,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        arg = space.str_w(space.getattr(w_node, space.wrap('arg')))
-        value = expr.from_object(space, space.getattr(w_node, space.wrap('value')))
+        arg = space.str_w(get_field(space, w_node, 'arg', False))
+        value = expr.from_object(space, get_field(space, w_node, 'value', False))
         return keyword(arg, value)
 
-State.ast_type('keyword', 'AST')
+State.ast_type('keyword', 'AST', None)
 
 class alias(AST):
 
@@ -3119,11 +3126,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.str_w(space.getattr(w_node, space.wrap('name')))
-        asname = space.str_w(space.getattr(w_node, space.wrap('asname')))
+        name = space.str_w(get_field(space, w_node, 'name', False))
+        asname = space.str_w(get_field(space, w_node, 'asname', True))
         return alias(name, asname)
 
-State.ast_type('alias', 'AST')
+State.ast_type('alias', 'AST', None)
 
 class ASTVisitor(object):
 
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -28,10 +28,17 @@
         w_op = space.getattr(w_node, space.wrap("op"))
         assert space.isinstance_w(w_op, ast.get(space).w_operator)
 
-    def from_object(self, space):
+    def test_from_object(self, space):
         value = space.wrap(42)
         w_node = space.call_function(ast.get(space).w_Num)
         space.setattr(w_node, space.wrap('n'), value)
-        node = ast.Num.from_object(space, w_node, lineno=1, col_offset=1)
+        space.setattr(w_node, space.wrap('lineno'), space.wrap(1))
+        space.setattr(w_node, space.wrap('col_offset'), space.wrap(1))
+        node = ast.Num.from_object(space, w_node)
         assert node.n is value
+
+    def test_fields(self, space):
+        w_fields = space.getattr(ast.get(space).w_FunctionDef, space.wrap("_fields"))
+        assert (space.listview_str(w_fields) == 
+                ['name', 'args', 'body', 'decorator_list'])
         
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -65,13 +65,13 @@
                 self.emit("return %i" % (i+1,), 3)
             self.emit("raise operationerrfmt(space.w_TypeError,", 2)
             self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
-            self.emit("State.ast_type('%s', 'AST')" % (base,))
+            self.emit("State.ast_type('%s', 'AST', None)" % (base,))
             self.emit("")
             for i, cons in enumerate(sum.types):
                 self.emit("class _%s(%s):" % (cons.name, base))
                 self.emit("def to_object(self, space):", 1)
                 self.emit("return space.call_function(get(space).w_%s)" % (cons.name,), 2)
-                self.emit("State.ast_type('%s', '%s')" % (cons.name, base))
+                self.emit("State.ast_type('%s', '%s', None)" % (cons.name, base))
                 self.emit("")
             for i, cons in enumerate(sum.types):
                 self.emit("%s = %i" % (cons.name, i + 1))
@@ -99,7 +99,7 @@
                           % (typ.name,), 3)
             self.emit("raise operationerrfmt(space.w_TypeError,", 2)
             self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
-            self.emit("State.ast_type('%r', 'AST')" % (base,))
+            self.emit("State.ast_type('%r', 'AST', None)" % (base,))
             self.emit("")
             for cons in sum.types:
                 self.visit(cons, base, sum.attributes)
@@ -115,7 +115,7 @@
         self.emit("visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
         self.make_converters(product.fields, name)
-        self.emit("State.ast_type('%r', 'AST')" % (name,))
+        self.emit("State.ast_type('%r', 'AST', None)" % (name,))
         self.emit("")
 
     def get_value_converter(self, field, value):
@@ -168,8 +168,8 @@
                          (field.name, value, field.name))
         else:
             value = self.get_value_extractor(
-                field, 
-                "space.getattr(w_node, space.wrap('%s'))" % (field.name,))
+                field,
+                "get_field(space, w_node, '%s', %s)" % (field.name, field.opt))
             lines = ["%s = %s" % (field.name, value)]
         return lines
 
@@ -236,7 +236,8 @@
         self.emit("")
         self.make_mutate_over(cons, cons.name)
         self.make_converters(cons.fields, cons.name, extra_attributes)
-        self.emit("State.ast_type('%r', '%s')" % (cons.name, base))
+        self.emit("State.ast_type('%r', '%s', %s)" % 
+                  (cons.name, base, [repr(f.name) for f in cons.fields]))
         self.emit("")
 
     def visitField(self, field):
@@ -448,18 +449,21 @@
             space.setattr(self, w_name,
                           space.getitem(w_state, w_name))
 
-def get_W_AST_new(node_class):
-    def generic_W_AST_new(space, w_type, __args__):
-        node = space.allocate_instance(node_class, w_type)
-        return space.wrap(node)
-    return func_with_new_name(generic_W_AST_new, "new_%s" % node_class.__name__)
-
+def W_AST_new(space, w_type, __args__):
+    node = space.allocate_instance(W_AST, w_type)
+    return space.wrap(node)
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
-    if args_w and len(args_w) != 0:
-        w_err = space.wrap("_ast.AST constructor takes 0 positional arguments")
-        raise OperationError(space.w_TypeError, w_err)
+    fields_w = space.unpackiterable(space.getattr(space.type(w_self),
+                                    space.wrap("_fields")))
+    num_fields = len(fields_w) if fields_w else 0
+    if args_w and len(args_w) != num_fields:
+        raise operationerrfmt(space.w_TypeError,
+                "_ast.%T constructor takes %s positional arguments", w_self, num_fields)
+    if args_w:
+        for i, w_field in enumerate(fields_w):
+            space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
         space.setattr(w_self, space.wrap(field), w_value)
 
@@ -472,7 +476,7 @@
     __setstate__=interp2app(W_AST.setstate_w),
     __dict__ = typedef.GetSetProperty(typedef.descr_get_dict,
                                       typedef.descr_set_dict, cls=W_AST),
-    __new__=interp2app(get_W_AST_new(W_AST)),
+    __new__=interp2app(W_AST_new),
     __init__=interp2app(W_AST_init),
 )
 
@@ -480,19 +484,23 @@
     AST_TYPES = []
 
     @classmethod
-    def ast_type(cls, name, base):
-        cls.AST_TYPES.append((name, base))
+    def ast_type(cls, name, base, fields):
+        cls.AST_TYPES.append((name, base, fields))
 
     def __init__(self, space):
         self.w_AST = space.gettypeobject(W_AST.typedef)
-        for (name, base) in self.AST_TYPES:
-            self.make_new_type(space, name, base)
+        for (name, base, fields) in self.AST_TYPES:
+            self.make_new_type(space, name, base, fields)
         
-    def make_new_type(self, space, name, base):
+    def make_new_type(self, space, name, base, fields):
         w_base = getattr(self, 'w_%s' % base)
+        w_dict = space.newdict()
+        if fields is not None:
+            space.setitem_str(w_dict, "_fields",
+                              space.newtuple([space.wrap(f) for f in fields]))
         w_type = space.call_function(
             space.w_type, 
-            space.wrap(name), space.newtuple([w_base]), space.newdict())
+            space.wrap(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
diff --git a/pypy/module/_ast/__init__.py b/pypy/module/_ast/__init__.py
--- a/pypy/module/_ast/__init__.py
+++ b/pypy/module/_ast/__init__.py
@@ -13,6 +13,6 @@
 
 def _setup():
     defs = Module.interpleveldefs
-    for (name, base) in ast.State.AST_TYPES:
+    for (name, base, fields) in ast.State.AST_TYPES:
         defs[name] = "pypy.interpreter.astcompiler.ast.get(space).w_" + name
 _setup()

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:12 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:12 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Fixes for optional fields
Message-ID: <20130626212212.7D5DC1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65020:f4e2683859c2
Date: 2013-06-24 23:17 +0200
http://bitbucket.org/pypy/pypy/changeset/f4e2683859c2/

Log:	Fixes for optional fields

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -137,6 +137,8 @@
 class mod(AST):
     @staticmethod
     def from_object(space, w_node):
+        if space.is_w(w_node, space.w_None):
+            return None
         if space.isinstance_w(w_node, get(space).w_Module):
             return Module.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Interactive):
@@ -282,6 +284,8 @@
 
     @staticmethod
     def from_object(space, w_node):
+        if space.is_w(w_node, space.w_None):
+            return None
         if space.isinstance_w(w_node, get(space).w_FunctionDef):
             return FunctionDef.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_ClassDef):
@@ -477,7 +481,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space) if self.value else space.w_None  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -642,7 +646,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Print)
-        w_dest = self.dest.to_object(space)  # expr
+        w_dest = self.dest.to_object(space) if self.dest else space.w_None  # expr
         space.setattr(w_node, space.wrap('dest'), w_dest)
         if self.values is None:
             values_w = []
@@ -871,7 +875,7 @@
         w_node = space.call_function(get(space).w_With)
         w_context_expr = self.context_expr.to_object(space)  # expr
         space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
-        w_optional_vars = self.optional_vars.to_object(space)  # expr
+        w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars else space.w_None  # expr
         space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
         if self.body is None:
             body_w = []
@@ -921,11 +925,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
-        w_type = self.type.to_object(space)  # expr
+        w_type = self.type.to_object(space) if self.type else space.w_None  # expr
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_inst = self.inst.to_object(space)  # expr
+        w_inst = self.inst.to_object(space) if self.inst else space.w_None  # expr
         space.setattr(w_node, space.wrap('inst'), w_inst)
-        w_tback = self.tback.to_object(space)  # expr
+        w_tback = self.tback.to_object(space) if self.tback else space.w_None  # expr
         space.setattr(w_node, space.wrap('tback'), w_tback)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -1081,7 +1085,7 @@
         w_node = space.call_function(get(space).w_Assert)
         w_test = self.test.to_object(space)  # expr
         space.setattr(w_node, space.wrap('test'), w_test)
-        w_msg = self.msg.to_object(space)  # expr
+        w_msg = self.msg.to_object(space) if self.msg else space.w_None  # expr
         space.setattr(w_node, space.wrap('msg'), w_msg)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -1211,9 +1215,9 @@
         w_node = space.call_function(get(space).w_Exec)
         w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
-        w_globals = self.globals.to_object(space)  # expr
+        w_globals = self.globals.to_object(space) if self.globals else space.w_None  # expr
         space.setattr(w_node, space.wrap('globals'), w_globals)
-        w_locals = self.locals.to_object(space)  # expr
+        w_locals = self.locals.to_object(space) if self.locals else space.w_None  # expr
         space.setattr(w_node, space.wrap('locals'), w_locals)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -1396,6 +1400,8 @@
 
     @staticmethod
     def from_object(space, w_node):
+        if space.is_w(w_node, space.w_None):
+            return None
         if space.isinstance_w(w_node, get(space).w_BoolOp):
             return BoolOp.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_BinOp):
@@ -1943,7 +1949,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
-        w_value = self.value.to_object(space)  # expr
+        w_value = self.value.to_object(space) if self.value else space.w_None  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -2057,9 +2063,9 @@
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
         space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_starargs = self.starargs.to_object(space)  # expr
+        w_starargs = self.starargs.to_object(space) if self.starargs else space.w_None  # expr
         space.setattr(w_node, space.wrap('starargs'), w_starargs)
-        w_kwargs = self.kwargs.to_object(space)  # expr
+        w_kwargs = self.kwargs.to_object(space) if self.kwargs else space.w_None  # expr
         space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -2489,6 +2495,8 @@
 class slice(AST):
     @staticmethod
     def from_object(space, w_node):
+        if space.is_w(w_node, space.w_None):
+            return None
         if space.isinstance_w(w_node, get(space).w_Ellipsis):
             return Ellipsis.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Slice):
@@ -2542,11 +2550,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
-        w_lower = self.lower.to_object(space)  # expr
+        w_lower = self.lower.to_object(space) if self.lower else space.w_None  # expr
         space.setattr(w_node, space.wrap('lower'), w_lower)
-        w_upper = self.upper.to_object(space)  # expr
+        w_upper = self.upper.to_object(space) if self.upper else space.w_None  # expr
         space.setattr(w_node, space.wrap('upper'), w_upper)
-        w_step = self.step.to_object(space)  # expr
+        w_step = self.step.to_object(space) if self.step else space.w_None  # expr
         space.setattr(w_node, space.wrap('step'), w_step)
         return w_node
 
@@ -2965,6 +2973,8 @@
 
     @staticmethod
     def from_object(space, w_node):
+        if space.is_w(w_node, space.w_None):
+            return None
         if space.isinstance_w(w_node, get(space).w_ExceptHandler):
             return ExceptHandler.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
@@ -2993,9 +3003,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
-        w_type = self.type.to_object(space)  # expr
+        w_type = self.type.to_object(space) if self.type else space.w_None  # expr
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_name = self.name.to_object(space)  # expr
+        w_name = self.name.to_object(space) if self.name else space.w_None  # expr
         space.setattr(w_node, space.wrap('name'), w_name)
         if self.body is None:
             body_w = []
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -92,6 +92,8 @@
                 self.emit("")
             self.emit("@staticmethod", 1)
             self.emit("def from_object(space, w_node):", 1)
+            self.emit("if space.is_w(w_node, space.w_None):", 2)
+            self.emit("    return None", 2)
             for typ in sum.types:
                 self.emit("if space.isinstance_w(w_node, get(space).w_%s):"
                           % (typ.name,), 2)
@@ -126,8 +128,11 @@
         elif field.type.value in ("identifier", "int"):
             return "space.wrap(%s)" % (value,)
         else:
-            return "%s.to_object(space)" % (value,)
-
+            wrapper = "%s.to_object(space)" % (value,)
+            if field.opt:
+                wrapper += " if %s else space.w_None" % (value,)
+            return wrapper
+        
     def get_value_extractor(self, field, value):
         if field.type.value in self.data.simple_types:
             return "%s.from_object(space, %s)" % (field.type, value)
diff --git a/pypy/module/_ast/__init__.py b/pypy/module/_ast/__init__.py
--- a/pypy/module/_ast/__init__.py
+++ b/pypy/module/_ast/__init__.py
@@ -13,6 +13,7 @@
 
 def _setup():
     defs = Module.interpleveldefs
+    defs['AST'] = "pypy.interpreter.astcompiler.ast.get(space).w_AST"
     for (name, base, fields) in ast.State.AST_TYPES:
         defs[name] = "pypy.interpreter.astcompiler.ast.get(space).w_" + name
 _setup()

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:13 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:13 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Add _attributes
Message-ID: <20130626212213.AC0AD1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65021:c8620404fbf2
Date: 2013-06-24 23:39 +0200
http://bitbucket.org/pypy/pypy/changeset/c8620404fbf2/

Log:	Add _attributes

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -112,20 +112,23 @@
     AST_TYPES = []
 
     @classmethod
-    def ast_type(cls, name, base, fields):
-        cls.AST_TYPES.append((name, base, fields))
+    def ast_type(cls, name, base, fields, attributes=None):
+        cls.AST_TYPES.append((name, base, fields, attributes))
 
     def __init__(self, space):
         self.w_AST = space.gettypeobject(W_AST.typedef)
-        for (name, base, fields) in self.AST_TYPES:
-            self.make_new_type(space, name, base, fields)
+        for (name, base, fields, attributes) in self.AST_TYPES:
+            self.make_new_type(space, name, base, fields, attributes)
         
-    def make_new_type(self, space, name, base, fields):
+    def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
                               space.newtuple([space.wrap(f) for f in fields]))
+        if attributes is not None:
+            space.setitem_str(w_dict, "_attributes",
+                              space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type, 
             space.wrap(name), space.newtuple([w_base]), w_dict)
@@ -149,7 +152,7 @@
             return Suite.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected mod node, got %T", w_node)
-State.ast_type('mod', 'AST', None)
+State.ast_type('mod', 'AST', None, [])
 
 class Module(mod):
 
@@ -334,7 +337,7 @@
             return Continue.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected stmt node, got %T", w_node)
-State.ast_type('stmt', 'AST', None)
+State.ast_type('stmt', 'AST', None, ['lineno', 'col_offset'])
 
 class FunctionDef(stmt):
 
@@ -1450,7 +1453,7 @@
             return Const.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected expr node, got %T", w_node)
-State.ast_type('expr', 'AST', None)
+State.ast_type('expr', 'AST', None, ['lineno', 'col_offset'])
 
 class BoolOp(expr):
 
@@ -2507,7 +2510,7 @@
             return Index.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected slice node, got %T", w_node)
-State.ast_type('slice', 'AST', None)
+State.ast_type('slice', 'AST', None, [])
 
 class Ellipsis(slice):
 
@@ -2979,7 +2982,7 @@
             return ExceptHandler.from_object(space, w_node)
         raise operationerrfmt(space.w_TypeError,
                 "Expected excepthandler node, got %T", w_node)
-State.ast_type('excepthandler', 'AST', None)
+State.ast_type('excepthandler', 'AST', None, ['lineno', 'col_offset'])
 
 class ExceptHandler(excepthandler):
 
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -38,7 +38,13 @@
         assert node.n is value
 
     def test_fields(self, space):
-        w_fields = space.getattr(ast.get(space).w_FunctionDef, space.wrap("_fields"))
-        assert (space.listview_str(w_fields) == 
-                ['name', 'args', 'body', 'decorator_list'])
+        w_fields = space.getattr(ast.get(space).w_FunctionDef,
+                                 space.wrap("_fields"))
+        assert space.eq_w(w_fields, space.wrap(
+            ('name', 'args', 'body', 'decorator_list')))
         
+    def test_attributes(self, space):
+        w_attrs = space.getattr(ast.get(space).w_FunctionDef,
+                                space.wrap("_attributes"))
+        assert space.eq_w(w_attrs, space.wrap(('lineno', 'col_offset')))
+        
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -101,7 +101,8 @@
                           % (typ.name,), 3)
             self.emit("raise operationerrfmt(space.w_TypeError,", 2)
             self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
-            self.emit("State.ast_type('%r', 'AST', None)" % (base,))
+            self.emit("State.ast_type('%r', 'AST', None, %s)" %
+                      (base, [repr(attr.name) for attr in sum.attributes]))
             self.emit("")
             for cons in sum.types:
                 self.visit(cons, base, sum.attributes)
@@ -489,20 +490,23 @@
     AST_TYPES = []
 
     @classmethod
-    def ast_type(cls, name, base, fields):
-        cls.AST_TYPES.append((name, base, fields))
+    def ast_type(cls, name, base, fields, attributes=None):
+        cls.AST_TYPES.append((name, base, fields, attributes))
 
     def __init__(self, space):
         self.w_AST = space.gettypeobject(W_AST.typedef)
-        for (name, base, fields) in self.AST_TYPES:
-            self.make_new_type(space, name, base, fields)
+        for (name, base, fields, attributes) in self.AST_TYPES:
+            self.make_new_type(space, name, base, fields, attributes)
         
-    def make_new_type(self, space, name, base, fields):
+    def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
                               space.newtuple([space.wrap(f) for f in fields]))
+        if attributes is not None:
+            space.setitem_str(w_dict, "_attributes",
+                              space.newtuple([space.wrap(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type, 
             space.wrap(name), space.newtuple([w_base]), w_dict)
diff --git a/pypy/module/_ast/__init__.py b/pypy/module/_ast/__init__.py
--- a/pypy/module/_ast/__init__.py
+++ b/pypy/module/_ast/__init__.py
@@ -14,6 +14,6 @@
 def _setup():
     defs = Module.interpleveldefs
     defs['AST'] = "pypy.interpreter.astcompiler.ast.get(space).w_AST"
-    for (name, base, fields) in ast.State.AST_TYPES:
+    for (name, base, fields, attributes) in ast.State.AST_TYPES:
         defs[name] = "pypy.interpreter.astcompiler.ast.get(space).w_" + name
 _setup()

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:14 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:14 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Fix test_pickle()
Message-ID: <20130626212214.CE9E61C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65022:ef60384c7186
Date: 2013-06-24 23:48 +0200
http://bitbucket.org/pypy/pypy/changeset/ef60384c7186/

Log:	Fix test_pickle()

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -59,7 +59,7 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = w_type.getdictvalue(space, "_fields")
+        w_fields = space.getattr(w_type, space.wrap("_fields"))
         for w_name in space.fixedview(w_fields):
             space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
@@ -83,8 +83,8 @@
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
-    fields_w = space.unpackiterable(space.getattr(space.type(w_self),
-                                    space.wrap("_fields")))
+    fields_w = space.fixedview(space.getattr(space.type(w_self),
+                               space.wrap("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         raise operationerrfmt(space.w_TypeError,
@@ -123,6 +123,7 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
+        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
                               space.newtuple([space.wrap(f) for f in fields]))
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -437,6 +437,7 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
+        w_fields = space.getattr(w_type, space.wrap("_fields"))
         w_fields = w_type.getdictvalue(space, "_fields")
         for w_name in space.fixedview(w_fields):
             space.setitem(w_dict, w_name,
@@ -461,8 +462,8 @@
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
-    fields_w = space.unpackiterable(space.getattr(space.type(w_self),
-                                    space.wrap("_fields")))
+    fields_w = space.fixedview(space.getattr(space.type(w_self),
+                               space.wrap("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         raise operationerrfmt(space.w_TypeError,
@@ -501,6 +502,7 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
+        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
                               space.newtuple([space.wrap(f) for f in fields]))

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:16 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:16 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: More fixes
Message-ID: <20130626212216.011EF1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65023:cef40a00180c
Date: 2013-06-24 23:56 +0200
http://bitbucket.org/pypy/pypy/changeset/cef40a00180c/

Log:	More fixes

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -87,8 +87,12 @@
                                space.wrap("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
-        raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes %s positional arguments", w_self, num_fields)
+        if num_fields:
+            raise operationerrfmt(space.w_TypeError,
+                "_ast.%T constructor takes either 0 or %s positional arguments", w_self, num_fields)
+        else:
+            raise operationerrfmt(space.w_TypeError,
+                "_ast.%T constructor takes 0 positional arguments", w_self)
     if args_w:
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
@@ -485,7 +489,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
-        w_value = self.value.to_object(space) if self.value else space.w_None  # expr
+        w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -650,7 +654,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Print)
-        w_dest = self.dest.to_object(space) if self.dest else space.w_None  # expr
+        w_dest = self.dest.to_object(space) if self.dest is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('dest'), w_dest)
         if self.values is None:
             values_w = []
@@ -879,7 +883,7 @@
         w_node = space.call_function(get(space).w_With)
         w_context_expr = self.context_expr.to_object(space)  # expr
         space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
-        w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars else space.w_None  # expr
+        w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
         if self.body is None:
             body_w = []
@@ -929,11 +933,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
-        w_type = self.type.to_object(space) if self.type else space.w_None  # expr
+        w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_inst = self.inst.to_object(space) if self.inst else space.w_None  # expr
+        w_inst = self.inst.to_object(space) if self.inst is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('inst'), w_inst)
-        w_tback = self.tback.to_object(space) if self.tback else space.w_None  # expr
+        w_tback = self.tback.to_object(space) if self.tback is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('tback'), w_tback)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -1089,7 +1093,7 @@
         w_node = space.call_function(get(space).w_Assert)
         w_test = self.test.to_object(space)  # expr
         space.setattr(w_node, space.wrap('test'), w_test)
-        w_msg = self.msg.to_object(space) if self.msg else space.w_None  # expr
+        w_msg = self.msg.to_object(space) if self.msg is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('msg'), w_msg)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -1184,7 +1188,7 @@
 
     @staticmethod
     def from_object(space, w_node):
-        module = space.str_w(get_field(space, w_node, 'module', True))
+        module = space.str_or_None_w(get_field(space, w_node, 'module', True))
         names_w = space.unpackiterable(
                 get_field(space, w_node, 'names', False))
         names = [alias.from_object(space, w_item) for w_item in names_w]
@@ -1219,9 +1223,9 @@
         w_node = space.call_function(get(space).w_Exec)
         w_body = self.body.to_object(space)  # expr
         space.setattr(w_node, space.wrap('body'), w_body)
-        w_globals = self.globals.to_object(space) if self.globals else space.w_None  # expr
+        w_globals = self.globals.to_object(space) if self.globals is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('globals'), w_globals)
-        w_locals = self.locals.to_object(space) if self.locals else space.w_None  # expr
+        w_locals = self.locals.to_object(space) if self.locals is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('locals'), w_locals)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -1953,7 +1957,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
-        w_value = self.value.to_object(space) if self.value else space.w_None  # expr
+        w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -2067,9 +2071,9 @@
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
         space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_starargs = self.starargs.to_object(space) if self.starargs else space.w_None  # expr
+        w_starargs = self.starargs.to_object(space) if self.starargs is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('starargs'), w_starargs)
-        w_kwargs = self.kwargs.to_object(space) if self.kwargs else space.w_None  # expr
+        w_kwargs = self.kwargs.to_object(space) if self.kwargs is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -2554,11 +2558,11 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
-        w_lower = self.lower.to_object(space) if self.lower else space.w_None  # expr
+        w_lower = self.lower.to_object(space) if self.lower is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('lower'), w_lower)
-        w_upper = self.upper.to_object(space) if self.upper else space.w_None  # expr
+        w_upper = self.upper.to_object(space) if self.upper is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('upper'), w_upper)
-        w_step = self.step.to_object(space) if self.step else space.w_None  # expr
+        w_step = self.step.to_object(space) if self.step is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('step'), w_step)
         return w_node
 
@@ -3007,9 +3011,9 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
-        w_type = self.type.to_object(space) if self.type else space.w_None  # expr
+        w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('type'), w_type)
-        w_name = self.name.to_object(space) if self.name else space.w_None  # expr
+        w_name = self.name.to_object(space) if self.name is not None else space.w_None  # expr
         space.setattr(w_node, space.wrap('name'), w_name)
         if self.body is None:
             body_w = []
@@ -3080,8 +3084,8 @@
         args_w = space.unpackiterable(
                 get_field(space, w_node, 'args', False))
         args = [expr.from_object(space, w_item) for w_item in args_w]
-        vararg = space.str_w(get_field(space, w_node, 'vararg', True))
-        kwarg = space.str_w(get_field(space, w_node, 'kwarg', True))
+        vararg = space.str_or_None_w(get_field(space, w_node, 'vararg', True))
+        kwarg = space.str_or_None_w(get_field(space, w_node, 'kwarg', True))
         defaults_w = space.unpackiterable(
                 get_field(space, w_node, 'defaults', False))
         defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
@@ -3141,7 +3145,7 @@
     @staticmethod
     def from_object(space, w_node):
         name = space.str_w(get_field(space, w_node, 'name', False))
-        asname = space.str_w(get_field(space, w_node, 'asname', True))
+        asname = space.str_or_None_w(get_field(space, w_node, 'asname', True))
         return alias(name, asname)
 
 State.ast_type('alias', 'AST', None)
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -131,7 +131,7 @@
         else:
             wrapper = "%s.to_object(space)" % (value,)
             if field.opt:
-                wrapper += " if %s else space.w_None" % (value,)
+                wrapper += " if %s is not None else space.w_None" % (value,)
             return wrapper
         
     def get_value_extractor(self, field, value):
@@ -142,6 +142,8 @@
         elif field.type.value in ("string",):
             return "check_string(space, %s)" % (value,)
         elif field.type.value in ("identifier",):
+            if field.opt:
+                return "space.str_or_None_w(%s)" % (value,)
             return "space.str_w(%s)" % (value,)
         elif field.type.value in ("int",):
             return "space.int_w(%s)" % (value,)
@@ -438,7 +440,6 @@
             w_dict = space.newdict()
         w_type = space.type(self)
         w_fields = space.getattr(w_type, space.wrap("_fields"))
-        w_fields = w_type.getdictvalue(space, "_fields")
         for w_name in space.fixedview(w_fields):
             space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
@@ -466,8 +467,12 @@
                                space.wrap("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
-        raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes %s positional arguments", w_self, num_fields)
+        if num_fields:
+            raise operationerrfmt(space.w_TypeError,
+                "_ast.%T constructor takes either 0 or %s positional arguments", w_self, num_fields)
+        else:
+            raise operationerrfmt(space.w_TypeError,
+                "_ast.%T constructor takes 0 positional arguments", w_self)
     if args_w:
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:17 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:17 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Translation fixes.
Message-ID: <20130626212217.32D8C1C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65024:e69a86a52c6a
Date: 2013-06-26 23:00 +0200
http://bitbucket.org/pypy/pypy/changeset/e69a86a52c6a/

Log:	Translation fixes.

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -89,7 +89,7 @@
     if args_w and len(args_w) != num_fields:
         if num_fields:
             raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes either 0 or %s positional arguments", w_self, num_fields)
+                "_ast.%T constructor takes either 0 or %d positional arguments", w_self, num_fields)
         else:
             raise operationerrfmt(space.w_TypeError,
                 "_ast.%T constructor takes 0 positional arguments", w_self)
@@ -186,8 +186,8 @@
     def from_object(space, w_node):
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
-        return Module(body)
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
+        return Module(_body)
 
 State.ast_type('Module', 'mod', ['body'])
 
@@ -219,8 +219,8 @@
     def from_object(space, w_node):
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
-        return Interactive(body)
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
+        return Interactive(_body)
 
 State.ast_type('Interactive', 'mod', ['body'])
 
@@ -245,8 +245,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = expr.from_object(space, get_field(space, w_node, 'body', False))
-        return Expression(body)
+        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        return Expression(_body)
 
 State.ast_type('Expression', 'mod', ['body'])
 
@@ -278,8 +278,8 @@
     def from_object(space, w_node):
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
-        return Suite(body)
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
+        return Suite(_body)
 
 State.ast_type('Suite', 'mod', ['body'])
 
@@ -390,17 +390,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.str_w(get_field(space, w_node, 'name', False))
-        args = arguments.from_object(space, get_field(space, w_node, 'args', False))
+        _name = space.str_w(get_field(space, w_node, 'name', False))
+        _args = arguments.from_object(space, get_field(space, w_node, 'args', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(
                 get_field(space, w_node, 'decorator_list', False))
-        decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return FunctionDef(name, args, body, decorator_list, lineno, col_offset)
+        _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return FunctionDef(_name, _args, _body, _decorator_list, _lineno, _col_offset)
 
 State.ast_type('FunctionDef', 'stmt', ['name', 'args', 'body', 'decorator_list'])
 
@@ -456,19 +456,19 @@
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.str_w(get_field(space, w_node, 'name', False))
+        _name = space.str_w(get_field(space, w_node, 'name', False))
         bases_w = space.unpackiterable(
                 get_field(space, w_node, 'bases', False))
-        bases = [expr.from_object(space, w_item) for w_item in bases_w]
+        _bases = [expr.from_object(space, w_item) for w_item in bases_w]
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(
                 get_field(space, w_node, 'decorator_list', False))
-        decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return ClassDef(name, bases, body, decorator_list, lineno, col_offset)
+        _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return ClassDef(_name, _bases, _body, _decorator_list, _lineno, _col_offset)
 
 State.ast_type('ClassDef', 'stmt', ['name', 'bases', 'body', 'decorator_list'])
 
@@ -499,10 +499,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, get_field(space, w_node, 'value', True))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Return(value, lineno, col_offset)
+        _value = expr.from_object(space, get_field(space, w_node, 'value', True))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Return(_value, _lineno, _col_offset)
 
 State.ast_type('Return', 'stmt', ['value'])
 
@@ -539,10 +539,10 @@
     def from_object(space, w_node):
         targets_w = space.unpackiterable(
                 get_field(space, w_node, 'targets', False))
-        targets = [expr.from_object(space, w_item) for w_item in targets_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Delete(targets, lineno, col_offset)
+        _targets = [expr.from_object(space, w_item) for w_item in targets_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Delete(_targets, _lineno, _col_offset)
 
 State.ast_type('Delete', 'stmt', ['targets'])
 
@@ -583,11 +583,11 @@
     def from_object(space, w_node):
         targets_w = space.unpackiterable(
                 get_field(space, w_node, 'targets', False))
-        targets = [expr.from_object(space, w_item) for w_item in targets_w]
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Assign(targets, value, lineno, col_offset)
+        _targets = [expr.from_object(space, w_item) for w_item in targets_w]
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Assign(_targets, _value, _lineno, _col_offset)
 
 State.ast_type('Assign', 'stmt', ['targets', 'value'])
 
@@ -624,12 +624,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        target = expr.from_object(space, get_field(space, w_node, 'target', False))
-        op = operator.from_object(space, get_field(space, w_node, 'op', False))
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return AugAssign(target, op, value, lineno, col_offset)
+        _target = expr.from_object(space, get_field(space, w_node, 'target', False))
+        _op = operator.from_object(space, get_field(space, w_node, 'op', False))
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return AugAssign(_target, _op, _value, _lineno, _col_offset)
 
 State.ast_type('AugAssign', 'stmt', ['target', 'op', 'value'])
 
@@ -662,7 +662,7 @@
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
         space.setattr(w_node, space.wrap('values'), w_values)
-        w_nl = self.nl.to_object(space)  # bool
+        w_nl = space.wrap(self.nl)  # bool
         space.setattr(w_node, space.wrap('nl'), w_nl)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -672,14 +672,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        dest = expr.from_object(space, get_field(space, w_node, 'dest', True))
+        _dest = expr.from_object(space, get_field(space, w_node, 'dest', True))
         values_w = space.unpackiterable(
                 get_field(space, w_node, 'values', False))
-        values = [expr.from_object(space, w_item) for w_item in values_w]
-        nl = bool.from_object(space, get_field(space, w_node, 'nl', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Print(dest, values, nl, lineno, col_offset)
+        _values = [expr.from_object(space, w_item) for w_item in values_w]
+        _nl = space.bool_w(get_field(space, w_node, 'nl', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Print(_dest, _values, _nl, _lineno, _col_offset)
 
 State.ast_type('Print', 'stmt', ['dest', 'values', 'nl'])
 
@@ -731,17 +731,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        target = expr.from_object(space, get_field(space, w_node, 'target', False))
-        iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
+        _target = expr.from_object(space, get_field(space, w_node, 'target', False))
+        _iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
-        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return For(target, iter, body, orelse, lineno, col_offset)
+        _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return For(_target, _iter, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('For', 'stmt', ['target', 'iter', 'body', 'orelse'])
 
@@ -789,16 +789,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, get_field(space, w_node, 'test', False))
+        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
-        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return While(test, body, orelse, lineno, col_offset)
+        _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return While(_test, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('While', 'stmt', ['test', 'body', 'orelse'])
 
@@ -846,16 +846,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, get_field(space, w_node, 'test', False))
+        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
-        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return If(test, body, orelse, lineno, col_offset)
+        _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return If(_test, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('If', 'stmt', ['test', 'body', 'orelse'])
 
@@ -899,14 +899,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        context_expr = expr.from_object(space, get_field(space, w_node, 'context_expr', False))
-        optional_vars = expr.from_object(space, get_field(space, w_node, 'optional_vars', True))
+        _context_expr = expr.from_object(space, get_field(space, w_node, 'context_expr', False))
+        _optional_vars = expr.from_object(space, get_field(space, w_node, 'optional_vars', True))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return With(context_expr, optional_vars, body, lineno, col_offset)
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return With(_context_expr, _optional_vars, _body, _lineno, _col_offset)
 
 State.ast_type('With', 'stmt', ['context_expr', 'optional_vars', 'body'])
 
@@ -947,12 +947,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        type = expr.from_object(space, get_field(space, w_node, 'type', True))
-        inst = expr.from_object(space, get_field(space, w_node, 'inst', True))
-        tback = expr.from_object(space, get_field(space, w_node, 'tback', True))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Raise(type, inst, tback, lineno, col_offset)
+        _type = expr.from_object(space, get_field(space, w_node, 'type', True))
+        _inst = expr.from_object(space, get_field(space, w_node, 'inst', True))
+        _tback = expr.from_object(space, get_field(space, w_node, 'tback', True))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Raise(_type, _inst, _tback, _lineno, _col_offset)
 
 State.ast_type('Raise', 'stmt', ['type', 'inst', 'tback'])
 
@@ -1007,16 +1007,16 @@
     def from_object(space, w_node):
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
         handlers_w = space.unpackiterable(
                 get_field(space, w_node, 'handlers', False))
-        handlers = [excepthandler.from_object(space, w_item) for w_item in handlers_w]
+        _handlers = [excepthandler.from_object(space, w_item) for w_item in handlers_w]
         orelse_w = space.unpackiterable(
                 get_field(space, w_node, 'orelse', False))
-        orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return TryExcept(body, handlers, orelse, lineno, col_offset)
+        _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return TryExcept(_body, _handlers, _orelse, _lineno, _col_offset)
 
 State.ast_type('TryExcept', 'stmt', ['body', 'handlers', 'orelse'])
 
@@ -1062,13 +1062,13 @@
     def from_object(space, w_node):
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
         finalbody_w = space.unpackiterable(
                 get_field(space, w_node, 'finalbody', False))
-        finalbody = [stmt.from_object(space, w_item) for w_item in finalbody_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return TryFinally(body, finalbody, lineno, col_offset)
+        _finalbody = [stmt.from_object(space, w_item) for w_item in finalbody_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return TryFinally(_body, _finalbody, _lineno, _col_offset)
 
 State.ast_type('TryFinally', 'stmt', ['body', 'finalbody'])
 
@@ -1103,11 +1103,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, get_field(space, w_node, 'test', False))
-        msg = expr.from_object(space, get_field(space, w_node, 'msg', True))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Assert(test, msg, lineno, col_offset)
+        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
+        _msg = expr.from_object(space, get_field(space, w_node, 'msg', True))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Assert(_test, _msg, _lineno, _col_offset)
 
 State.ast_type('Assert', 'stmt', ['test', 'msg'])
 
@@ -1144,10 +1144,10 @@
     def from_object(space, w_node):
         names_w = space.unpackiterable(
                 get_field(space, w_node, 'names', False))
-        names = [alias.from_object(space, w_item) for w_item in names_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Import(names, lineno, col_offset)
+        _names = [alias.from_object(space, w_item) for w_item in names_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Import(_names, _lineno, _col_offset)
 
 State.ast_type('Import', 'stmt', ['names'])
 
@@ -1188,14 +1188,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        module = space.str_or_None_w(get_field(space, w_node, 'module', True))
+        _module = space.str_or_None_w(get_field(space, w_node, 'module', True))
         names_w = space.unpackiterable(
                 get_field(space, w_node, 'names', False))
-        names = [alias.from_object(space, w_item) for w_item in names_w]
-        level = space.int_w(get_field(space, w_node, 'level', True))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return ImportFrom(module, names, level, lineno, col_offset)
+        _names = [alias.from_object(space, w_item) for w_item in names_w]
+        _level = space.int_w(get_field(space, w_node, 'level', True))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return ImportFrom(_module, _names, _level, _lineno, _col_offset)
 
 State.ast_type('ImportFrom', 'stmt', ['module', 'names', 'level'])
 
@@ -1235,12 +1235,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body = expr.from_object(space, get_field(space, w_node, 'body', False))
-        globals = expr.from_object(space, get_field(space, w_node, 'globals', True))
-        locals = expr.from_object(space, get_field(space, w_node, 'locals', True))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Exec(body, globals, locals, lineno, col_offset)
+        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        _globals = expr.from_object(space, get_field(space, w_node, 'globals', True))
+        _locals = expr.from_object(space, get_field(space, w_node, 'locals', True))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Exec(_body, _globals, _locals, _lineno, _col_offset)
 
 State.ast_type('Exec', 'stmt', ['body', 'globals', 'locals'])
 
@@ -1275,10 +1275,10 @@
     def from_object(space, w_node):
         names_w = space.unpackiterable(
                 get_field(space, w_node, 'names', False))
-        names = [space.str_w(w_item) for w_item in names_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Global(names, lineno, col_offset)
+        _names = [space.str_w(w_item) for w_item in names_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Global(_names, _lineno, _col_offset)
 
 State.ast_type('Global', 'stmt', ['names'])
 
@@ -1308,10 +1308,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Expr(value, lineno, col_offset)
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Expr(_value, _lineno, _col_offset)
 
 State.ast_type('Expr', 'stmt', ['value'])
 
@@ -1337,9 +1337,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Pass(lineno, col_offset)
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Pass(_lineno, _col_offset)
 
 State.ast_type('Pass', 'stmt', [])
 
@@ -1365,9 +1365,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Break(lineno, col_offset)
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Break(_lineno, _col_offset)
 
 State.ast_type('Break', 'stmt', [])
 
@@ -1393,9 +1393,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Continue(lineno, col_offset)
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Continue(_lineno, _col_offset)
 
 State.ast_type('Continue', 'stmt', [])
 
@@ -1493,13 +1493,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        op = boolop.from_object(space, get_field(space, w_node, 'op', False))
+        _op = boolop.from_object(space, get_field(space, w_node, 'op', False))
         values_w = space.unpackiterable(
                 get_field(space, w_node, 'values', False))
-        values = [expr.from_object(space, w_item) for w_item in values_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return BoolOp(op, values, lineno, col_offset)
+        _values = [expr.from_object(space, w_item) for w_item in values_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return BoolOp(_op, _values, _lineno, _col_offset)
 
 State.ast_type('BoolOp', 'expr', ['op', 'values'])
 
@@ -1536,12 +1536,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        left = expr.from_object(space, get_field(space, w_node, 'left', False))
-        op = operator.from_object(space, get_field(space, w_node, 'op', False))
-        right = expr.from_object(space, get_field(space, w_node, 'right', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return BinOp(left, op, right, lineno, col_offset)
+        _left = expr.from_object(space, get_field(space, w_node, 'left', False))
+        _op = operator.from_object(space, get_field(space, w_node, 'op', False))
+        _right = expr.from_object(space, get_field(space, w_node, 'right', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return BinOp(_left, _op, _right, _lineno, _col_offset)
 
 State.ast_type('BinOp', 'expr', ['left', 'op', 'right'])
 
@@ -1574,11 +1574,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        op = unaryop.from_object(space, get_field(space, w_node, 'op', False))
-        operand = expr.from_object(space, get_field(space, w_node, 'operand', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return UnaryOp(op, operand, lineno, col_offset)
+        _op = unaryop.from_object(space, get_field(space, w_node, 'op', False))
+        _operand = expr.from_object(space, get_field(space, w_node, 'operand', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return UnaryOp(_op, _operand, _lineno, _col_offset)
 
 State.ast_type('UnaryOp', 'expr', ['op', 'operand'])
 
@@ -1612,11 +1612,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        args = arguments.from_object(space, get_field(space, w_node, 'args', False))
-        body = expr.from_object(space, get_field(space, w_node, 'body', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Lambda(args, body, lineno, col_offset)
+        _args = arguments.from_object(space, get_field(space, w_node, 'args', False))
+        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Lambda(_args, _body, _lineno, _col_offset)
 
 State.ast_type('Lambda', 'expr', ['args', 'body'])
 
@@ -1654,12 +1654,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        test = expr.from_object(space, get_field(space, w_node, 'test', False))
-        body = expr.from_object(space, get_field(space, w_node, 'body', False))
-        orelse = expr.from_object(space, get_field(space, w_node, 'orelse', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return IfExp(test, body, orelse, lineno, col_offset)
+        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
+        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        _orelse = expr.from_object(space, get_field(space, w_node, 'orelse', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return IfExp(_test, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('IfExp', 'expr', ['test', 'body', 'orelse'])
 
@@ -1705,13 +1705,13 @@
     def from_object(space, w_node):
         keys_w = space.unpackiterable(
                 get_field(space, w_node, 'keys', False))
-        keys = [expr.from_object(space, w_item) for w_item in keys_w]
+        _keys = [expr.from_object(space, w_item) for w_item in keys_w]
         values_w = space.unpackiterable(
                 get_field(space, w_node, 'values', False))
-        values = [expr.from_object(space, w_item) for w_item in values_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Dict(keys, values, lineno, col_offset)
+        _values = [expr.from_object(space, w_item) for w_item in values_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Dict(_keys, _values, _lineno, _col_offset)
 
 State.ast_type('Dict', 'expr', ['keys', 'values'])
 
@@ -1748,10 +1748,10 @@
     def from_object(space, w_node):
         elts_w = space.unpackiterable(
                 get_field(space, w_node, 'elts', False))
-        elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Set(elts, lineno, col_offset)
+        _elts = [expr.from_object(space, w_item) for w_item in elts_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Set(_elts, _lineno, _col_offset)
 
 State.ast_type('Set', 'expr', ['elts'])
 
@@ -1790,13 +1790,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
+        _elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
-        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return ListComp(elt, generators, lineno, col_offset)
+        _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return ListComp(_elt, _generators, _lineno, _col_offset)
 
 State.ast_type('ListComp', 'expr', ['elt', 'generators'])
 
@@ -1835,13 +1835,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
+        _elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
-        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return SetComp(elt, generators, lineno, col_offset)
+        _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return SetComp(_elt, _generators, _lineno, _col_offset)
 
 State.ast_type('SetComp', 'expr', ['elt', 'generators'])
 
@@ -1884,14 +1884,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        key = expr.from_object(space, get_field(space, w_node, 'key', False))
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        _key = expr.from_object(space, get_field(space, w_node, 'key', False))
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
-        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return DictComp(key, value, generators, lineno, col_offset)
+        _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return DictComp(_key, _value, _generators, _lineno, _col_offset)
 
 State.ast_type('DictComp', 'expr', ['key', 'value', 'generators'])
 
@@ -1930,13 +1930,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
+        _elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
         generators_w = space.unpackiterable(
                 get_field(space, w_node, 'generators', False))
-        generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return GeneratorExp(elt, generators, lineno, col_offset)
+        _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return GeneratorExp(_elt, _generators, _lineno, _col_offset)
 
 State.ast_type('GeneratorExp', 'expr', ['elt', 'generators'])
 
@@ -1967,10 +1967,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, get_field(space, w_node, 'value', True))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Yield(value, lineno, col_offset)
+        _value = expr.from_object(space, get_field(space, w_node, 'value', True))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Yield(_value, _lineno, _col_offset)
 
 State.ast_type('Yield', 'expr', ['value'])
 
@@ -2016,16 +2016,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        left = expr.from_object(space, get_field(space, w_node, 'left', False))
+        _left = expr.from_object(space, get_field(space, w_node, 'left', False))
         ops_w = space.unpackiterable(
                 get_field(space, w_node, 'ops', False))
-        ops = [cmpop.from_object(space, w_item) for w_item in ops_w]
+        _ops = [cmpop.from_object(space, w_item) for w_item in ops_w]
         comparators_w = space.unpackiterable(
                 get_field(space, w_node, 'comparators', False))
-        comparators = [expr.from_object(space, w_item) for w_item in comparators_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Compare(left, ops, comparators, lineno, col_offset)
+        _comparators = [expr.from_object(space, w_item) for w_item in comparators_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Compare(_left, _ops, _comparators, _lineno, _col_offset)
 
 State.ast_type('Compare', 'expr', ['left', 'ops', 'comparators'])
 
@@ -2083,18 +2083,18 @@
 
     @staticmethod
     def from_object(space, w_node):
-        func = expr.from_object(space, get_field(space, w_node, 'func', False))
+        _func = expr.from_object(space, get_field(space, w_node, 'func', False))
         args_w = space.unpackiterable(
                 get_field(space, w_node, 'args', False))
-        args = [expr.from_object(space, w_item) for w_item in args_w]
+        _args = [expr.from_object(space, w_item) for w_item in args_w]
         keywords_w = space.unpackiterable(
                 get_field(space, w_node, 'keywords', False))
-        keywords = [keyword.from_object(space, w_item) for w_item in keywords_w]
-        starargs = expr.from_object(space, get_field(space, w_node, 'starargs', True))
-        kwargs = expr.from_object(space, get_field(space, w_node, 'kwargs', True))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Call(func, args, keywords, starargs, kwargs, lineno, col_offset)
+        _keywords = [keyword.from_object(space, w_item) for w_item in keywords_w]
+        _starargs = expr.from_object(space, get_field(space, w_node, 'starargs', True))
+        _kwargs = expr.from_object(space, get_field(space, w_node, 'kwargs', True))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Call(_func, _args, _keywords, _starargs, _kwargs, _lineno, _col_offset)
 
 State.ast_type('Call', 'expr', ['func', 'args', 'keywords', 'starargs', 'kwargs'])
 
@@ -2124,10 +2124,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Repr(value, lineno, col_offset)
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Repr(_value, _lineno, _col_offset)
 
 State.ast_type('Repr', 'expr', ['value'])
 
@@ -2156,10 +2156,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        n = get_field(space, w_node, 'n', False)
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Num(n, lineno, col_offset)
+        _n = get_field(space, w_node, 'n', False)
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Num(_n, _lineno, _col_offset)
 
 State.ast_type('Num', 'expr', ['n'])
 
@@ -2188,10 +2188,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        s = check_string(space, get_field(space, w_node, 's', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Str(s, lineno, col_offset)
+        _s = check_string(space, get_field(space, w_node, 's', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Str(_s, _lineno, _col_offset)
 
 State.ast_type('Str', 'expr', ['s'])
 
@@ -2227,12 +2227,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        attr = space.str_w(get_field(space, w_node, 'attr', False))
-        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Attribute(value, attr, ctx, lineno, col_offset)
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        _attr = space.str_w(get_field(space, w_node, 'attr', False))
+        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Attribute(_value, _attr, _ctx, _lineno, _col_offset)
 
 State.ast_type('Attribute', 'expr', ['value', 'attr', 'ctx'])
 
@@ -2269,12 +2269,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        slice = slice.from_object(space, get_field(space, w_node, 'slice', False))
-        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Subscript(value, slice, ctx, lineno, col_offset)
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        _slice = slice.from_object(space, get_field(space, w_node, 'slice', False))
+        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Subscript(_value, _slice, _ctx, _lineno, _col_offset)
 
 State.ast_type('Subscript', 'expr', ['value', 'slice', 'ctx'])
 
@@ -2306,11 +2306,11 @@
 
     @staticmethod
     def from_object(space, w_node):
-        id = space.str_w(get_field(space, w_node, 'id', False))
-        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Name(id, ctx, lineno, col_offset)
+        _id = space.str_w(get_field(space, w_node, 'id', False))
+        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Name(_id, _ctx, _lineno, _col_offset)
 
 State.ast_type('Name', 'expr', ['id', 'ctx'])
 
@@ -2350,11 +2350,11 @@
     def from_object(space, w_node):
         elts_w = space.unpackiterable(
                 get_field(space, w_node, 'elts', False))
-        elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return List(elts, ctx, lineno, col_offset)
+        _elts = [expr.from_object(space, w_item) for w_item in elts_w]
+        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return List(_elts, _ctx, _lineno, _col_offset)
 
 State.ast_type('List', 'expr', ['elts', 'ctx'])
 
@@ -2394,11 +2394,11 @@
     def from_object(space, w_node):
         elts_w = space.unpackiterable(
                 get_field(space, w_node, 'elts', False))
-        elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Tuple(elts, ctx, lineno, col_offset)
+        _elts = [expr.from_object(space, w_item) for w_item in elts_w]
+        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Tuple(_elts, _ctx, _lineno, _col_offset)
 
 State.ast_type('Tuple', 'expr', ['elts', 'ctx'])
 
@@ -2427,10 +2427,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = get_field(space, w_node, 'value', False)
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return Const(value, lineno, col_offset)
+        _value = get_field(space, w_node, 'value', False)
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return Const(_value, _lineno, _col_offset)
 
 State.ast_type('Const', 'expr', ['value'])
 
@@ -2568,10 +2568,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        lower = expr.from_object(space, get_field(space, w_node, 'lower', True))
-        upper = expr.from_object(space, get_field(space, w_node, 'upper', True))
-        step = expr.from_object(space, get_field(space, w_node, 'step', True))
-        return Slice(lower, upper, step)
+        _lower = expr.from_object(space, get_field(space, w_node, 'lower', True))
+        _upper = expr.from_object(space, get_field(space, w_node, 'upper', True))
+        _step = expr.from_object(space, get_field(space, w_node, 'step', True))
+        return Slice(_lower, _upper, _step)
 
 State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
 
@@ -2603,8 +2603,8 @@
     def from_object(space, w_node):
         dims_w = space.unpackiterable(
                 get_field(space, w_node, 'dims', False))
-        dims = [slice.from_object(space, w_item) for w_item in dims_w]
-        return ExtSlice(dims)
+        _dims = [slice.from_object(space, w_item) for w_item in dims_w]
+        return ExtSlice(_dims)
 
 State.ast_type('ExtSlice', 'slice', ['dims'])
 
@@ -2629,8 +2629,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        return Index(value)
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        return Index(_value)
 
 State.ast_type('Index', 'slice', ['value'])
 
@@ -2964,12 +2964,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        target = expr.from_object(space, get_field(space, w_node, 'target', False))
-        iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
+        _target = expr.from_object(space, get_field(space, w_node, 'target', False))
+        _iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
         ifs_w = space.unpackiterable(
                 get_field(space, w_node, 'ifs', False))
-        ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
-        return comprehension(target, iter, ifs)
+        _ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
+        return comprehension(_target, _iter, _ifs)
 
 State.ast_type('comprehension', 'AST', None)
 
@@ -3029,14 +3029,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        type = expr.from_object(space, get_field(space, w_node, 'type', True))
-        name = expr.from_object(space, get_field(space, w_node, 'name', True))
+        _type = expr.from_object(space, get_field(space, w_node, 'type', True))
+        _name = expr.from_object(space, get_field(space, w_node, 'name', True))
         body_w = space.unpackiterable(
                 get_field(space, w_node, 'body', False))
-        body = [stmt.from_object(space, w_item) for w_item in body_w]
-        lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
-        return ExceptHandler(type, name, body, lineno, col_offset)
+        _body = [stmt.from_object(space, w_item) for w_item in body_w]
+        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
+        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        return ExceptHandler(_type, _name, _body, _lineno, _col_offset)
 
 State.ast_type('ExceptHandler', 'excepthandler', ['type', 'name', 'body'])
 
@@ -3083,13 +3083,13 @@
     def from_object(space, w_node):
         args_w = space.unpackiterable(
                 get_field(space, w_node, 'args', False))
-        args = [expr.from_object(space, w_item) for w_item in args_w]
-        vararg = space.str_or_None_w(get_field(space, w_node, 'vararg', True))
-        kwarg = space.str_or_None_w(get_field(space, w_node, 'kwarg', True))
+        _args = [expr.from_object(space, w_item) for w_item in args_w]
+        _vararg = space.str_or_None_w(get_field(space, w_node, 'vararg', True))
+        _kwarg = space.str_or_None_w(get_field(space, w_node, 'kwarg', True))
         defaults_w = space.unpackiterable(
                 get_field(space, w_node, 'defaults', False))
-        defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
-        return arguments(args, vararg, kwarg, defaults)
+        _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
+        return arguments(_args, _vararg, _kwarg, _defaults)
 
 State.ast_type('arguments', 'AST', None)
 
@@ -3116,9 +3116,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        arg = space.str_w(get_field(space, w_node, 'arg', False))
-        value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        return keyword(arg, value)
+        _arg = space.str_w(get_field(space, w_node, 'arg', False))
+        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        return keyword(_arg, _value)
 
 State.ast_type('keyword', 'AST', None)
 
@@ -3144,9 +3144,9 @@
 
     @staticmethod
     def from_object(space, w_node):
-        name = space.str_w(get_field(space, w_node, 'name', False))
-        asname = space.str_or_None_w(get_field(space, w_node, 'asname', True))
-        return alias(name, asname)
+        _name = space.str_w(get_field(space, w_node, 'name', False))
+        _asname = space.str_or_None_w(get_field(space, w_node, 'asname', True))
+        return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', None)
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -126,7 +126,7 @@
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
         elif field.type.value in ("object", "string"):
             return value
-        elif field.type.value in ("identifier", "int"):
+        elif field.type.value in ("identifier", "int", "bool"):
             return "space.wrap(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
@@ -147,6 +147,8 @@
             return "space.str_w(%s)" % (value,)
         elif field.type.value in ("int",):
             return "space.int_w(%s)" % (value,)
+        elif field.type.value in ("bool",):
+            return "space.bool_w(%s)" % (value,)
         else:
             return "%s.from_object(space, %s)" % (field.type, value)
 
@@ -172,13 +174,13 @@
             lines.append("        get_field(space, w_node, '%s', %s))"
                          % (field.name, field.opt))
             value = self.get_value_extractor(field, "w_item")
-            lines.append("%s = [%s for w_item in %s_w]" %
+            lines.append("_%s = [%s for w_item in %s_w]" %
                          (field.name, value, field.name))
         else:
             value = self.get_value_extractor(
                 field,
                 "get_field(space, w_node, '%s', %s)" % (field.name, field.opt))
-            lines = ["%s = %s" % (field.name, value)]
+            lines = ["_%s = %s" % (field.name, value)]
         return lines
 
     def make_converters(self, fields, name, extras=None):
@@ -200,7 +202,7 @@
             for line in unwrapping_code:
                 self.emit(line, 2)
         self.emit("return %s(%s)" % (
-                name, ', '.join(str(field.name) for field in all_fields)), 2)
+                name, ', '.join("_%s" % (field.name,) for field in all_fields)), 2)
         self.emit("")
 
     def make_constructor(self, fields, node, extras=None, base=None):
@@ -469,7 +471,7 @@
     if args_w and len(args_w) != num_fields:
         if num_fields:
             raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes either 0 or %s positional arguments", w_self, num_fields)
+                "_ast.%T constructor takes either 0 or %d positional arguments", w_self, num_fields)
         else:
             raise operationerrfmt(space.w_TypeError,
                 "_ast.%T constructor takes 0 positional arguments", w_self)

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:18 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:18 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Do like CPython and check the
 presence of the fields before converting them.
Message-ID: <20130626212218.56CB11C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65025:7a553055220c
Date: 2013-06-26 23:05 +0200
http://bitbucket.org/pypy/pypy/changeset/7a553055220c/

Log:	Do like CPython and check the presence of the fields before
	converting them.

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -184,8 +184,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_body = get_field(space, w_node, 'body', False)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Module(_body)
 
@@ -217,8 +217,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_body = get_field(space, w_node, 'body', False)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Interactive(_body)
 
@@ -245,7 +245,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
+        w_body = get_field(space, w_node, 'body', False)
+        _body = expr.from_object(space, w_body)
         return Expression(_body)
 
 State.ast_type('Expression', 'mod', ['body'])
@@ -276,8 +277,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_body = get_field(space, w_node, 'body', False)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Suite(_body)
 
@@ -390,16 +391,20 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _name = space.str_w(get_field(space, w_node, 'name', False))
-        _args = arguments.from_object(space, get_field(space, w_node, 'args', False))
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_name = get_field(space, w_node, 'name', False)
+        w_args = get_field(space, w_node, 'args', False)
+        w_body = get_field(space, w_node, 'body', False)
+        w_decorator_list = get_field(space, w_node, 'decorator_list', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _name = space.str_w(w_name)
+        _args = arguments.from_object(space, w_args)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        decorator_list_w = space.unpackiterable(
-                get_field(space, w_node, 'decorator_list', False))
+        decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return FunctionDef(_name, _args, _body, _decorator_list, _lineno, _col_offset)
 
 State.ast_type('FunctionDef', 'stmt', ['name', 'args', 'body', 'decorator_list'])
@@ -456,18 +461,21 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _name = space.str_w(get_field(space, w_node, 'name', False))
-        bases_w = space.unpackiterable(
-                get_field(space, w_node, 'bases', False))
+        w_name = get_field(space, w_node, 'name', False)
+        w_bases = get_field(space, w_node, 'bases', False)
+        w_body = get_field(space, w_node, 'body', False)
+        w_decorator_list = get_field(space, w_node, 'decorator_list', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _name = space.str_w(w_name)
+        bases_w = space.unpackiterable(w_bases)
         _bases = [expr.from_object(space, w_item) for w_item in bases_w]
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        decorator_list_w = space.unpackiterable(
-                get_field(space, w_node, 'decorator_list', False))
+        decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return ClassDef(_name, _bases, _body, _decorator_list, _lineno, _col_offset)
 
 State.ast_type('ClassDef', 'stmt', ['name', 'bases', 'body', 'decorator_list'])
@@ -499,9 +507,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = expr.from_object(space, get_field(space, w_node, 'value', True))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_value = get_field(space, w_node, 'value', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Return(_value, _lineno, _col_offset)
 
 State.ast_type('Return', 'stmt', ['value'])
@@ -537,11 +548,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        targets_w = space.unpackiterable(
-                get_field(space, w_node, 'targets', False))
+        w_targets = get_field(space, w_node, 'targets', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        targets_w = space.unpackiterable(w_targets)
         _targets = [expr.from_object(space, w_item) for w_item in targets_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Delete(_targets, _lineno, _col_offset)
 
 State.ast_type('Delete', 'stmt', ['targets'])
@@ -581,12 +594,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        targets_w = space.unpackiterable(
-                get_field(space, w_node, 'targets', False))
+        w_targets = get_field(space, w_node, 'targets', False)
+        w_value = get_field(space, w_node, 'value', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        targets_w = space.unpackiterable(w_targets)
         _targets = [expr.from_object(space, w_item) for w_item in targets_w]
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _value = expr.from_object(space, w_value)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Assign(_targets, _value, _lineno, _col_offset)
 
 State.ast_type('Assign', 'stmt', ['targets', 'value'])
@@ -624,11 +640,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _target = expr.from_object(space, get_field(space, w_node, 'target', False))
-        _op = operator.from_object(space, get_field(space, w_node, 'op', False))
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_target = get_field(space, w_node, 'target', False)
+        w_op = get_field(space, w_node, 'op', False)
+        w_value = get_field(space, w_node, 'value', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _target = expr.from_object(space, w_target)
+        _op = operator.from_object(space, w_op)
+        _value = expr.from_object(space, w_value)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return AugAssign(_target, _op, _value, _lineno, _col_offset)
 
 State.ast_type('AugAssign', 'stmt', ['target', 'op', 'value'])
@@ -672,13 +693,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _dest = expr.from_object(space, get_field(space, w_node, 'dest', True))
-        values_w = space.unpackiterable(
-                get_field(space, w_node, 'values', False))
+        w_dest = get_field(space, w_node, 'dest', True)
+        w_values = get_field(space, w_node, 'values', False)
+        w_nl = get_field(space, w_node, 'nl', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _dest = expr.from_object(space, w_dest)
+        values_w = space.unpackiterable(w_values)
         _values = [expr.from_object(space, w_item) for w_item in values_w]
-        _nl = space.bool_w(get_field(space, w_node, 'nl', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _nl = space.bool_w(w_nl)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Print(_dest, _values, _nl, _lineno, _col_offset)
 
 State.ast_type('Print', 'stmt', ['dest', 'values', 'nl'])
@@ -731,16 +756,20 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _target = expr.from_object(space, get_field(space, w_node, 'target', False))
-        _iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_target = get_field(space, w_node, 'target', False)
+        w_iter = get_field(space, w_node, 'iter', False)
+        w_body = get_field(space, w_node, 'body', False)
+        w_orelse = get_field(space, w_node, 'orelse', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _target = expr.from_object(space, w_target)
+        _iter = expr.from_object(space, w_iter)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        orelse_w = space.unpackiterable(
-                get_field(space, w_node, 'orelse', False))
+        orelse_w = space.unpackiterable(w_orelse)
         _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return For(_target, _iter, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('For', 'stmt', ['target', 'iter', 'body', 'orelse'])
@@ -789,15 +818,18 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_test = get_field(space, w_node, 'test', False)
+        w_body = get_field(space, w_node, 'body', False)
+        w_orelse = get_field(space, w_node, 'orelse', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _test = expr.from_object(space, w_test)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        orelse_w = space.unpackiterable(
-                get_field(space, w_node, 'orelse', False))
+        orelse_w = space.unpackiterable(w_orelse)
         _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return While(_test, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('While', 'stmt', ['test', 'body', 'orelse'])
@@ -846,15 +878,18 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_test = get_field(space, w_node, 'test', False)
+        w_body = get_field(space, w_node, 'body', False)
+        w_orelse = get_field(space, w_node, 'orelse', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _test = expr.from_object(space, w_test)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        orelse_w = space.unpackiterable(
-                get_field(space, w_node, 'orelse', False))
+        orelse_w = space.unpackiterable(w_orelse)
         _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return If(_test, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('If', 'stmt', ['test', 'body', 'orelse'])
@@ -899,13 +934,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _context_expr = expr.from_object(space, get_field(space, w_node, 'context_expr', False))
-        _optional_vars = expr.from_object(space, get_field(space, w_node, 'optional_vars', True))
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_context_expr = get_field(space, w_node, 'context_expr', False)
+        w_optional_vars = get_field(space, w_node, 'optional_vars', True)
+        w_body = get_field(space, w_node, 'body', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _context_expr = expr.from_object(space, w_context_expr)
+        _optional_vars = expr.from_object(space, w_optional_vars)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return With(_context_expr, _optional_vars, _body, _lineno, _col_offset)
 
 State.ast_type('With', 'stmt', ['context_expr', 'optional_vars', 'body'])
@@ -947,11 +986,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _type = expr.from_object(space, get_field(space, w_node, 'type', True))
-        _inst = expr.from_object(space, get_field(space, w_node, 'inst', True))
-        _tback = expr.from_object(space, get_field(space, w_node, 'tback', True))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_type = get_field(space, w_node, 'type', True)
+        w_inst = get_field(space, w_node, 'inst', True)
+        w_tback = get_field(space, w_node, 'tback', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _type = expr.from_object(space, w_type)
+        _inst = expr.from_object(space, w_inst)
+        _tback = expr.from_object(space, w_tback)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Raise(_type, _inst, _tback, _lineno, _col_offset)
 
 State.ast_type('Raise', 'stmt', ['type', 'inst', 'tback'])
@@ -1005,17 +1049,19 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_body = get_field(space, w_node, 'body', False)
+        w_handlers = get_field(space, w_node, 'handlers', False)
+        w_orelse = get_field(space, w_node, 'orelse', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        handlers_w = space.unpackiterable(
-                get_field(space, w_node, 'handlers', False))
+        handlers_w = space.unpackiterable(w_handlers)
         _handlers = [excepthandler.from_object(space, w_item) for w_item in handlers_w]
-        orelse_w = space.unpackiterable(
-                get_field(space, w_node, 'orelse', False))
+        orelse_w = space.unpackiterable(w_orelse)
         _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return TryExcept(_body, _handlers, _orelse, _lineno, _col_offset)
 
 State.ast_type('TryExcept', 'stmt', ['body', 'handlers', 'orelse'])
@@ -1060,14 +1106,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_body = get_field(space, w_node, 'body', False)
+        w_finalbody = get_field(space, w_node, 'finalbody', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        finalbody_w = space.unpackiterable(
-                get_field(space, w_node, 'finalbody', False))
+        finalbody_w = space.unpackiterable(w_finalbody)
         _finalbody = [stmt.from_object(space, w_item) for w_item in finalbody_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return TryFinally(_body, _finalbody, _lineno, _col_offset)
 
 State.ast_type('TryFinally', 'stmt', ['body', 'finalbody'])
@@ -1103,10 +1151,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
-        _msg = expr.from_object(space, get_field(space, w_node, 'msg', True))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_test = get_field(space, w_node, 'test', False)
+        w_msg = get_field(space, w_node, 'msg', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _test = expr.from_object(space, w_test)
+        _msg = expr.from_object(space, w_msg)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Assert(_test, _msg, _lineno, _col_offset)
 
 State.ast_type('Assert', 'stmt', ['test', 'msg'])
@@ -1142,11 +1194,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        names_w = space.unpackiterable(
-                get_field(space, w_node, 'names', False))
+        w_names = get_field(space, w_node, 'names', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Import(_names, _lineno, _col_offset)
 
 State.ast_type('Import', 'stmt', ['names'])
@@ -1188,13 +1242,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _module = space.str_or_None_w(get_field(space, w_node, 'module', True))
-        names_w = space.unpackiterable(
-                get_field(space, w_node, 'names', False))
+        w_module = get_field(space, w_node, 'module', True)
+        w_names = get_field(space, w_node, 'names', False)
+        w_level = get_field(space, w_node, 'level', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _module = space.str_or_None_w(w_module)
+        names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
-        _level = space.int_w(get_field(space, w_node, 'level', True))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _level = space.int_w(w_level)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return ImportFrom(_module, _names, _level, _lineno, _col_offset)
 
 State.ast_type('ImportFrom', 'stmt', ['module', 'names', 'level'])
@@ -1235,11 +1293,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
-        _globals = expr.from_object(space, get_field(space, w_node, 'globals', True))
-        _locals = expr.from_object(space, get_field(space, w_node, 'locals', True))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_body = get_field(space, w_node, 'body', False)
+        w_globals = get_field(space, w_node, 'globals', True)
+        w_locals = get_field(space, w_node, 'locals', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _body = expr.from_object(space, w_body)
+        _globals = expr.from_object(space, w_globals)
+        _locals = expr.from_object(space, w_locals)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Exec(_body, _globals, _locals, _lineno, _col_offset)
 
 State.ast_type('Exec', 'stmt', ['body', 'globals', 'locals'])
@@ -1273,11 +1336,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        names_w = space.unpackiterable(
-                get_field(space, w_node, 'names', False))
+        w_names = get_field(space, w_node, 'names', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        names_w = space.unpackiterable(w_names)
         _names = [space.str_w(w_item) for w_item in names_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
 
 State.ast_type('Global', 'stmt', ['names'])
@@ -1308,9 +1373,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_value = get_field(space, w_node, 'value', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Expr(_value, _lineno, _col_offset)
 
 State.ast_type('Expr', 'stmt', ['value'])
@@ -1337,8 +1405,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Pass(_lineno, _col_offset)
 
 State.ast_type('Pass', 'stmt', [])
@@ -1365,8 +1435,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Break(_lineno, _col_offset)
 
 State.ast_type('Break', 'stmt', [])
@@ -1393,8 +1465,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Continue(_lineno, _col_offset)
 
 State.ast_type('Continue', 'stmt', [])
@@ -1493,12 +1567,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _op = boolop.from_object(space, get_field(space, w_node, 'op', False))
-        values_w = space.unpackiterable(
-                get_field(space, w_node, 'values', False))
+        w_op = get_field(space, w_node, 'op', False)
+        w_values = get_field(space, w_node, 'values', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _op = boolop.from_object(space, w_op)
+        values_w = space.unpackiterable(w_values)
         _values = [expr.from_object(space, w_item) for w_item in values_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return BoolOp(_op, _values, _lineno, _col_offset)
 
 State.ast_type('BoolOp', 'expr', ['op', 'values'])
@@ -1536,11 +1613,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _left = expr.from_object(space, get_field(space, w_node, 'left', False))
-        _op = operator.from_object(space, get_field(space, w_node, 'op', False))
-        _right = expr.from_object(space, get_field(space, w_node, 'right', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_left = get_field(space, w_node, 'left', False)
+        w_op = get_field(space, w_node, 'op', False)
+        w_right = get_field(space, w_node, 'right', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _left = expr.from_object(space, w_left)
+        _op = operator.from_object(space, w_op)
+        _right = expr.from_object(space, w_right)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return BinOp(_left, _op, _right, _lineno, _col_offset)
 
 State.ast_type('BinOp', 'expr', ['left', 'op', 'right'])
@@ -1574,10 +1656,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _op = unaryop.from_object(space, get_field(space, w_node, 'op', False))
-        _operand = expr.from_object(space, get_field(space, w_node, 'operand', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_op = get_field(space, w_node, 'op', False)
+        w_operand = get_field(space, w_node, 'operand', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _op = unaryop.from_object(space, w_op)
+        _operand = expr.from_object(space, w_operand)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return UnaryOp(_op, _operand, _lineno, _col_offset)
 
 State.ast_type('UnaryOp', 'expr', ['op', 'operand'])
@@ -1612,10 +1698,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _args = arguments.from_object(space, get_field(space, w_node, 'args', False))
-        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_args = get_field(space, w_node, 'args', False)
+        w_body = get_field(space, w_node, 'body', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _args = arguments.from_object(space, w_args)
+        _body = expr.from_object(space, w_body)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Lambda(_args, _body, _lineno, _col_offset)
 
 State.ast_type('Lambda', 'expr', ['args', 'body'])
@@ -1654,11 +1744,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _test = expr.from_object(space, get_field(space, w_node, 'test', False))
-        _body = expr.from_object(space, get_field(space, w_node, 'body', False))
-        _orelse = expr.from_object(space, get_field(space, w_node, 'orelse', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_test = get_field(space, w_node, 'test', False)
+        w_body = get_field(space, w_node, 'body', False)
+        w_orelse = get_field(space, w_node, 'orelse', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _test = expr.from_object(space, w_test)
+        _body = expr.from_object(space, w_body)
+        _orelse = expr.from_object(space, w_orelse)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return IfExp(_test, _body, _orelse, _lineno, _col_offset)
 
 State.ast_type('IfExp', 'expr', ['test', 'body', 'orelse'])
@@ -1703,14 +1798,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        keys_w = space.unpackiterable(
-                get_field(space, w_node, 'keys', False))
+        w_keys = get_field(space, w_node, 'keys', False)
+        w_values = get_field(space, w_node, 'values', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        keys_w = space.unpackiterable(w_keys)
         _keys = [expr.from_object(space, w_item) for w_item in keys_w]
-        values_w = space.unpackiterable(
-                get_field(space, w_node, 'values', False))
+        values_w = space.unpackiterable(w_values)
         _values = [expr.from_object(space, w_item) for w_item in values_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Dict(_keys, _values, _lineno, _col_offset)
 
 State.ast_type('Dict', 'expr', ['keys', 'values'])
@@ -1746,11 +1843,13 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elts_w = space.unpackiterable(
-                get_field(space, w_node, 'elts', False))
+        w_elts = get_field(space, w_node, 'elts', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        elts_w = space.unpackiterable(w_elts)
         _elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Set(_elts, _lineno, _col_offset)
 
 State.ast_type('Set', 'expr', ['elts'])
@@ -1790,12 +1889,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
-        generators_w = space.unpackiterable(
-                get_field(space, w_node, 'generators', False))
+        w_elt = get_field(space, w_node, 'elt', False)
+        w_generators = get_field(space, w_node, 'generators', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _elt = expr.from_object(space, w_elt)
+        generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return ListComp(_elt, _generators, _lineno, _col_offset)
 
 State.ast_type('ListComp', 'expr', ['elt', 'generators'])
@@ -1835,12 +1937,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
-        generators_w = space.unpackiterable(
-                get_field(space, w_node, 'generators', False))
+        w_elt = get_field(space, w_node, 'elt', False)
+        w_generators = get_field(space, w_node, 'generators', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _elt = expr.from_object(space, w_elt)
+        generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return SetComp(_elt, _generators, _lineno, _col_offset)
 
 State.ast_type('SetComp', 'expr', ['elt', 'generators'])
@@ -1884,13 +1989,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _key = expr.from_object(space, get_field(space, w_node, 'key', False))
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        generators_w = space.unpackiterable(
-                get_field(space, w_node, 'generators', False))
+        w_key = get_field(space, w_node, 'key', False)
+        w_value = get_field(space, w_node, 'value', False)
+        w_generators = get_field(space, w_node, 'generators', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _key = expr.from_object(space, w_key)
+        _value = expr.from_object(space, w_value)
+        generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return DictComp(_key, _value, _generators, _lineno, _col_offset)
 
 State.ast_type('DictComp', 'expr', ['key', 'value', 'generators'])
@@ -1930,12 +2039,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _elt = expr.from_object(space, get_field(space, w_node, 'elt', False))
-        generators_w = space.unpackiterable(
-                get_field(space, w_node, 'generators', False))
+        w_elt = get_field(space, w_node, 'elt', False)
+        w_generators = get_field(space, w_node, 'generators', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _elt = expr.from_object(space, w_elt)
+        generators_w = space.unpackiterable(w_generators)
         _generators = [comprehension.from_object(space, w_item) for w_item in generators_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return GeneratorExp(_elt, _generators, _lineno, _col_offset)
 
 State.ast_type('GeneratorExp', 'expr', ['elt', 'generators'])
@@ -1967,9 +2079,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = expr.from_object(space, get_field(space, w_node, 'value', True))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_value = get_field(space, w_node, 'value', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Yield(_value, _lineno, _col_offset)
 
 State.ast_type('Yield', 'expr', ['value'])
@@ -2016,15 +2131,18 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _left = expr.from_object(space, get_field(space, w_node, 'left', False))
-        ops_w = space.unpackiterable(
-                get_field(space, w_node, 'ops', False))
+        w_left = get_field(space, w_node, 'left', False)
+        w_ops = get_field(space, w_node, 'ops', False)
+        w_comparators = get_field(space, w_node, 'comparators', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _left = expr.from_object(space, w_left)
+        ops_w = space.unpackiterable(w_ops)
         _ops = [cmpop.from_object(space, w_item) for w_item in ops_w]
-        comparators_w = space.unpackiterable(
-                get_field(space, w_node, 'comparators', False))
+        comparators_w = space.unpackiterable(w_comparators)
         _comparators = [expr.from_object(space, w_item) for w_item in comparators_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Compare(_left, _ops, _comparators, _lineno, _col_offset)
 
 State.ast_type('Compare', 'expr', ['left', 'ops', 'comparators'])
@@ -2083,17 +2201,22 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _func = expr.from_object(space, get_field(space, w_node, 'func', False))
-        args_w = space.unpackiterable(
-                get_field(space, w_node, 'args', False))
+        w_func = get_field(space, w_node, 'func', False)
+        w_args = get_field(space, w_node, 'args', False)
+        w_keywords = get_field(space, w_node, 'keywords', False)
+        w_starargs = get_field(space, w_node, 'starargs', True)
+        w_kwargs = get_field(space, w_node, 'kwargs', True)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _func = expr.from_object(space, w_func)
+        args_w = space.unpackiterable(w_args)
         _args = [expr.from_object(space, w_item) for w_item in args_w]
-        keywords_w = space.unpackiterable(
-                get_field(space, w_node, 'keywords', False))
+        keywords_w = space.unpackiterable(w_keywords)
         _keywords = [keyword.from_object(space, w_item) for w_item in keywords_w]
-        _starargs = expr.from_object(space, get_field(space, w_node, 'starargs', True))
-        _kwargs = expr.from_object(space, get_field(space, w_node, 'kwargs', True))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _starargs = expr.from_object(space, w_starargs)
+        _kwargs = expr.from_object(space, w_kwargs)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Call(_func, _args, _keywords, _starargs, _kwargs, _lineno, _col_offset)
 
 State.ast_type('Call', 'expr', ['func', 'args', 'keywords', 'starargs', 'kwargs'])
@@ -2124,9 +2247,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_value = get_field(space, w_node, 'value', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Repr(_value, _lineno, _col_offset)
 
 State.ast_type('Repr', 'expr', ['value'])
@@ -2156,9 +2282,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _n = get_field(space, w_node, 'n', False)
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_n = get_field(space, w_node, 'n', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _n = w_n
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Num(_n, _lineno, _col_offset)
 
 State.ast_type('Num', 'expr', ['n'])
@@ -2188,9 +2317,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _s = check_string(space, get_field(space, w_node, 's', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_s = get_field(space, w_node, 's', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _s = check_string(space, w_s)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Str(_s, _lineno, _col_offset)
 
 State.ast_type('Str', 'expr', ['s'])
@@ -2227,11 +2359,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        _attr = space.str_w(get_field(space, w_node, 'attr', False))
-        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_value = get_field(space, w_node, 'value', False)
+        w_attr = get_field(space, w_node, 'attr', False)
+        w_ctx = get_field(space, w_node, 'ctx', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        _attr = space.str_w(w_attr)
+        _ctx = expr_context.from_object(space, w_ctx)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Attribute(_value, _attr, _ctx, _lineno, _col_offset)
 
 State.ast_type('Attribute', 'expr', ['value', 'attr', 'ctx'])
@@ -2269,11 +2406,16 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
-        _slice = slice.from_object(space, get_field(space, w_node, 'slice', False))
-        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_value = get_field(space, w_node, 'value', False)
+        w_slice = get_field(space, w_node, 'slice', False)
+        w_ctx = get_field(space, w_node, 'ctx', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = expr.from_object(space, w_value)
+        _slice = slice.from_object(space, w_slice)
+        _ctx = expr_context.from_object(space, w_ctx)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Subscript(_value, _slice, _ctx, _lineno, _col_offset)
 
 State.ast_type('Subscript', 'expr', ['value', 'slice', 'ctx'])
@@ -2306,10 +2448,14 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _id = space.str_w(get_field(space, w_node, 'id', False))
-        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_id = get_field(space, w_node, 'id', False)
+        w_ctx = get_field(space, w_node, 'ctx', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _id = space.str_w(w_id)
+        _ctx = expr_context.from_object(space, w_ctx)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Name(_id, _ctx, _lineno, _col_offset)
 
 State.ast_type('Name', 'expr', ['id', 'ctx'])
@@ -2348,12 +2494,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elts_w = space.unpackiterable(
-                get_field(space, w_node, 'elts', False))
+        w_elts = get_field(space, w_node, 'elts', False)
+        w_ctx = get_field(space, w_node, 'ctx', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        elts_w = space.unpackiterable(w_elts)
         _elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _ctx = expr_context.from_object(space, w_ctx)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return List(_elts, _ctx, _lineno, _col_offset)
 
 State.ast_type('List', 'expr', ['elts', 'ctx'])
@@ -2392,12 +2541,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        elts_w = space.unpackiterable(
-                get_field(space, w_node, 'elts', False))
+        w_elts = get_field(space, w_node, 'elts', False)
+        w_ctx = get_field(space, w_node, 'ctx', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        elts_w = space.unpackiterable(w_elts)
         _elts = [expr.from_object(space, w_item) for w_item in elts_w]
-        _ctx = expr_context.from_object(space, get_field(space, w_node, 'ctx', False))
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _ctx = expr_context.from_object(space, w_ctx)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Tuple(_elts, _ctx, _lineno, _col_offset)
 
 State.ast_type('Tuple', 'expr', ['elts', 'ctx'])
@@ -2427,9 +2579,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = get_field(space, w_node, 'value', False)
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        w_value = get_field(space, w_node, 'value', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = w_value
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return Const(_value, _lineno, _col_offset)
 
 State.ast_type('Const', 'expr', ['value'])
@@ -2568,9 +2723,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _lower = expr.from_object(space, get_field(space, w_node, 'lower', True))
-        _upper = expr.from_object(space, get_field(space, w_node, 'upper', True))
-        _step = expr.from_object(space, get_field(space, w_node, 'step', True))
+        w_lower = get_field(space, w_node, 'lower', True)
+        w_upper = get_field(space, w_node, 'upper', True)
+        w_step = get_field(space, w_node, 'step', True)
+        _lower = expr.from_object(space, w_lower)
+        _upper = expr.from_object(space, w_upper)
+        _step = expr.from_object(space, w_step)
         return Slice(_lower, _upper, _step)
 
 State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
@@ -2601,8 +2759,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        dims_w = space.unpackiterable(
-                get_field(space, w_node, 'dims', False))
+        w_dims = get_field(space, w_node, 'dims', False)
+        dims_w = space.unpackiterable(w_dims)
         _dims = [slice.from_object(space, w_item) for w_item in dims_w]
         return ExtSlice(_dims)
 
@@ -2629,7 +2787,8 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        w_value = get_field(space, w_node, 'value', False)
+        _value = expr.from_object(space, w_value)
         return Index(_value)
 
 State.ast_type('Index', 'slice', ['value'])
@@ -2964,10 +3123,12 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _target = expr.from_object(space, get_field(space, w_node, 'target', False))
-        _iter = expr.from_object(space, get_field(space, w_node, 'iter', False))
-        ifs_w = space.unpackiterable(
-                get_field(space, w_node, 'ifs', False))
+        w_target = get_field(space, w_node, 'target', False)
+        w_iter = get_field(space, w_node, 'iter', False)
+        w_ifs = get_field(space, w_node, 'ifs', False)
+        _target = expr.from_object(space, w_target)
+        _iter = expr.from_object(space, w_iter)
+        ifs_w = space.unpackiterable(w_ifs)
         _ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(_target, _iter, _ifs)
 
@@ -3029,13 +3190,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _type = expr.from_object(space, get_field(space, w_node, 'type', True))
-        _name = expr.from_object(space, get_field(space, w_node, 'name', True))
-        body_w = space.unpackiterable(
-                get_field(space, w_node, 'body', False))
+        w_type = get_field(space, w_node, 'type', True)
+        w_name = get_field(space, w_node, 'name', True)
+        w_body = get_field(space, w_node, 'body', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _type = expr.from_object(space, w_type)
+        _name = expr.from_object(space, w_name)
+        body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
-        _lineno = space.int_w(get_field(space, w_node, 'lineno', False))
-        _col_offset = space.int_w(get_field(space, w_node, 'col_offset', False))
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
         return ExceptHandler(_type, _name, _body, _lineno, _col_offset)
 
 State.ast_type('ExceptHandler', 'excepthandler', ['type', 'name', 'body'])
@@ -3081,13 +3246,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        args_w = space.unpackiterable(
-                get_field(space, w_node, 'args', False))
+        w_args = get_field(space, w_node, 'args', False)
+        w_vararg = get_field(space, w_node, 'vararg', True)
+        w_kwarg = get_field(space, w_node, 'kwarg', True)
+        w_defaults = get_field(space, w_node, 'defaults', False)
+        args_w = space.unpackiterable(w_args)
         _args = [expr.from_object(space, w_item) for w_item in args_w]
-        _vararg = space.str_or_None_w(get_field(space, w_node, 'vararg', True))
-        _kwarg = space.str_or_None_w(get_field(space, w_node, 'kwarg', True))
-        defaults_w = space.unpackiterable(
-                get_field(space, w_node, 'defaults', False))
+        _vararg = space.str_or_None_w(w_vararg)
+        _kwarg = space.str_or_None_w(w_kwarg)
+        defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwarg, _defaults)
 
@@ -3116,8 +3283,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _arg = space.str_w(get_field(space, w_node, 'arg', False))
-        _value = expr.from_object(space, get_field(space, w_node, 'value', False))
+        w_arg = get_field(space, w_node, 'arg', False)
+        w_value = get_field(space, w_node, 'value', False)
+        _arg = space.str_w(w_arg)
+        _value = expr.from_object(space, w_value)
         return keyword(_arg, _value)
 
 State.ast_type('keyword', 'AST', None)
@@ -3144,8 +3313,10 @@
 
     @staticmethod
     def from_object(space, w_node):
-        _name = space.str_w(get_field(space, w_node, 'name', False))
-        _asname = space.str_or_None_w(get_field(space, w_node, 'asname', True))
+        w_name = get_field(space, w_node, 'name', False)
+        w_asname = get_field(space, w_node, 'asname', True)
+        _name = space.str_w(w_name)
+        _asname = space.str_or_None_w(w_asname)
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', None)
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -170,16 +170,13 @@
     def get_field_extractor(self, field):
         if field.seq:
             lines = []
-            lines.append("%s_w = space.unpackiterable(" % (field.name,))
-            lines.append("        get_field(space, w_node, '%s', %s))"
-                         % (field.name, field.opt))
+            lines.append("%s_w = space.unpackiterable(w_%s)" %
+                         (field.name, field.name))
             value = self.get_value_extractor(field, "w_item")
             lines.append("_%s = [%s for w_item in %s_w]" %
                          (field.name, value, field.name))
         else:
-            value = self.get_value_extractor(
-                field,
-                "get_field(space, w_node, '%s', %s)" % (field.name, field.opt))
+            value = self.get_value_extractor(field, "w_%s" % (field.name,))
             lines = ["_%s = %s" % (field.name, value)]
         return lines
 
@@ -198,6 +195,9 @@
         self.emit("@staticmethod", 1)
         self.emit("def from_object(space, w_node):", 1)
         for field in all_fields:
+            self.emit("w_%s = get_field(space, w_node, '%s', %s)" % (
+                    field.name, field.name, field.opt), 2)
+        for field in all_fields:
             unwrapping_code = self.get_field_extractor(field)
             for line in unwrapping_code:
                 self.emit(line, 2)

From noreply at buildbot.pypy.org  Wed Jun 26 23:22:19 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:22:19 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Fix last failures
Message-ID: <20130626212219.86F761C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65026:0ae5aec118b1
Date: 2013-06-26 23:21 +0200
http://bitbucket.org/pypy/pypy/changeset/0ae5aec118b1/

Log:	Fix last failures

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -87,12 +87,15 @@
                                space.wrap("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
-        if num_fields:
+        if num_fields == 0:
             raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes either 0 or %d positional arguments", w_self, num_fields)
+                "%T constructor takes 0 positional arguments", w_self)
+        elif num_fields == 1:
+            raise operationerrfmt(space.w_TypeError,
+                "%T constructor takes either 0 or %d positional argument", w_self, num_fields)
         else:
             raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes 0 positional arguments", w_self)
+                "%T constructor takes either 0 or %d positional arguments", w_self, num_fields)
     if args_w:
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
@@ -100,10 +103,9 @@
         space.setattr(w_self, space.wrap(field), w_value)
 
 
-W_AST.typedef = typedef.TypeDef("AST",
+W_AST.typedef = typedef.TypeDef("_ast.AST",
     _fields=_FieldsWrapper([]),
     _attributes=_FieldsWrapper([]),
-    __module__='_ast',
     __reduce__=interp2app(W_AST.reduce_w),
     __setstate__=interp2app(W_AST.setstate_w),
     __dict__ = typedef.GetSetProperty(typedef.descr_get_dict,
@@ -397,7 +399,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.str_w(w_name)
+        _name = space.realstr_w(w_name)
         _args = arguments.from_object(space, w_args)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
@@ -467,7 +469,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.str_w(w_name)
+        _name = space.realstr_w(w_name)
         bases_w = space.unpackiterable(w_bases)
         _bases = [expr.from_object(space, w_item) for w_item in bases_w]
         body_w = space.unpackiterable(w_body)
@@ -1340,7 +1342,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         names_w = space.unpackiterable(w_names)
-        _names = [space.str_w(w_item) for w_item in names_w]
+        _names = [space.realstr_w(w_item) for w_item in names_w]
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
@@ -2365,7 +2367,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _value = expr.from_object(space, w_value)
-        _attr = space.str_w(w_attr)
+        _attr = space.realstr_w(w_attr)
         _ctx = expr_context.from_object(space, w_ctx)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
@@ -2452,7 +2454,7 @@
         w_ctx = get_field(space, w_node, 'ctx', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _id = space.str_w(w_id)
+        _id = space.realstr_w(w_id)
         _ctx = expr_context.from_object(space, w_ctx)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
@@ -3285,7 +3287,7 @@
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', False)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.str_w(w_arg)
+        _arg = space.realstr_w(w_arg)
         _value = expr.from_object(space, w_value)
         return keyword(_arg, _value)
 
@@ -3315,7 +3317,7 @@
     def from_object(space, w_node):
         w_name = get_field(space, w_node, 'name', False)
         w_asname = get_field(space, w_node, 'asname', True)
-        _name = space.str_w(w_name)
+        _name = space.realstr_w(w_name)
         _asname = space.str_or_None_w(w_asname)
         return alias(_name, _asname)
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -144,7 +144,7 @@
         elif field.type.value in ("identifier",):
             if field.opt:
                 return "space.str_or_None_w(%s)" % (value,)
-            return "space.str_w(%s)" % (value,)
+            return "space.realstr_w(%s)" % (value,)
         elif field.type.value in ("int",):
             return "space.int_w(%s)" % (value,)
         elif field.type.value in ("bool",):
@@ -469,12 +469,15 @@
                                space.wrap("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
-        if num_fields:
+        if num_fields == 0:
             raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes either 0 or %d positional arguments", w_self, num_fields)
+                "%T constructor takes 0 positional arguments", w_self)
+        elif num_fields == 1:
+            raise operationerrfmt(space.w_TypeError,
+                "%T constructor takes either 0 or %d positional argument", w_self, num_fields)
         else:
             raise operationerrfmt(space.w_TypeError,
-                "_ast.%T constructor takes 0 positional arguments", w_self)
+                "%T constructor takes either 0 or %d positional arguments", w_self, num_fields)
     if args_w:
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
@@ -482,10 +485,9 @@
         space.setattr(w_self, space.wrap(field), w_value)
 
 
-W_AST.typedef = typedef.TypeDef("AST",
+W_AST.typedef = typedef.TypeDef("_ast.AST",
     _fields=_FieldsWrapper([]),
     _attributes=_FieldsWrapper([]),
-    __module__='_ast',
     __reduce__=interp2app(W_AST.reduce_w),
     __setstate__=interp2app(W_AST.setstate_w),
     __dict__ = typedef.GetSetProperty(typedef.descr_get_dict,

From noreply at buildbot.pypy.org  Wed Jun 26 23:34:14 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 26 Jun 2013 23:34:14 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Remove bad import
Message-ID: <20130626213414.B31211C0325@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65027:308470eb0d26
Date: 2013-06-26 23:33 +0200
http://bitbucket.org/pypy/pypy/changeset/308470eb0d26/

Log:	Remove bad import

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -5,7 +5,6 @@
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.astcompiler import consts, ast
-from pypy.module._ast import interp_ast
 from pypy.interpreter.gateway import unwrap_spec
 
 

From noreply at buildbot.pypy.org  Thu Jun 27 00:15:34 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Jun 2013 00:15:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Random style cleanups to rbigint.
Message-ID: <20130626221534.93A701C0397@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r65028:8a8a1b0c08c8
Date: 2013-06-26 15:13 -0700
http://bitbucket.org/pypy/pypy/changeset/8a8a1b0c08c8/

Log:	Random style cleanups to rbigint.

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -61,7 +61,7 @@
     KARATSUBA_CUTOFF = 19
 else:
     KARATSUBA_CUTOFF = 38
-    
+
 KARATSUBA_SQUARE_CUTOFF = 2 * KARATSUBA_CUTOFF
 
 # For exponentiation, use the binary left-to-right algorithm
@@ -85,38 +85,41 @@
 
 def _load_unsigned_digit(x):
     return rffi.cast(UNSIGNED_TYPE, x)
-        
+
 _load_unsigned_digit._always_inline_ = True
 
 NULLDIGIT = _store_digit(0)
-ONEDIGIT  = _store_digit(1)
+ONEDIGIT = _store_digit(1)
 
 def _check_digits(l):
     for x in l:
         assert type(x) is type(NULLDIGIT)
         assert UDIGIT_MASK(x) & MASK == UDIGIT_MASK(x)
-            
+
 class InvalidEndiannessError(Exception):
     pass
 
 class InvalidSignednessError(Exception):
     pass
 
+
 class Entry(extregistry.ExtRegistryEntry):
     _about_ = _check_digits
+
     def compute_result_annotation(self, s_list):
         from rpython.annotator import model as annmodel
         assert isinstance(s_list, annmodel.SomeList)
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
+
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
 
+
 class rbigint(object):
     """This is a reimplementation of longs using a list of digits."""
     _immutable_ = True
     _immutable_fields_ = ["_digits"]
-    
 
     def __init__(self, digits=[NULLDIGIT], sign=0, size=0):
         if not we_are_translated():
@@ -164,7 +167,7 @@
     def numdigits(self):
         return self.size
     numdigits._always_inline_ = True
-    
+
     @staticmethod
     @jit.elidable
     def fromint(intval):
@@ -180,15 +183,14 @@
             ival = r_uint(intval)
         else:
             return NULLRBIGINT
-        
+
         carry = ival >> SHIFT
         if carry:
             return rbigint([_store_digit(ival & MASK),
                 _store_digit(carry)], sign, 2)
         else:
             return rbigint([_store_digit(ival & MASK)], sign, 1)
-            
-        
+
     @staticmethod
     @jit.elidable
     def frombool(b):
@@ -319,7 +321,7 @@
                 # Avoid bogus 0's
                 s = d ^ MASK if self.sign == -1 else d
                 while s:
-                    s >>=1
+                    s >>= 1
                     accumbits += 1
             else:
                 accumbits += SHIFT
@@ -340,7 +342,7 @@
 
             if self.sign == -1:
                 # Add a sign bit
-                accum |= (~_widen_digit(0)) << accumbits;
+                accum |= (~_widen_digit(0)) << accumbits
 
             result.append(chr(accum & 0xFF))
 
@@ -539,15 +541,15 @@
     def mul(self, b):
         asize = self.numdigits()
         bsize = b.numdigits()
-        
+
         a = self
-        
+
         if asize > bsize:
             a, b, asize, bsize = b, a, bsize, asize
 
         if a.sign == 0 or b.sign == 0:
             return NULLRBIGINT
-        
+
         if asize == 1:
             if a._digits[0] == NULLDIGIT:
                 return NULLRBIGINT
@@ -560,14 +562,14 @@
                     return rbigint([_store_digit(res & MASK), _store_digit(carry)], a.sign * b.sign, 2)
                 else:
                     return rbigint([_store_digit(res & MASK)], a.sign * b.sign, 1)
-                
-            result =  _x_mul(a, b, a.digit(0))
+
+            result = _x_mul(a, b, a.digit(0))
         elif USE_KARATSUBA:
             if a is b:
                 i = KARATSUBA_SQUARE_CUTOFF
             else:
                 i = KARATSUBA_CUTOFF
-                
+
             if asize <= i:
                 result = _x_mul(a, b)
                 """elif 2 * asize <= bsize:
@@ -593,13 +595,13 @@
                 return rbigint(self._digits[:self.size], 1, self.size)
             elif digit and digit & (digit - 1) == 0:
                 return self.rshift(ptwotable[digit])
-            
+
         div, mod = _divrem(self, other)
         if mod.sign * other.sign == -1:
             if div.sign == 0:
                 return ONENEGATIVERBIGINT
             div = div.sub(ONERBIGINT)
-            
+
         return div
 
     @jit.look_inside
@@ -610,7 +612,7 @@
     def mod(self, other):
         if self.sign == 0:
             return NULLRBIGINT
-        
+
         if other.sign != 0 and other.numdigits() == 1:
             digit = other.digit(0)
             if digit == 1:
@@ -633,7 +635,7 @@
                         size -= 1
                 else:
                     rem = self.digit(0) % digit
-                    
+
                 if rem == 0:
                     return NULLRBIGINT
                 mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1)
@@ -684,9 +686,9 @@
                     "cannot be negative when 3rd argument specified")
             # XXX failed to implement
             raise ValueError("bigint pow() too negative")
-        
+
         size_b = b.numdigits()
-        
+
         if c is not None:
             if c.sign == 0:
                 raise ValueError("pow() 3rd argument cannot be 0")
@@ -702,13 +704,13 @@
             #     return 0
             if c.numdigits() == 1 and c._digits[0] == ONEDIGIT:
                 return NULLRBIGINT
-   
+
             # if base < 0:
             #     base = base % modulus
             # Having the base positive just makes things easier.
             if a.sign < 0:
                 a = a.mod(c)
-            
+
         elif b.sign == 0:
             return ONERBIGINT
         elif a.sign == 0:
@@ -730,12 +732,12 @@
                     if a.sign == -1 and not digit % 2:
                         ret.sign = 1
                     return ret
-                
+
         # At this point a, b, and c are guaranteed non-negative UNLESS
         # c is NULL, in which case a may be negative. */
 
         z = rbigint([ONEDIGIT], 1, 1)
-        
+
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
         if size_b <= FIVEARY_CUTOFF:
@@ -751,7 +753,7 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 size_b -= 1
-                
+
         else:
             # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
             # This is only useful in the case where c != None.
@@ -784,7 +786,7 @@
                     # must get the next digit from 'b' in order to complete
                     if size_b == 0:
                         break # Done
-                        
+
                     size_b -= 1
                     assert size_b >= 0
                     bi = b.udigit(size_b)
@@ -798,7 +800,7 @@
                     z = _help_mult(z, table[index], c)
             #
             assert j == -5
-        
+
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
         return z
@@ -817,12 +819,12 @@
     def invert(self): #Implement ~x as -(x + 1)
         if self.sign == 0:
             return ONENEGATIVERBIGINT
-        
+
         ret = self.add(ONERBIGINT)
         ret.sign = -ret.sign
         return ret
-        
-    @jit.elidable    
+
+    @jit.elidable
     def lshift(self, int_other):
         if int_other < 0:
             raise ValueError("negative shift count")
@@ -831,14 +833,14 @@
 
         # wordshift, remshift = divmod(int_other, SHIFT)
         wordshift = int_other // SHIFT
-        remshift  = int_other - wordshift * SHIFT
+        remshift = int_other - wordshift * SHIFT
 
         if not remshift:
             # So we can avoid problems with eq, AND avoid the need for normalize.
             if self.sign == 0:
                 return self
             return rbigint([NULLDIGIT] * wordshift + self._digits, self.sign, self.size + wordshift)
-        
+
         oldsize = self.numdigits()
         newsize = oldsize + wordshift + 1
         z = rbigint([NULLDIGIT] * newsize, self.sign, newsize)
@@ -850,7 +852,7 @@
             accum >>= SHIFT
             wordshift += 1
             j += 1
-        
+
         newsize -= 1
         assert newsize >= 0
         z.setdigit(newsize, accum)
@@ -858,7 +860,7 @@
         z._normalize()
         return z
     lshift._always_inline_ = True # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def lqshift(self, int_other):
         " A quicker one with much less checks, int_other is valid and for the most part constant."
@@ -878,7 +880,7 @@
         z._normalize()
         return z
     lqshift._always_inline_ = True # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def rshift(self, int_other, dont_invert=False):
         if int_other < 0:
@@ -910,7 +912,7 @@
         z._normalize()
         return z
     rshift._always_inline_ = 'try' # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def and_(self, other):
         return _bitwise(self, '&', other)
@@ -968,7 +970,7 @@
             self._digits = [NULLDIGIT]
 
     _normalize._always_inline_ = True
-    
+
     @jit.elidable
     def bit_length(self):
         i = self.numdigits()
@@ -1018,7 +1020,7 @@
     # is NULL.
     if c is not None:
         res = res.mod(c)
-        
+
     return res
 
 def digits_from_nonneg_long(l):
@@ -1103,7 +1105,7 @@
 
 def _x_sub(a, b):
     """ Subtract the absolute values of two integers. """
-    
+
     size_a = a.numdigits()
     size_b = b.numdigits()
     sign = 1
@@ -1124,7 +1126,7 @@
             sign = -1
             a, b = b, a
         size_a = size_b = i+1
-        
+
     z = rbigint([NULLDIGIT] * size_a, sign, size_a)
     borrow = UDIGIT_TYPE(0)
     i = _load_unsigned_digit(0)
@@ -1142,7 +1144,7 @@
         borrow >>= SHIFT
         #borrow &= 1
         i += 1
-        
+
     assert borrow == 0
     z._normalize()
     return z
@@ -1152,7 +1154,7 @@
 for x in range(SHIFT-1):
     ptwotable[r_longlong(2 << x)] = x+1
     ptwotable[r_longlong(-2 << x)] = x+1
-    
+
 def _x_mul(a, b, digit=0):
     """
     Grade school multiplication, ignoring the signs.
@@ -1201,14 +1203,14 @@
             i += 1
         z._normalize()
         return z
-    
+
     elif digit:
         if digit & (digit - 1) == 0:
             return b.lqshift(ptwotable[digit])
-        
+
         # Even if it's not power of two it can still be useful.
         return _muladd1(b, digit)
-        
+
     z = rbigint([NULLDIGIT] * (size_a + size_b), 1)
     # gradeschool long mult
     i = UDIGIT_TYPE(0)
@@ -1258,7 +1260,7 @@
     """
     asize = a.numdigits()
     bsize = b.numdigits()
-    
+
     # (ah*X+al)(bh*X+bl) = ah*bh*X*X + (ah*bl + al*bh)*X + al*bl
     # Let k = (ah+al)*(bh+bl) = ah*bl + al*bh  + ah*bh + al*bl
     # Then the original product is
@@ -1341,7 +1343,7 @@
         t2 = _x_add(bh, bl)
 
     t3 = t1.mul(t2)
-    assert t3.sign >=0
+    assert t3.sign >= 0
 
     # Add t3.  It's not obvious why we can't run out of room here.
     # See the (*) comment after this function.
@@ -1421,9 +1423,9 @@
     #bslice = rbigint([0] * asize, 1)
     # XXX we cannot pre-allocate, see comments below!
     # XXX prevent one list from being created.
-    bslice = rbigint(sign = 1)
-    
-    nbdone = 0;
+    bslice = rbigint(sign=1)
+
+    nbdone = 0
     while bsize > 0:
         nbtouse = min(bsize, asize)
 
@@ -1472,7 +1474,7 @@
     The sign of a is ignored; n should not be zero.
     """
     assert n > 0 and n <= MASK
-        
+
     size = a.numdigits()
     z = rbigint([NULLDIGIT] * size, 1, size)
     rem = _inplace_divrem1(z, a, n)
@@ -1553,7 +1555,7 @@
     """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the top.
     """
-    
+
     carry = 0
     assert 0 <= d and d < SHIFT
     i = 0
@@ -1562,18 +1564,18 @@
         z.setdigit(i, acc)
         carry = acc >> SHIFT
         i += 1
-        
+
     return carry
 
 def _v_rshift(z, a, m, d):
     """ Shift digit vector a[0:m] d bits right, with 0 <= d < PyLong_SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the bottom.
     """
-    
+
     carry = _widen_digit(0)
     acc = _widen_digit(0)
     mask = (1 << d) - 1
-    
+
     assert 0 <= d and d < SHIFT
     i = m-1
     while i >= 0:
@@ -1581,7 +1583,7 @@
         carry = acc & mask
         z.setdigit(i, acc >> d)
         i -= 1
-        
+
     return carry
 
 def _x_divrem(v1, w1):
@@ -1589,13 +1591,13 @@
     size_v = v1.numdigits()
     size_w = w1.numdigits()
     assert size_v >= size_w and size_w > 1
-    
+
     v = rbigint([NULLDIGIT] * (size_v + 1), 1, size_v + 1)
     w = rbigint([NULLDIGIT] * size_w, 1, size_w)
-    
+
     """ normalize: shift w1 left so that its top digit is >= PyLong_BASE/2.
         shift v1 left by the same amount. Results go into w and v. """
-        
+
     d = SHIFT - bits_in_digit(w1.digit(abs(size_w-1)))
     carry = _v_lshift(w, w1, size_w, d)
     assert carry == 0
@@ -1603,7 +1605,7 @@
     if carry != 0 or v.digit(abs(size_v-1)) >= w.digit(abs(size_w-1)):
         v.setdigit(size_v, carry)
         size_v += 1
-        
+
     """ Now v->ob_digit[size_v-1] < w->ob_digit[size_w-1], so quotient has
         at most (and usually exactly) k = size_v - size_w digits. """
     k = size_v - size_w
@@ -1612,10 +1614,10 @@
         assert _v_rshift(w, v, size_w, d) == 0
         w._normalize()
         return rbigint([NULLDIGIT]), w
-    
+
     assert k > 0
     a = rbigint([NULLDIGIT] * k, 1, k)
-    
+
     wm1 = w.widedigit(abs(size_w-1))
     wm2 = w.widedigit(abs(size_w-2))
 
@@ -1625,7 +1627,7 @@
         assert j >= 0
         """ inner loop: divide vk[0:size_w+1] by w0[0:size_w], giving
             single-digit quotient q, remainder in vk[0:size_w]. """
-            
+
         # estimate quotient digit q; may overestimate by 1 (rare)
         if j >= size_v:
             vtop = 0
@@ -1638,9 +1640,9 @@
         while wm2 * q > ((r << SHIFT) | v.widedigit(abs(j-2))):
             q -= 1
             r += wm1
-            
+
         #assert q <= MASK+1, We need to compare to BASE <=, but ehm, it gives a buildin long error. So we ignore this.
-        
+
         # subtract q*w0[0:size_w] from vk[0:size_w+1]
         zhi = 0
         i = 0
@@ -1649,7 +1651,7 @@
             v.setdigit(k+i, z)
             zhi = z >> SHIFT
             i += 1
-        
+
         # add w back if q was too large (this branch taken rarely)
         if vtop + zhi < 0:
             carry = UDIGIT_TYPE(0)
@@ -1660,21 +1662,20 @@
                 carry >>= SHIFT
                 i += 1
             q -= 1
-            
+
         # store quotient digit
         a.setdigit(k, q)
         k -= 1
         j -= 1
-        
-        
+
     carry = _v_rshift(w, v, size_w, d)
     assert carry == 0
-    
+
     a._normalize()
     w._normalize()
-    
+
     return a, w
-        
+
 def _divrem(a, b):
     """ Long division with remainder, top-level routine """
     size_a = a.numdigits()
@@ -1891,14 +1892,16 @@
         da = float(a.digit(a_size))
         while True:
             a_size -= 1
-            if a_size < 0: break
+            if a_size < 0:
+                break
             da = da * BASE_AS_FLOAT + a.digit(a_size)
 
         b_size -= 1
         db = float(b.digit(b_size))
         while True:
             b_size -= 1
-            if b_size < 0: break
+            if b_size < 0:
+                break
             db = db * BASE_AS_FLOAT + b.digit(b_size)
 
         return _truediv_result(da / db, negate)
@@ -2033,7 +2036,6 @@
 
 def _format_int(val, digits):
     base = len(digits)
-    j = 0
     out = []
     while val:
         out.append(digits[val % base])
@@ -2086,7 +2088,7 @@
     if negative:
         output.append('-')
     output.append(prefix)
-    _format_recursive(x,len(pts)-1, output, pts, digits, output.getlength())
+    _format_recursive(x, len(pts)-1, output, pts, digits, output.getlength())
 
     output.append(suffix)
     return output.build()
@@ -2156,7 +2158,7 @@
             digb = b.digit(i) ^ maskb
         else:
             digb = maskb
-            
+
         if op == '&':
             z.setdigit(i, diga & digb)
         elif op == '|':
@@ -2164,11 +2166,11 @@
         elif op == '^':
             z.setdigit(i, diga ^ digb)
         i += 1
-        
+
     z._normalize()
     if negz == 0:
         return z
-    
+
     return z.invert()
 _bitwise._annspecialcase_ = "specialize:arg(1)"
 

From noreply at buildbot.pypy.org  Thu Jun 27 00:24:54 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 27 Jun 2013 00:24:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix mmap tests
Message-ID: <20130626222454.509C01C0397@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65029:49725b81d6b9
Date: 2013-06-27 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/49725b81d6b9/

Log:	Fix mmap tests

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -170,7 +170,7 @@
         if step == 0:  # index only
             return space.wrap(ord(self.mmap.getitem(start)))
         elif step == 1:
-            return space.wrap(self.mmap.getslice(start, stop - start))
+            return space.wrapbytes(self.mmap.getslice(start, stop - start))
         else:
             res = "".join([self.mmap.getitem(i)
                            for i in range(start, stop, step)])
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -522,7 +522,7 @@
 
         f.seek(0)
         m = mmap(f.fileno(), 6)
-        assert m[-3:7] == b"bar"
+        assert eval('m[-3:7]') == b"bar"
 
         f.close()
 
@@ -542,20 +542,18 @@
 
     def test_buffer_write(self):
         from mmap import mmap
-        f = open(self.tmpname + "y", "w+")
-        f.write("foobar")
+        f = open(self.tmpname + "y", "wb+")
+        f.write(b"foobar")
         f.flush()
         m = mmap(f.fileno(), 6)
-        m[5] = '?'
-        b = buffer(m)
-        try:
-            b[:3] = "FOO"
-        except TypeError:     # on CPython: "buffer is read-only" :-/
-            skip("on CPython: buffer is read-only")
+        m[5] = ord('?')
+        b = memoryview(m)
+        b[:3] = b"FOO"
+        del b  # For CPython: "cannot close exported pointers exist"
         m.close()
         f.seek(0)
         got = f.read()
-        assert got == "FOOba?"
+        assert got == b"FOOba?"
         f.close()
 
     def test_offset(self):

From noreply at buildbot.pypy.org  Thu Jun 27 00:30:23 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 27 Jun 2013 00:30:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Oops,
 this eval() thing is not necessary if the assert does not fail.
Message-ID: <20130626223023.0949A1C0C97@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65030:c473fd4a7af5
Date: 2013-06-27 00:29 +0200
http://bitbucket.org/pypy/pypy/changeset/c473fd4a7af5/

Log:	Oops, this eval() thing is not necessary if the assert does not
	fail.

diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -522,7 +522,7 @@
 
         f.seek(0)
         m = mmap(f.fileno(), 6)
-        assert eval('m[-3:7]') == b"bar"
+        assert m[-3:7] == b"bar"
 
         f.close()
 

From noreply at buildbot.pypy.org  Thu Jun 27 01:04:32 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 27 Jun 2013 01:04:32 +0200 (CEST)
Subject: [pypy-commit] pypy split-ast-classes: Some missing _fields
Message-ID: <20130626230432.556241C021A@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: split-ast-classes
Changeset: r65031:2b627c6e72e0
Date: 2013-06-27 01:01 +0200
http://bitbucket.org/pypy/pypy/changeset/2b627c6e72e0/

Log:	Some missing _fields

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -3134,7 +3134,7 @@
         _ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(_target, _iter, _ifs)
 
-State.ast_type('comprehension', 'AST', None)
+State.ast_type('comprehension', 'AST', ['target', 'iter', 'ifs'])
 
 class excepthandler(AST):
 
@@ -3260,7 +3260,7 @@
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwarg, _defaults)
 
-State.ast_type('arguments', 'AST', None)
+State.ast_type('arguments', 'AST', ['args', 'vararg', 'kwarg', 'defaults'])
 
 class keyword(AST):
 
@@ -3291,7 +3291,7 @@
         _value = expr.from_object(space, w_value)
         return keyword(_arg, _value)
 
-State.ast_type('keyword', 'AST', None)
+State.ast_type('keyword', 'AST', ['arg', 'value'])
 
 class alias(AST):
 
@@ -3321,7 +3321,7 @@
         _asname = space.str_or_None_w(w_asname)
         return alias(_name, _asname)
 
-State.ast_type('alias', 'AST', None)
+State.ast_type('alias', 'AST', ['name', 'asname'])
 
 class ASTVisitor(object):
 
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -42,6 +42,10 @@
                                  space.wrap("_fields"))
         assert space.eq_w(w_fields, space.wrap(
             ('name', 'args', 'body', 'decorator_list')))
+        w_fields = space.getattr(ast.get(space).w_arguments,
+                                 space.wrap("_fields"))
+        assert space.eq_w(w_fields, space.wrap(
+            ('args', 'vararg', 'kwarg', 'defaults')))
         
     def test_attributes(self, space):
         w_attrs = space.getattr(ast.get(space).w_FunctionDef,
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -118,7 +118,8 @@
         self.emit("visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
         self.make_converters(product.fields, name)
-        self.emit("State.ast_type('%r', 'AST', None)" % (name,))
+        self.emit("State.ast_type('%r', 'AST', %s)" %
+                  (name, [repr(f.name) for f in product.fields]))
         self.emit("")
 
     def get_value_converter(self, field, value):

From noreply at buildbot.pypy.org  Thu Jun 27 09:19:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 09:19:05 +0200 (CEST)
Subject: [pypy-commit] stmgc default: The 'clear_section_count' was broken:
 it was never different than
Message-ID: <20130627071905.DFCC71C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r297:6fe714d82eba
Date: 2013-06-27 08:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/6fe714d82eba/

Log:	The 'clear_section_count' was broken: it was never different than
	GC_NURSERY_SECTION. Try to fix it another way.

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -3,6 +3,8 @@
 
 static int is_in_nursery(struct tx_descriptor *d, gcptr obj)
 {
+    assert(!(d->nursery_base_for_next_collect <= (char*)obj &&
+             ((char*)obj) < d->nursery_base));
     return (d->nursery_base <= (char*)obj && ((char*)obj) < d->nursery_end);
 }
 
@@ -29,6 +31,7 @@
     d->nursery_end = d->nursery_base + GC_NURSERY;  /* end of nursery */
     d->nursery_current = d->nursery_base;           /* current position */
     d->nursery_nextlimit = d->nursery_base;         /* next section limit */
+    d->nursery_base_for_next_collect = d->nursery_base;
 
     dprintf(("minor: nursery is at [%p to %p]\n", d->nursery_base,
              d->nursery_end));
@@ -537,12 +540,27 @@
     stm_free(d->nursery_base, GC_NURSERY);
     d->nursery_base = stm_malloc(GC_NURSERY);
     d->nursery_end = d->nursery_base + GC_NURSERY;
+    d->nursery_current = d->nursery_base;
+    d->nursery_nextlimit = d->nursery_base;
     dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
              d->nursery_end));
 #endif
 
-    d->nursery_current = d->nursery_base;
-    d->nursery_nextlimit = d->nursery_base;
+    /* when doing minor collections with the nursery "mostly empty",
+       as occurs when other threads force major collections but this
+       thread didn't do much at all, then we don't actually reset
+       'nursery_current' to 'nursery_base'.  Instead we keep it
+       unmodified.  It will continue to use the already-partially-
+       cleared section.
+    */
+    if ((d->nursery_nextlimit - d->nursery_base) < GC_NURSERY / 8 &&
+        (d->nursery_nextlimit - d->nursery_current) > GC_NURSERY_SECTION / 4) {
+    }
+    else {
+        d->nursery_current = d->nursery_base;
+        d->nursery_nextlimit = d->nursery_base;
+    }
+    d->nursery_base_for_next_collect = d->nursery_current;
 
     assert(!stmgc_minor_collect_anything_to_do(d));
 }
@@ -563,7 +581,7 @@
 
 int stmgc_minor_collect_anything_to_do(struct tx_descriptor *d)
 {
-    if (d->nursery_current == d->nursery_base /*&&
+    if (d->nursery_current == d->nursery_base_for_next_collect /*&&
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
         assert(gcptrlist_size(&d->public_with_young_copy) == 0);
@@ -612,8 +630,6 @@
         return P;
     }
 
-    revision_t clear_section_count = GC_NURSERY_SECTION;
-
     /* Are we at the end of the nursery? */
     if (d->nursery_nextlimit == d->nursery_end) {
         /* Yes */
@@ -622,20 +638,16 @@
 
         /* Start a minor collection
          */
-        if (d->nursery_current - d->nursery_base < clear_section_count) {
-            /* help cases where we do a large amount of minor collections */
-            clear_section_count = d->nursery_current - d->nursery_base;
-        }
-
         stmgc_minor_collect();
         stmgcpage_possibly_major_collect(0);
 
+        assert(d->nursery_base_for_next_collect == d->nursery_base);
         assert(d->nursery_current == d->nursery_base);
         assert(d->nursery_nextlimit == d->nursery_base);
     }
 
     /* Clear the next section */
-    memset(d->nursery_nextlimit, 0, clear_section_count);
+    memset(d->nursery_nextlimit, 0, GC_NURSERY_SECTION);
     d->nursery_nextlimit += GC_NURSERY_SECTION;
 
     /* Return the object from there */
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -25,6 +25,7 @@
     char *nursery_nextlimit;                                            \
     char *nursery_end;                                                  \
     char *nursery_base;                                                 \
+    char *nursery_base_for_next_collect;                                \
                                                                         \
     /* Between collections, we add to 'old_objects_to_trace' the        \
        private objects that are old but may contain pointers to         \

From noreply at buildbot.pypy.org  Thu Jun 27 09:19:07 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 09:19:07 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Improve the previous hack.
Message-ID: <20130627071907.05F201C2FB2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r298:c27e67b3f4ff
Date: 2013-06-27 09:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/c27e67b3f4ff/

Log:	Improve the previous hack.

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -1,10 +1,10 @@
 #include "stmimpl.h"
+#include <sys/mman.h>
+
 
 #ifdef _GC_DEBUG
 /************************************************************/
 
-#include <sys/mman.h>
-
 #define PAGE_SIZE  4096
 #define MMAP_TOTAL  671088640   /* 640MB */
 
@@ -89,3 +89,40 @@
 
 /************************************************************/
 #endif
+
+
+void stm_clear_large_memory_chunk(void *base, size_t size,
+                                  size_t already_cleared)
+{
+    char *baseaddr = base;
+
+    if (size > 2 * PAGE_SIZE) {
+        int lowbits = ((intptr_t)baseaddr) & (PAGE_SIZE-1);
+        if (lowbits) {   /*  clear the initial misaligned part, if any */
+            int partpage = PAGE_SIZE - lowbits;
+            memset(baseaddr, 0, partpage);
+            baseaddr += partpage;
+            size -= partpage;
+        }
+        /* 'already_cleared' bytes at the end are assumed to be already
+           cleared.  Reduce 'size' accordingly, but avoid getting a
+           misaligned 'size'. */
+        size_t length = size & (-PAGE_SIZE);
+        if (already_cleared > (size - length)) {
+            already_cleared -= (size - length);
+            already_cleared &= -PAGE_SIZE;
+            length -= already_cleared;
+            size = length;
+            already_cleared = 0;
+        }
+
+        int err = madvise(baseaddr, length, MADV_DONTNEED);
+        if (err == 0) {   /*  madvise() worked */
+            baseaddr += length;
+            size -= length;
+        }
+    }
+    if (size > already_cleared) { /* clear the final misaligned part, if any */
+        memset(baseaddr, 0, size - already_cleared);
+    }
+}
diff --git a/c4/dbgmem.h b/c4/dbgmem.h
--- a/c4/dbgmem.h
+++ b/c4/dbgmem.h
@@ -15,5 +15,7 @@
 
 #endif
 
+void stm_clear_large_memory_chunk(void *, size_t, size_t);
+
 
 #endif
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -3,8 +3,6 @@
 
 static int is_in_nursery(struct tx_descriptor *d, gcptr obj)
 {
-    assert(!(d->nursery_base_for_next_collect <= (char*)obj &&
-             ((char*)obj) < d->nursery_base));
     return (d->nursery_base <= (char*)obj && ((char*)obj) < d->nursery_end);
 }
 
@@ -20,6 +18,10 @@
     stmcb_trace(obj, visit);
 }
 
+/* forward declarations */
+static int minor_collect_anything_to_do(struct tx_descriptor *);
+static gcptr allocate_next_section(size_t size, revision_t tid);
+
 /************************************************************/
 
 void stmgc_init_nursery(void)
@@ -31,7 +33,7 @@
     d->nursery_end = d->nursery_base + GC_NURSERY;  /* end of nursery */
     d->nursery_current = d->nursery_base;           /* current position */
     d->nursery_nextlimit = d->nursery_base;         /* next section limit */
-    d->nursery_base_for_next_collect = d->nursery_base;
+    d->nursery_cleared = NC_REGULAR;
 
     dprintf(("minor: nursery is at [%p to %p]\n", d->nursery_base,
              d->nursery_end));
@@ -40,7 +42,7 @@
 void stmgc_done_nursery(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    assert(!stmgc_minor_collect_anything_to_do(d));
+    assert(!minor_collect_anything_to_do(d));
     stm_free(d->nursery_base, GC_NURSERY);
 
     gcptrlist_delete(&d->old_objects_to_trace);
@@ -53,8 +55,6 @@
     d->nursery_current = d->nursery_end;
 }
 
-static gcptr allocate_next_section(size_t size, revision_t tid);  /* forward */
-
 inline static gcptr allocate_nursery(size_t size, revision_t tid)
 {
     /* if 'tid == -1', we must not collect */
@@ -540,29 +540,37 @@
     stm_free(d->nursery_base, GC_NURSERY);
     d->nursery_base = stm_malloc(GC_NURSERY);
     d->nursery_end = d->nursery_base + GC_NURSERY;
-    d->nursery_current = d->nursery_base;
     d->nursery_nextlimit = d->nursery_base;
     dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
              d->nursery_end));
 #endif
 
-    /* when doing minor collections with the nursery "mostly empty",
+    /* When doing minor collections with the nursery "mostly empty",
        as occurs when other threads force major collections but this
-       thread didn't do much at all, then we don't actually reset
-       'nursery_current' to 'nursery_base'.  Instead we keep it
-       unmodified.  It will continue to use the already-partially-
-       cleared section.
+       thread didn't do much at all, then we clear the nursery using
+       the system's madvise().  The goal is twofold: first, if this
+       thread only uses very small amounts of memory, it avoids doing
+       a memset() to clear a complete section after every major GC.
+       Second, if the thread is really idle, then its nursery is sent
+       back to the system until it's really needed.
     */
-    if ((d->nursery_nextlimit - d->nursery_base) < GC_NURSERY / 8 &&
-        (d->nursery_nextlimit - d->nursery_current) > GC_NURSERY_SECTION / 4) {
+    if ((d->nursery_nextlimit - d->nursery_base) < GC_NURSERY / 10) {
+        size_t already_cleared = 0;
+        if (d->nursery_cleared == NC_ALREADY_CLEARED) {
+            already_cleared = d->nursery_end - d->nursery_current;
+        }
+        stm_clear_large_memory_chunk(d->nursery_base, GC_NURSERY,
+                                     already_cleared);
+        d->nursery_cleared = NC_ALREADY_CLEARED;
     }
     else {
-        d->nursery_current = d->nursery_base;
-        d->nursery_nextlimit = d->nursery_base;
+        d->nursery_cleared = NC_REGULAR;
     }
-    d->nursery_base_for_next_collect = d->nursery_current;
 
-    assert(!stmgc_minor_collect_anything_to_do(d));
+    d->nursery_current = d->nursery_base;
+    d->nursery_nextlimit = d->nursery_base;
+
+    assert(!minor_collect_anything_to_do(d));
 }
 
 void stmgc_minor_collect(void)
@@ -579,9 +587,10 @@
     minor_collect(d);
 }
 
-int stmgc_minor_collect_anything_to_do(struct tx_descriptor *d)
+#ifndef NDEBUG
+int minor_collect_anything_to_do(struct tx_descriptor *d)
 {
-    if (d->nursery_current == d->nursery_base_for_next_collect /*&&
+    if (d->nursery_current == d->nursery_base /*&&
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
         assert(gcptrlist_size(&d->public_with_young_copy) == 0);
@@ -600,6 +609,7 @@
         return 1;
     }
 }
+#endif
 
 static gcptr allocate_next_section(size_t allocate_size, revision_t tid)
 {
@@ -641,13 +651,13 @@
         stmgc_minor_collect();
         stmgcpage_possibly_major_collect(0);
 
-        assert(d->nursery_base_for_next_collect == d->nursery_base);
         assert(d->nursery_current == d->nursery_base);
         assert(d->nursery_nextlimit == d->nursery_base);
     }
 
     /* Clear the next section */
-    memset(d->nursery_nextlimit, 0, GC_NURSERY_SECTION);
+    if (d->nursery_cleared != NC_ALREADY_CLEARED)
+        memset(d->nursery_nextlimit, 0, GC_NURSERY_SECTION);
     d->nursery_nextlimit += GC_NURSERY_SECTION;
 
     /* Return the object from there */
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -25,7 +25,7 @@
     char *nursery_nextlimit;                                            \
     char *nursery_end;                                                  \
     char *nursery_base;                                                 \
-    char *nursery_base_for_next_collect;                                \
+    enum { NC_REGULAR, NC_ALREADY_CLEARED } nursery_cleared;            \
                                                                         \
     /* Between collections, we add to 'old_objects_to_trace' the        \
        private objects that are old but may contain pointers to         \
@@ -56,7 +56,6 @@
 void stmgc_done_nursery(void);
 void stmgc_minor_collect(void);
 void stmgc_minor_collect_no_abort(void);
-int stmgc_minor_collect_anything_to_do(struct tx_descriptor *);
 gcptr stmgc_duplicate(gcptr);
 gcptr stmgc_duplicate_old(gcptr);
 size_t stmgc_size(gcptr);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -84,6 +84,7 @@
     void *STUB_THREAD(gcptr);
     void stm_clear_read_cache(void);
     int in_nursery(gcptr);
+    void stm_clear_large_memory_chunk(void *, size_t, size_t);
 
     gcptr getptr(gcptr, long);
     void setptr(gcptr, long, gcptr);
@@ -95,6 +96,8 @@
     long rawgetlong(gcptr, long);
     void rawsetlong(gcptr, long, long);
 
+    void *memset(void *s, int c, size_t n);
+
     gcptr pseudoprebuilt(size_t size, int tid);
     gcptr pseudoprebuilt_with_hash(size_t size, int tid, revision_t hash);
     revision_t get_private_rev_num(void);
diff --git a/c4/test/test_dbgmem.py b/c4/test/test_dbgmem.py
new file mode 100644
--- /dev/null
+++ b/c4/test/test_dbgmem.py
@@ -0,0 +1,45 @@
+from support import *
+import random
+
+
+def test_clear_large_memory_chunk():
+    PAGE_SIZE = 4096
+    blocksize = 8 * PAGE_SIZE
+    block = ffi.new("char[]", blocksize)
+    blockbase = int(ffi.cast("intptr_t", block))
+
+    for i in range(1000):
+
+        if random.random() < 0.25:
+            base = PAGE_SIZE - (blockbase & (PAGE_SIZE-1))
+            assert ((blockbase + base) & (PAGE_SIZE-1)) == 0
+        else:
+            base = random.randrange(1, PAGE_SIZE+1)
+        assert base > 0
+
+        length = random.randrange(1, 6 * PAGE_SIZE)
+        if random.random() < 0.25:
+            blockend = blockbase + base + length
+            blockend &= ~(PAGE_SIZE-1)
+            length = blockend - (blockbase + base)
+            if length <= 0:
+                continue
+
+        assert length > 0
+        assert base + length + 1 < blocksize
+
+        already_cleared = random.randrange(0, length+1)
+        already_cleared &= ((1 << random.randrange(0, 16)) - 1)
+        assert 0 <= already_cleared <= length
+
+        block[base - 1] = '<'
+        lib.memset(block + base, ord('.'), length - already_cleared)
+        lib.memset(block + base + length - already_cleared, 0, already_cleared)
+        block[base + length] = '>'
+
+        print hex(blockbase + base), hex(length), hex(already_cleared)
+        lib.stm_clear_large_memory_chunk(block + base, length, already_cleared)
+
+        assert block[base - 1] == '<'
+        assert block[base + length] == '>'
+        assert ffi.buffer(block + base, length)[:] == '\x00' * length

From noreply at buildbot.pypy.org  Thu Jun 27 09:19:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 09:19:08 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fixes
Message-ID: <20130627071908.1353E1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r299:4413d3ad4a81
Date: 2013-06-27 09:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/4413d3ad4a81/

Log:	Fixes

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -2,10 +2,12 @@
 #include <sys/mman.h>
 
 
+#define PAGE_SIZE  4096
+
+
 #ifdef _GC_DEBUG
 /************************************************************/
 
-#define PAGE_SIZE  4096
 #define MMAP_TOTAL  671088640   /* 640MB */
 
 static pthread_mutex_t malloc_mutex = PTHREAD_MUTEX_INITIALIZER;
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -19,7 +19,9 @@
 }
 
 /* forward declarations */
+#ifndef NDEBUG
 static int minor_collect_anything_to_do(struct tx_descriptor *);
+#endif
 static gcptr allocate_next_section(size_t size, revision_t tid);
 
 /************************************************************/

From noreply at buildbot.pypy.org  Thu Jun 27 09:35:04 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Jun 2013 09:35:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: replace an import *,
	fijal said it caused a bug
Message-ID: <20130627073504.3BEA61C01E5@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r65032:be858ed67028
Date: 2013-06-27 00:34 -0700
http://bitbucket.org/pypy/pypy/changeset/be858ed67028/

Log:	replace an import *, fijal said it caused a bug

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -3,35 +3,34 @@
 """
 
 import os
-from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
-                                            ConstFloat, BoxInt,
-                                            BoxFloat, INT, REF, FLOAT,
-                                            TargetToken)
-from rpython.jit.backend.x86.regloc import *
-from rpython.rtyper.lltypesystem import lltype, rffi, rstr
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rgc
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.codewriter import longlong
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
+    ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken)
 from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.backend.llsupport.descr import ArrayDescr
-from rpython.jit.backend.llsupport.descr import CallDescr
-from rpython.jit.backend.llsupport.descr import unpack_arraydescr
-from rpython.jit.backend.llsupport.descr import unpack_fielddescr
-from rpython.jit.backend.llsupport.descr import unpack_interiorfielddescr
+from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
+    unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
-from rpython.jit.backend.llsupport.regalloc import FrameManager, BaseRegalloc,\
-     RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op
-from rpython.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE
-from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64
+from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc,
+     RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op)
 from rpython.jit.backend.x86 import rx86
+from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
+    IS_X86_64)
+from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
+    FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
+    ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
+    xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
+    X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG)
+from rpython.rlib import rgc
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
+
 
 class X86RegisterManager(RegisterManager):
-
     box_types = [INT, REF]
     all_regs = [ecx, eax, edx, ebx, esi, edi]
     no_lower_byte_regs = [esi, edi]
@@ -886,7 +885,6 @@
                 gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8))
         return gcmap
 
-
     def consider_setfield_gc(self, op):
         ofs, size, _ = unpack_fielddescr(op.getdescr())
         ofs_loc = imm(ofs)
@@ -950,7 +948,7 @@
     def consider_setarrayitem_gc(self, op):
         itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
         args = op.getarglist()
-        base_loc  = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
         if itemsize == 1:
             need_lower_byte = True
         else:
@@ -1311,7 +1309,6 @@
         #if jump_op is not None and jump_op.getdescr() is descr:
         #    self._compute_hint_frame_locations_from_descr(descr)
 
-
     def consider_keepalive(self, op):
         pass
 

From noreply at buildbot.pypy.org  Thu Jun 27 09:39:20 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Jun 2013 09:39:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: rearrange
Message-ID: <20130627073920.507461C1067@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r65033:d22b26973268
Date: 2013-06-27 00:38 -0700
http://bitbucket.org/pypy/pypy/changeset/d22b26973268/

Log:	rearrange

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -4,12 +4,6 @@
 
 import os
 from rpython.jit.backend.llsupport import symbolic
-from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
-from rpython.jit.codewriter import longlong
-from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
-    ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken)
-from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
     unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
@@ -18,11 +12,17 @@
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
     IS_X86_64)
+from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
     FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
     ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG)
+from rpython.jit.codewriter import longlong
+from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
+    ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken)
+from rpython.jit.metainterp.resoperation import rop
 from rpython.rlib import rgc
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint

From noreply at buildbot.pypy.org  Thu Jun 27 10:10:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 10:10:39 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Return a boolean, not a revision_t.
Message-ID: <20130627081039.4C5071C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r300:4c3aa2e18d97
Date: 2013-06-27 10:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/4c3aa2e18d97/

Log:	Return a boolean, not a revision_t.

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -226,7 +226,7 @@
     return result;
 }
 
-revision_t stm_pointer_equal(gcptr p1, gcptr p2)
+_Bool stm_pointer_equal(gcptr p1, gcptr p2)
 {
     /* fast path for two equal pointers */
     if (p1 == p2)
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -36,7 +36,7 @@
 revision_t stm_id(gcptr);
 /* returns nonzero if the two object-copy pointers belong to the
 same original object */
-revision_t stm_pointer_equal(gcptr, gcptr);
+_Bool stm_pointer_equal(gcptr, gcptr);
 
 /* to push/pop objects into the local shadowstack */
 /* (could be turned into macros or something later) */
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -49,7 +49,7 @@
     gcptr stm_allocate(size_t size, unsigned int tid);
     revision_t stm_hash(gcptr);
     revision_t stm_id(gcptr);
-    revision_t stm_pointer_equal(gcptr, gcptr);
+    _Bool stm_pointer_equal(gcptr, gcptr);
     void stm_push_root(gcptr);
     gcptr stm_pop_root(void);
     void stm_set_max_aborts(int max_aborts);

From noreply at buildbot.pypy.org  Thu Jun 27 11:04:49 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Jun 2013 11:04:49 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: rename virtualizable2 to
 virtualizable. Remove the ootype part (it never
Message-ID: <20130627090449.129D01C2FED@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r65034:5bf501d6e094
Date: 2013-06-27 11:02 +0200
http://bitbucket.org/pypy/pypy/changeset/5bf501d6e094/

Log:	rename virtualizable2 to virtualizable. Remove the ootype part (it
	never worked I think)

diff --git a/pypy/doc/jit/virtualizable.rst b/pypy/doc/jit/virtualizable.rst
--- a/pypy/doc/jit/virtualizable.rst
+++ b/pypy/doc/jit/virtualizable.rst
@@ -26,7 +26,7 @@
 virtualizable works like this:
 
     class Frame(object):
-       _virtualizable2_ = ['locals[*]', 'stackdepth']
+       _virtualizable_ = ['locals[*]', 'stackdepth']
 
 And we use them in ``JitDriver`` like this::
 
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -15,15 +15,15 @@
 from pypy.interpreter.pyopcode import ExitFrame
 from opcode import opmap
 
-PyFrame._virtualizable2_ = ['last_instr', 'pycode',
-                            'valuestackdepth', 'locals_stack_w[*]',
-                            'cells[*]',
-                            'last_exception',
-                            'lastblock',
-                            'is_being_profiled',
-                            'w_globals',
-                            'w_f_trace',
-                            ]
+PyFrame._virtualizable_ = ['last_instr', 'pycode',
+                           'valuestackdepth', 'locals_stack_w[*]',
+                           'cells[*]',
+                           'last_exception',
+                           'lastblock',
+                           'is_being_profiled',
+                           'w_globals',
+                           'w_f_trace',
+                           ]
 
 JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE']
 
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3059,7 +3059,7 @@
         from rpython.rlib.jit import hint
 
         class A:
-            _virtualizable2_ = []
+            _virtualizable_ = []
         class B(A):
             def meth(self):
                 return self
@@ -3099,7 +3099,7 @@
         from rpython.rlib.jit import hint
 
         class A:
-            _virtualizable2_ = []
+            _virtualizable_ = []
 
         class I:
             pass
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -23,7 +23,7 @@
         # - floats neg and abs
 
         class Frame(object):
-            _virtualizable2_ = ['i']
+            _virtualizable_ = ['i']
 
             def __init__(self, i):
                 self.i = i
@@ -98,7 +98,7 @@
                 self.val = val
 
         class Frame(object):
-            _virtualizable2_ = ['thing']
+            _virtualizable_ = ['thing']
 
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'],
                            virtualizables = ['frame'],
diff --git a/rpython/jit/codewriter/test/test_policy.py b/rpython/jit/codewriter/test/test_policy.py
--- a/rpython/jit/codewriter/test/test_policy.py
+++ b/rpython/jit/codewriter/test/test_policy.py
@@ -131,7 +131,7 @@
 
 def test_access_directly_but_not_seen():
     class X:
-        _virtualizable2_ = ["a"]
+        _virtualizable_ = ["a"]
     def h(x, y):
         w = 0
         for i in range(y):
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -643,7 +643,7 @@
         # exactly the same logic as the previous test, but with 'frame.j'
         # instead of just 'j'
         class Frame(object):
-            _virtualizable2_ = ['j']
+            _virtualizable_ = ['j']
             def __init__(self, j):
                 self.j = j
 
@@ -767,7 +767,7 @@
                 self.val = val
 
         class Frame(object):
-            _virtualizable2_ = ['thing']
+            _virtualizable_ = ['thing']
 
         driver = JitDriver(greens = ['codeno'], reds = ['i', 's', 'frame'],
                            virtualizables = ['frame'],
@@ -809,7 +809,7 @@
                 self.val = val
 
         class Frame(object):
-            _virtualizable2_ = ['thing']
+            _virtualizable_ = ['thing']
 
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'],
                            virtualizables = ['frame'],
@@ -860,7 +860,7 @@
                 self.val = val
 
         class Frame(object):
-            _virtualizable2_ = ['thing']
+            _virtualizable_ = ['thing']
 
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'],
                            virtualizables = ['frame'],
@@ -911,7 +911,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['l[*]', 's']
+            _virtualizable_ = ['l[*]', 's']
 
             def __init__(self, l, s):
                 self = hint(self, access_directly=True,
@@ -954,7 +954,7 @@
                 self.val = val
 
         class Frame(object):
-            _virtualizable2_ = ['thing']
+            _virtualizable_ = ['thing']
 
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'],
                            virtualizables = ['frame'],
diff --git a/rpython/jit/metainterp/test/test_tracingopts.py b/rpython/jit/metainterp/test/test_tracingopts.py
--- a/rpython/jit/metainterp/test/test_tracingopts.py
+++ b/rpython/jit/metainterp/test/test_tracingopts.py
@@ -344,7 +344,7 @@
                                     virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['l[*]', 's']
+            _virtualizable_ = ['l[*]', 's']
 
             def __init__(self, a, s):
                 self = jit.hint(self, access_directly=True, fresh_virtualizable=True)
diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -46,8 +46,8 @@
         ('vable_token', llmemory.GCREF),
         ('inst_x', lltype.Signed),
         ('inst_node', lltype.Ptr(LLtypeMixin.NODE)),
-        hints = {'virtualizable2_accessor': FieldListAccessor()})
-    XY._hints['virtualizable2_accessor'].initialize(
+        hints = {'virtualizable_accessor': FieldListAccessor()})
+    XY._hints['virtualizable_accessor'].initialize(
         XY, {'inst_x': IR_IMMUTABLE, 'inst_node': IR_IMMUTABLE})
 
     xy_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
@@ -214,8 +214,8 @@
         ('inst_x', lltype.Signed),
         ('inst_l1', lltype.Ptr(lltype.GcArray(lltype.Signed))),
         ('inst_l2', lltype.Ptr(lltype.GcArray(lltype.Signed))),
-        hints = {'virtualizable2_accessor': FieldListAccessor()})
-    XY2._hints['virtualizable2_accessor'].initialize(
+        hints = {'virtualizable_accessor': FieldListAccessor()})
+    XY2._hints['virtualizable_accessor'].initialize(
         XY2, {'inst_x': IR_IMMUTABLE,
               'inst_l1': IR_IMMUTABLE_ARRAY, 'inst_l2': IR_IMMUTABLE_ARRAY})
 
@@ -445,7 +445,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
             def __init__(self, x, y):
                 self.x = x
@@ -474,7 +474,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['l[*]', 's']
+            _virtualizable_ = ['l[*]', 's']
 
             def __init__(self, l, s):
                 self.l = l
@@ -509,7 +509,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
             def __init__(self, x, y):
                 self.x = x
@@ -537,7 +537,7 @@
                               virtualizables = ['frame'])
 
         class BaseFrame(object):
-            _virtualizable2_ = ['x[*]']
+            _virtualizable_ = ['x[*]']
 
             def __init__(self, x):
                 self.x = x
@@ -568,7 +568,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -601,7 +601,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -641,7 +641,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -674,7 +674,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -711,7 +711,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class Y:
             pass
@@ -756,7 +756,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class Y:
             pass
@@ -806,7 +806,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class FooBarError(Exception):
             pass
@@ -850,7 +850,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -887,7 +887,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -939,7 +939,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -976,7 +976,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
         class SomewhereElse:
             pass
@@ -1010,7 +1010,7 @@
                               virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['stackpos', 'stack[*]']
+            _virtualizable_ = ['stackpos', 'stack[*]']
 
         def f(n):
             frame = Frame()
@@ -1039,7 +1039,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
             def __init__(self, x, y):
                 self = hint(self, access_directly=True)
@@ -1093,7 +1093,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
             def __init__(self, x, y):
                 self = hint(self, access_directly=True)
@@ -1125,7 +1125,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y', 'z']
+            _virtualizable_ = ['x', 'y', 'z']
 
             def __init__(self, x, y, z=1):
                 self = hint(self, access_directly=True)
@@ -1160,7 +1160,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x[*]']
+            _virtualizable_ = ['x[*]']
 
             def __init__(self, x, y):
                 self = hint(self, access_directly=True,
@@ -1192,7 +1192,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
             def __init__(self, x, y):
                 self = hint(self, access_directly=True)
@@ -1231,7 +1231,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
             def __init__(self, x, y):
                 self.x = x
@@ -1271,7 +1271,7 @@
                                 virtualizables = ['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x', 'y']
+            _virtualizable_ = ['x', 'y']
 
             def __init__(self, x, y):
                 self = hint(self, access_directly=True)
@@ -1315,7 +1315,7 @@
 
     def test_inlining(self):
         class Frame(object):
-            _virtualizable2_ = ['x', 'next']
+            _virtualizable_ = ['x', 'next']
 
             def __init__(self, x):
                 self = hint(self, access_directly=True)
@@ -1350,7 +1350,7 @@
     def test_guard_failure_in_inlined_function(self):
 
         class Frame(object):
-            _virtualizable2_ = ['n', 'next']
+            _virtualizable_ = ['n', 'next']
 
             def __init__(self, n):
                 self = hint(self, access_directly=True)
@@ -1404,7 +1404,7 @@
                 self.val = val
 
         class Frame(object):
-            _virtualizable2_ = ['thing']
+            _virtualizable_ = ['thing']
 
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'],
                            virtualizables = ['frame'],
@@ -1455,7 +1455,7 @@
         )
 
         class Frame(object):
-            _virtualizable2_ = ['x']
+            _virtualizable_ = ['x']
 
         def main(n):
             f = Frame()
@@ -1481,7 +1481,7 @@
                             virtualizables=['frame'])
 
         class Frame(object):
-            _virtualizable2_ = ['x']
+            _virtualizable_ = ['x']
 
         def g(frame):
             driver2.jit_merge_point(frame=frame)
@@ -1517,10 +1517,10 @@
                             virtualizables=['subframe'])
 
         class Frame(object):
-            _virtualizable2_ = ['x']
+            _virtualizable_ = ['x']
 
         class SubFrame(object):
-            _virtualizable2_ = ['x']
+            _virtualizable_ = ['x']
 
         def g(subframe):
             driver2.jit_merge_point(subframe=subframe)
@@ -1544,6 +1544,7 @@
         res = self.meta_interp(f, [])
         assert res == f()
 
+
 class TestLLtype(ExplicitVirtualizableTests,
                  ImplicitVirtualizableTests,
                  LLJitMixin):
diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem
 from rpython.jit.metainterp.warmstate import wrap, unwrap
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rtyper import rvirtualizable2
+from rpython.rtyper import rvirtualizable
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.ootypesystem import ootype
 from rpython.rtyper.rclass import IR_IMMUTABLE_ARRAY, IR_IMMUTABLE
@@ -21,13 +21,13 @@
         self.cpu = cpu
         self.BoxArray = cpu.ts.BoxRef
         #
-        while 'virtualizable2_accessor' not in deref(VTYPEPTR)._hints:
+        while 'virtualizable_accessor' not in deref(VTYPEPTR)._hints:
             VTYPEPTR = cpu.ts.get_superclass(VTYPEPTR)
         self.VTYPEPTR = VTYPEPTR
         self.VTYPE = VTYPE = deref(VTYPEPTR)
         self.vable_token_descr = cpu.fielddescrof(VTYPE, 'vable_token')
         #
-        accessor = VTYPE._hints['virtualizable2_accessor']
+        accessor = VTYPE._hints['virtualizable_accessor']
         all_fields = accessor.fields
         static_fields = []
         array_fields = []
@@ -294,7 +294,7 @@
         (_, FUNCPTR) = ts.get_FuncType([self.VTYPEPTR], lltype.Void)
         funcptr = self.warmrunnerdesc.helper_func(
             FUNCPTR, force_virtualizable_if_necessary)
-        rvirtualizable2.replace_force_virtualizable_with_call(
+        rvirtualizable.replace_force_virtualizable_with_call(
             all_graphs, self.VTYPEPTR, funcptr)
         (_, FUNCPTR) = ts.get_FuncType([llmemory.GCREF], lltype.Void)
         self.clear_vable_ptr = self.warmrunnerdesc.helper_func(
@@ -308,7 +308,7 @@
         return virtualizable_box.getref(llmemory.GCREF)
 
     def is_vtypeptr(self, TYPE):
-        return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR)
+        return rvirtualizable.match_virtualizable_type(TYPE, self.VTYPEPTR)
 
 # ____________________________________________________________
 #
diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py
--- a/rpython/jit/metainterp/virtualref.py
+++ b/rpython/jit/metainterp/virtualref.py
@@ -39,7 +39,7 @@
         return True
 
     def replace_force_virtual_with_call(self, graphs):
-        # similar to rvirtualizable2.replace_force_virtualizable_with_call().
+        # similar to rvirtualizable.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
         c_is_virtual_ptr = None
         force_virtual_count = 0
diff --git a/rpython/jit/tl/tinyframe/tinyframe.py b/rpython/jit/tl/tinyframe/tinyframe.py
--- a/rpython/jit/tl/tinyframe/tinyframe.py
+++ b/rpython/jit/tl/tinyframe/tinyframe.py
@@ -214,7 +214,7 @@
                    virtualizables = ['self'])
 
 class Frame(object):
-    _virtualizable2_ = ['registers[*]', 'code']
+    _virtualizable_ = ['registers[*]', 'code']
     
     def __init__(self, code, arg=None):
         self = hint(self, access_directly=True, fresh_virtualizable=True)
diff --git a/rpython/jit/tl/tl.py b/rpython/jit/tl/tl.py
--- a/rpython/jit/tl/tl.py
+++ b/rpython/jit/tl/tl.py
@@ -11,7 +11,7 @@
     return t
 
 class Stack(object):
-    _virtualizable2_ = ['stackpos', 'stack[*]']
+    _virtualizable_ = ['stackpos', 'stack[*]']
 
     def __init__(self, size):
         self = hint(self, access_directly=True, fresh_virtualizable=True)
diff --git a/rpython/jit/tl/tla/tla.py b/rpython/jit/tl/tla/tla.py
--- a/rpython/jit/tl/tla/tla.py
+++ b/rpython/jit/tl/tla/tla.py
@@ -80,7 +80,7 @@
                       get_printable_location=get_printable_location)
 
 class Frame(object):
-    _virtualizable2_ = ['stackpos', 'stack[*]']
+    _virtualizable_ = ['stackpos', 'stack[*]']
     
     def __init__(self, bytecode):
         self.bytecode = bytecode
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -235,7 +235,7 @@
             if isinstance(s_x, annmodel.SomeInstance):
                 from rpython.flowspace.model import Constant
                 classdesc = s_x.classdef.classdesc
-                virtualizable = classdesc.read_attribute('_virtualizable2_',
+                virtualizable = classdesc.read_attribute('_virtualizable_',
                                                          Constant(None)).value
                 if virtualizable is not None:
                     flags = s_x.flags.copy()
diff --git a/rpython/rtyper/lltypesystem/rvirtualizable2.py b/rpython/rtyper/lltypesystem/rvirtualizable.py
rename from rpython/rtyper/lltypesystem/rvirtualizable2.py
rename to rpython/rtyper/lltypesystem/rvirtualizable.py
--- a/rpython/rtyper/lltypesystem/rvirtualizable2.py
+++ b/rpython/rtyper/lltypesystem/rvirtualizable.py
@@ -1,27 +1,12 @@
-from rpython.rtyper.rmodel import inputconst
-from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rtyper.lltypesystem.rclass import InstanceRepr, OBJECTPTR
-from rpython.rtyper.rvirtualizable2 import AbstractVirtualizable2InstanceRepr
+from rpython.rtyper.lltypesystem import llmemory
+from rpython.rtyper.lltypesystem.rclass import InstanceRepr
+from rpython.rtyper.rvirtualizable import AbstractVirtualizableInstanceRepr
 
 
-class Virtualizable2InstanceRepr(AbstractVirtualizable2InstanceRepr, InstanceRepr):
-
+class VirtualizableInstanceRepr(AbstractVirtualizableInstanceRepr,
+                                InstanceRepr):
     def _setup_repr_llfields(self):
         llfields = []
         if self.top_of_virtualizable_hierarchy:
             llfields.append(('vable_token', llmemory.GCREF))
         return llfields
-
-##  The code below is commented out because vtable_token is always
-##  initialized to NULL anyway.
-##
-##    def set_vable(self, llops, vinst, force_cast=False):
-##        if self.top_of_virtualizable_hierarchy:
-##            if force_cast:
-##                vinst = llops.genop('cast_pointer', [vinst], resulttype=self)
-##            cname = inputconst(lltype.Void, 'vable_token')
-##            cvalue = inputconst(llmemory.GCREF,
-##                                lltype.nullptr(llmemory.GCREF.TO))
-##            llops.genop('setfield', [vinst, cname, cvalue])
-##        else:
-##            self.rbase.set_vable(llops, vinst, force_cast=True)
diff --git a/rpython/rtyper/ootypesystem/rvirtualizable2.py b/rpython/rtyper/ootypesystem/rvirtualizable2.py
deleted file mode 100644
--- a/rpython/rtyper/ootypesystem/rvirtualizable2.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from rpython.rtyper.rmodel import inputconst
-from rpython.rtyper.ootypesystem import ootype
-from rpython.rtyper.ootypesystem.rclass import InstanceRepr, mangle, OBJECT
-from rpython.rtyper.rvirtualizable2 import AbstractVirtualizable2InstanceRepr
-
-VABLERTI = OBJECT
-
-
-class Virtualizable2InstanceRepr(AbstractVirtualizable2InstanceRepr, InstanceRepr):
-
-    def _setup_repr_llfields(self):
-        llfields = []
-        if self.top_of_virtualizable_hierarchy:
-            llfields.append(('vable_token', VABLERTI))
-        return llfields
-
-    def set_vable(self, llops, vinst, force_cast=False):
-        pass # TODO
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -78,20 +78,20 @@
 
     if classdef is None:
         unboxed = []
-        virtualizable2 = False
+        virtualizable = False
     else:
         unboxed = [subdef for subdef in classdef.getallsubdefs()
                           if subdef.classdesc.pyobj is not None and
                              issubclass(subdef.classdesc.pyobj, UnboxedValue)]
-        virtualizable2 = classdef.classdesc.read_attribute('_virtualizable2_',
+        virtualizable = classdef.classdesc.read_attribute('_virtualizable_',
                                                            Constant(False)).value
     config = rtyper.annotator.translator.config
     usetagging = len(unboxed) != 0 and config.translation.taggedpointers
 
-    if virtualizable2:
+    if virtualizable:
         assert len(unboxed) == 0
         assert gcflavor == 'gc'
-        return rtyper.type_system.rvirtualizable2.Virtualizable2InstanceRepr(rtyper, classdef)
+        return rtyper.type_system.rvirtualizable.VirtualizableInstanceRepr(rtyper, classdef)
     elif usetagging and rtyper.type_system.name == 'lltypesystem':
         # the UnboxedValue class and its parent classes need a
         # special repr for their instances
@@ -298,7 +298,7 @@
                         fieldname, base, self))
 
     def hook_access_field(self, vinst, cname, llops, flags):
-        pass        # for virtualizables; see rvirtualizable2.py
+        pass        # for virtualizables; see rvirtualizable.py
 
     def hook_setfield(self, vinst, fieldname, llops):
         if self.is_quasi_immutable(fieldname):
diff --git a/rpython/rtyper/rvirtualizable2.py b/rpython/rtyper/rvirtualizable.py
rename from rpython/rtyper/rvirtualizable2.py
rename to rpython/rtyper/rvirtualizable.py
--- a/rpython/rtyper/rvirtualizable2.py
+++ b/rpython/rtyper/rvirtualizable.py
@@ -4,17 +4,20 @@
 from rpython.rtyper.rclass import AbstractInstanceRepr, FieldListAccessor
 
 
-class AbstractVirtualizable2InstanceRepr(AbstractInstanceRepr):
+class AbstractVirtualizableInstanceRepr(AbstractInstanceRepr):
 
     def _super(self):
-        return super(AbstractVirtualizable2InstanceRepr, self)
+        return super(AbstractVirtualizableInstanceRepr, self)
 
     def __init__(self, rtyper, classdef):
         self._super().__init__(rtyper, classdef)
         classdesc = classdef.classdesc
         if '_virtualizable2_' in classdesc.classdict:
+            raise Exception("_virtualizable2_ is now called _virtualizable_, "
+                            "please rename")
+        if '_virtualizable_' in classdesc.classdict:
             basedesc = classdesc.basedesc
-            assert basedesc is None or basedesc.lookup('_virtualizable2_') is None
+            assert basedesc is None or basedesc.lookup('_virtualizable_') is None
             self.top_of_virtualizable_hierarchy = True
             self.accessor = FieldListAccessor()
         else:
@@ -28,13 +31,13 @@
 
     def _setup_repr(self):
         if self.top_of_virtualizable_hierarchy:
-            hints = {'virtualizable2_accessor': self.accessor}
+            hints = {'virtualizable_accessor': self.accessor}
             llfields = self._setup_repr_llfields()
             if llfields:
                 self._super()._setup_repr(llfields, hints = hints)
             else:
                 self._super()._setup_repr(hints = hints)
-            c_vfields = self.classdef.classdesc.classdict['_virtualizable2_']
+            c_vfields = self.classdef.classdesc.classdict['_virtualizable_']
             self.my_redirected_fields = self._parse_field_list(c_vfields.value,
                                                                self.accessor)
         else:
diff --git a/rpython/rtyper/test/test_rvirtualizable2.py b/rpython/rtyper/test/test_rvirtualizable.py
rename from rpython/rtyper/test/test_rvirtualizable2.py
rename to rpython/rtyper/test/test_rvirtualizable.py
--- a/rpython/rtyper/test/test_rvirtualizable2.py
+++ b/rpython/rtyper/test/test_rvirtualizable.py
@@ -1,7 +1,7 @@
 import py
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
-from rpython.rtyper.rvirtualizable2 import replace_force_virtualizable_with_call
+from rpython.rtyper.rvirtualizable import replace_force_virtualizable_with_call
 from rpython.rlib.jit import hint
 from rpython.flowspace.model import summary
 from rpython.rtyper.llinterp import LLInterpreter
@@ -10,7 +10,7 @@
 
 
 class V(object):
-    _virtualizable2_ = ['v']
+    _virtualizable_ = ['v']
     v = -12
     w = -62
 
@@ -22,12 +22,12 @@
     pass
 
 class VArray(object):
-    _virtualizable2_ = ['lst[*]']
+    _virtualizable_ = ['lst[*]']
 
     def __init__(self, lst):
         self.lst = lst
 class B(object):
-    _virtualizable2_ = ['v0']
+    _virtualizable_ = ['v0']
 
     x = "XX"
     
@@ -102,7 +102,7 @@
         class Base(object):
             pass
         class V(Base):
-            _virtualizable2_ = ['v1', 'v2[*]']
+            _virtualizable_ = ['v1', 'v2[*]']
         class W(V):
             pass
         #
@@ -115,7 +115,7 @@
         _, _, graph = self.gengraph(fn1, [int])
         v_inst = graph.getreturnvar()
         TYPE = self.gettype(v_inst)
-        accessor = TYPE._hints['virtualizable2_accessor']
+        accessor = TYPE._hints['virtualizable_accessor']
         assert accessor.TYPE == TYPE
         assert accessor.fields == {self.prefix + 'v1': IR_IMMUTABLE,
                                    self.prefix + 'v2': IR_IMMUTABLE_ARRAY}
@@ -129,7 +129,7 @@
         _, _, graph = self.gengraph(fn2, [int])
         w_inst = graph.getreturnvar()
         TYPE = self.gettype(w_inst)
-        assert 'virtualizable2_accessor' not in TYPE._hints
+        assert 'virtualizable_accessor' not in TYPE._hints
 
     def replace_force_virtualizable(self, rtyper, graphs):
         from rpython.annotator import model as annmodel
@@ -285,7 +285,7 @@
 
     def test_access_directly_method(self):
         class A:
-            _virtualizable2_ = ['v0']
+            _virtualizable_ = ['v0']
 
             def __init__(self, v):
                 self.v0 = v
@@ -315,7 +315,7 @@
         from rpython.rlib.jit import dont_look_inside
 
         class A:
-            _virtualizable2_ = ['x']
+            _virtualizable_ = ['x']
 
         def h(a):
             g(a)
diff --git a/rpython/rtyper/typesystem.py b/rpython/rtyper/typesystem.py
--- a/rpython/rtyper/typesystem.py
+++ b/rpython/rtyper/typesystem.py
@@ -22,7 +22,7 @@
                 return None
         if name in ('rclass', 'rpbc', 'rbuiltin', 'rtuple', 'rlist',
                     'rslice', 'rdict', 'rrange', 'rstr',
-                    'll_str', 'rbuilder', 'rvirtualizable2', 'rbytearray',
+                    'll_str', 'rbuilder', 'rvirtualizable', 'rbytearray',
                     'exceptiondata'):
             mod = load(name)
             if mod is not None:

From noreply at buildbot.pypy.org  Thu Jun 27 11:04:50 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Jun 2013 11:04:50 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: kill the ootype tests too
Message-ID: <20130627090450.3D6F21C2FED@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r65035:a6063d1bdd68
Date: 2013-06-27 11:04 +0200
http://bitbucket.org/pypy/pypy/changeset/a6063d1bdd68/

Log:	kill the ootype tests too

diff --git a/rpython/rtyper/test/test_rvirtualizable.py b/rpython/rtyper/test/test_rvirtualizable.py
--- a/rpython/rtyper/test/test_rvirtualizable.py
+++ b/rpython/rtyper/test/test_rvirtualizable.py
@@ -375,10 +375,3 @@
         assert res.inst_v == 42
         assert res.vable_token == lltype.nullptr(llmemory.GCREF.TO)
 
-class TestOOtype(OORtypeMixin, BaseTest):
-    prefix = 'o'
-    GETFIELD = 'oogetfield'
-    SETFIELD = 'oosetfield'    
-    
-    def gettype(self, v):
-        return v.concretetype

From noreply at buildbot.pypy.org  Thu Jun 27 11:20:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 11:20:09 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Update
Message-ID: <20130627092009.B14241C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r301:9d65f11cc049
Date: 2013-06-27 11:20 +0200
http://bitbucket.org/pypy/stmgc/changeset/9d65f11cc049/

Log:	Update

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -3,3 +3,6 @@
 *~
 c4/build-demo*
 c4/debug-demo*
+c4/release-demo*
+*.orig
+c4/test/__pycache__

From noreply at buildbot.pypy.org  Thu Jun 27 12:17:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 12:17:49 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tweak
Message-ID: <20130627101749.2D1051C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r302:e7249873dcda
Date: 2013-06-27 11:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/e7249873dcda/

Log:	Tweak

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -565,11 +565,11 @@
             p->h_tid &= ~GCFLAG_VISITED;
         }
         else {
-            g2l_mark_as_deleted(item);
+            G2L_LOOP_DELETE(item);
             stm_free(p, stmgc_size(p));
         }
 
-    } G2L_LOOP_END;
+    } G2L_LOOP_END_AND_COMPRESS;
 }
 
 static void free_all_unused_local_pages(void)
diff --git a/c4/lists.c b/c4/lists.c
--- a/c4/lists.c
+++ b/c4/lists.c
@@ -22,6 +22,22 @@
   memset(g2l, 0, sizeof(struct G2L));
 }
 
+void _g2l_compress(struct G2L *g2l)
+{
+  wlog_t *item;
+  struct G2L g2l_copy;
+  memset(&g2l_copy, 0, sizeof(struct G2L));
+
+  G2L_LOOP_FORWARD(*g2l, item)
+    {
+      g2l_insert(&g2l_copy, item->addr, item->val);
+
+    } G2L_LOOP_END;
+
+  g2l_delete_not_used_any_more(g2l);
+  *g2l = g2l_copy;
+}
+
 wlog_t *_g2l_find(char *entry, gcptr addr)
 {
   revision_t key = (revision_t)addr;
@@ -120,7 +136,7 @@
 {
     wlog_t *entry;
     G2L_FIND(*g2l, addr, entry, goto missing);
-    g2l_mark_as_deleted(entry);
+    entry->addr = NULL;
     return;
 
  missing:
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -47,9 +47,11 @@
 {                                                                       \
   struct { char **next; char **end; } _stack[TREE_DEPTH_MAX], *_stackp; \
   char **_next, **_end, *_entry;                                        \
+  long _deleted_factor = 0;                                             \
+  struct G2L *_g2l = &(g2l);                                            \
   /* initialization */                                                  \
   _stackp = _stack;      /* empty stack */                              \
-  _next = (g2l).toplevel.items + INITIAL;                               \
+  _next = _g2l->toplevel.items + INITIAL;                               \
   _end = _next _PLUS_ TREE_ARITY;                                       \
   /* loop */                                                            \
   while (1)                                                             \
@@ -78,8 +80,11 @@
           continue;                                                     \
         }                                                               \
       /* points to a wlog_t item */                                     \
-      if (((wlog_t *)_entry)->addr == NULL)   /* deleted entry */       \
+      if (((wlog_t *)_entry)->addr == NULL) {   /* deleted entry */     \
+          _deleted_factor += 3;                                         \
           continue;                                                     \
+      }                                                                 \
+      _deleted_factor -= 4;                                             \
       item = (wlog_t *)_entry;
 
 #define G2L_LOOP_FORWARD(g2l, item)                             \
@@ -87,6 +92,9 @@
 #define G2L_LOOP_BACKWARD(g2l, item)                            \
                        _G2L_LOOP(g2l, item, (TREE_ARITY-1), -)
 #define G2L_LOOP_END     } }
+#define G2L_LOOP_END_AND_COMPRESS                                       \
+                         } if (_deleted_factor > 9) _g2l_compress(_g2l); }
+#define G2L_LOOP_DELETE(item)  { (item)->addr = NULL; _deleted_factor += 6; }
 
 #define G2L_FIND(g2l, addr1, result, goto_not_found)            \
 {                                                               \
@@ -101,6 +109,7 @@
 }
 
 wlog_t *_g2l_find(char *entry, gcptr addr);
+void _g2l_compress(struct G2L *g2l);
 void g2l_insert(struct G2L *g2l, gcptr addr, gcptr val);
 void g2l_delete_item(struct G2L *g2l, gcptr addr);
 
@@ -111,11 +120,6 @@
     return 1;
 }
 
-static inline void g2l_mark_as_deleted(wlog_t *entry)
-{
-    entry->addr = NULL;
-}
-
 /************************************************************/
 
 /* The gcptrlist_xx functions are implemented as an array that grows

From noreply at buildbot.pypy.org  Thu Jun 27 14:21:57 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Thu, 27 Jun 2013 14:21:57 +0200 (CEST)
Subject: [pypy-commit] pypy py3k-memoryview: SubBuffers need a specific
	format and itemsize.
Message-ID: <20130627122157.6F7C81C0351@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: py3k-memoryview
Changeset: r65036:9467784455fc
Date: 2013-06-27 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/9467784455fc/

Log:	SubBuffers need a specific format and itemsize.

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -177,9 +177,9 @@
         raise OperationError(space.w_ValueError,
                              space.wrap("size must be zero or positive"))
     if isinstance(buffer, RWBuffer):
-        buffer = RWSubBuffer(buffer, offset, size)
+        buffer = RWSubBuffer(buffer, offset, size, 'B', 1)
     else:
-        buffer = SubBuffer(buffer, offset, size)
+        buffer = SubBuffer(buffer, offset, size, 'B', 1)
     return space.wrap(buffer)
 
 
@@ -285,12 +285,12 @@
 class SubBufferMixin(object):
     _mixin_ = True
 
-    def __init__(self, buffer, offset, size):
+    def __init__(self, buffer, offset, size, format, itemsize):
         self.buffer = buffer
         self.offset = offset
         self.size = size
-        self.format = 'B'
-        self.itemsize = 1
+        self.format = format
+        self.itemsize = itemsize
 
     def getlength(self):
         at_most = self.buffer.getlength() - self.offset
diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py
--- a/pypy/module/__builtin__/interp_memoryview.py
+++ b/pypy/module/__builtin__/interp_memoryview.py
@@ -57,9 +57,9 @@
             size = 0
         buf = self.buf
         if isinstance(buf, buffer.RWBuffer):
-            buf = buffer.RWSubBuffer(buf, start, size)
+            buf = buffer.RWSubBuffer(buf, start, size, buf.format, buf.itemsize)
         else:
-            buf = buffer.SubBuffer(buf, start, size)
+            buf = buffer.SubBuffer(buf, start, size, buf.format, buf.itemsize)
         return W_MemoryView(buf)
 
     def descr_buffer(self, space):

From noreply at buildbot.pypy.org  Thu Jun 27 14:24:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 14:24:39 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Progress
Message-ID: <20130627122439.AA3C01C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r303:08f615ac1415
Date: 2013-06-27 14:24 +0200
http://bitbucket.org/pypy/stmgc/changeset/08f615ac1415/

Log:	Progress

diff --git a/README.txt b/README.txt
--- a/README.txt
+++ b/README.txt
@@ -9,10 +9,11 @@
 
 The library interface is in "c4/stmgc.h".
 
-Progress (these revisions are roughly stable versions, pick the latest):
+Progress (these revisions are roughly stable versions, pick the last one):
 - 3aea86a96daf: last rev of "c3", the previous version
 - f1ccf5bbcb6f: first step, working but with no GC
 - 8da924453f29: minor collection seems to be working, no major GC
+- e7249873dcda: major collection seems to be working
 
 The file "c4/doc-objects.txt" contains some low-level explanations.
 

From noreply at buildbot.pypy.org  Thu Jun 27 14:59:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 14:59:47 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Redo the thread-local object.
Message-ID: <20130627125947.C81141C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r304:d91b7aa21484
Date: 2013-06-27 14:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/d91b7aa21484/

Log:	Redo the thread-local object.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1467,6 +1467,8 @@
   return pd;
 }
 
+__thread gcptr stm_thread_local_obj;
+
 int DescriptorInit(void)
 {
   if (GCFLAG_PREBUILT != PREBUILT_FLAGS)
@@ -1517,6 +1519,8 @@
       stm_private_rev_num = -d->my_lock;
       d->private_revision_ref = &stm_private_rev_num;
       d->read_barrier_cache_ref = &stm_read_barrier_cache;
+      stm_thread_local_obj = NULL;
+      d->thread_local_obj_ref = &stm_thread_local_obj;
       d->max_aborts = -1;
       d->tx_prev = NULL;
       d->tx_next = stm_tx_head;
@@ -1563,6 +1567,8 @@
 
     thread_descriptor = NULL;
 
+    stm_thread_local_obj = (gcptr)0xBB;   /* to detect misuses */
+
     g2l_delete(&d->public_to_private);
     assert(d->private_from_protected.size == 0);
     gcptrlist_delete(&d->private_from_protected);
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -135,6 +135,7 @@
   revision_t my_lock;
   gcptr *shadowstack;
   gcptr **shadowstack_end_ref;
+  gcptr *thread_local_obj_ref;
 
   NURSERY_FIELDS_DECL
 
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -339,10 +339,8 @@
         /* the roots pushed on the shadowstack */
         mark_roots(d->shadowstack, *d->shadowstack_end_ref);
 
-#if 0
         /* the thread-local object */
         visit(d->thread_local_obj_ref);
-#endif
 
         /* the current transaction's private copies of public objects */
         wlog_t *item;
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -521,6 +521,8 @@
 
     mark_young_roots(d);
 
+    visit_if_young(d->thread_local_obj_ref);
+
     mark_stolen_young_stubs(d);
 
     mark_private_from_protected(d);
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -103,6 +103,7 @@
     revision_t get_private_rev_num(void);
     revision_t get_start_time(void);
     void *my_stub_thread(void);
+    gcptr *addr_of_thread_local(void);
 
     int _stm_can_access_memory(char *);
     void stm_initialize_and_set_max_abort(int max_aborts);
@@ -245,6 +246,11 @@
         return (void *)thread_descriptor->public_descriptor;
     }
 
+    gcptr *addr_of_thread_local(void)
+    {
+        return &stm_thread_local_obj;
+    }
+
     void stm_initialize_and_set_max_abort(int max_aborts)
     {
         stm_initialize();
diff --git a/c3/test/test_threadlocal.py b/c4/test/test_threadlocal.py
copy from c3/test/test_threadlocal.py
copy to c4/test/test_threadlocal.py
--- a/c3/test/test_threadlocal.py
+++ b/c4/test/test_threadlocal.py
@@ -14,6 +14,18 @@
     lib.addr_of_thread_local()[0] = p
     major_collect()
     check_not_free(p)
+    assert lib.addr_of_thread_local()[0] == p
+
+def test_threadlocal_nursery():
+    p = nalloc(HDR + WORD)
+    lib.rawsetlong(p, 0, 654321)
+    lib.addr_of_thread_local()[0] = p
+    minor_collect()
+    check_nursery_free(p)
+    p1 = lib.addr_of_thread_local()[0]
+    assert p1 != ffi.NULL
+    check_not_free(p1)
+    assert lib.rawgetlong(p1, 0) == 654321
 
 def test_reset():
     p = oalloc(HDR + 1)

From noreply at buildbot.pypy.org  Thu Jun 27 15:28:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 15:28:38 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix threadlocals.
Message-ID: <20130627132838.7F5731C1067@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r305:5e0ebfe06495
Date: 2013-06-27 15:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/5e0ebfe06495/

Log:	Fix threadlocals.

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -842,6 +842,13 @@
   gcptrlist_clear(&d->list_of_read_objects);
   g2l_clear(&d->public_to_private);
 
+  /* 'old_thread_local_obj' contains the old value from stm_thread_local_obj,
+     but only when the transaction can be aborted; when it is inevitable
+     old_thread_local_obj will be reset to NULL. */
+  assert(d->thread_local_obj_ref = &stm_thread_local_obj);
+  stm_thread_local_obj = d->old_thread_local_obj;
+  d->old_thread_local_obj = NULL;
+
   /* release the lock */
   spinlock_release(d->public_descriptor->collection_lock);
 
@@ -908,6 +915,7 @@
   assert(d->num_private_from_protected_known_old == 0);
   assert(d->num_read_objects_known_old == 0);
   assert(!g2l_any_entry(&d->public_to_private));
+  assert(d->old_thread_local_obj == NULL);
 
   d->count_reads = 1;
   fxcache_clear(&d->recent_reads_cache);
@@ -923,6 +931,7 @@
   init_transaction(d);
   d->active = 1;
   d->setjmp_buf = buf;
+  d->old_thread_local_obj = stm_thread_local_obj;
   d->start_time = GetGlobalCurTime(d);
   update_reads_size_limit(d);
 }
@@ -1250,6 +1259,7 @@
 
   /* we cannot abort any more from here */
   d->setjmp_buf = NULL;
+  d->old_thread_local_obj = NULL;
   gcptrlist_clear(&d->list_of_read_objects);
 
   dprintf(("\n"
@@ -1282,6 +1292,7 @@
 static void make_inevitable(struct tx_descriptor *d)
 {
   d->setjmp_buf = NULL;
+  d->old_thread_local_obj = NULL;
   d->active = 2;
   d->reads_size_limit_nonatomic = 0;
   update_reads_size_limit(d);
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -136,6 +136,7 @@
   gcptr *shadowstack;
   gcptr **shadowstack_end_ref;
   gcptr *thread_local_obj_ref;
+  gcptr old_thread_local_obj;
 
   NURSERY_FIELDS_DECL
 
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -341,6 +341,7 @@
 
         /* the thread-local object */
         visit(d->thread_local_obj_ref);
+        visit(&d->old_thread_local_obj);
 
         /* the current transaction's private copies of public objects */
         wlog_t *item;
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -522,6 +522,7 @@
     mark_young_roots(d);
 
     visit_if_young(d->thread_local_obj_ref);
+    visit_if_young(&d->old_thread_local_obj);
 
     mark_stolen_young_stubs(d);
 
diff --git a/c4/test/test_threadlocal.py b/c4/test/test_threadlocal.py
--- a/c4/test/test_threadlocal.py
+++ b/c4/test/test_threadlocal.py
@@ -33,3 +33,46 @@
     lib.stm_finalize()
     lib.stm_initialize_tests(0)
     assert lib.addr_of_thread_local()[0] == ffi.NULL
+
+def test_threadlocal_commit():
+    p1 = oalloc(HDR)
+    lib.addr_of_thread_local()[0] = p1
+    #
+    @perform_transaction
+    def run(retry_counter):
+        assert retry_counter == 0
+        p2 = nalloc(HDR + 5 * WORD)
+        lib.setlong(p2, 4, -2891922)
+        lib.addr_of_thread_local()[0] = p2
+    #
+    p2b = lib.addr_of_thread_local()[0]
+    assert p2b != p1
+    assert lib.getlong(p2b, 4) == -2891922
+
+def test_threadlocal_abort(case=0):
+    p1 = oalloc(HDR + 5 * WORD)
+    lib.rawsetlong(p1, 4, 38972389)
+    lib.addr_of_thread_local()[0] = p1
+    #
+    @perform_transaction
+    def run(retry_counter):
+        if retry_counter == 0:
+            p2 = nalloc(HDR)
+            lib.addr_of_thread_local()[0] = p2
+            if case == 1:
+                minor_collect()
+            if case == 2:
+                major_collect()
+            abort_and_retry()
+        else:
+            check_not_free(p1)
+            assert lib.addr_of_thread_local()[0] == p1
+    #
+    assert lib.addr_of_thread_local()[0] == p1
+    assert lib.getlong(p1, 4) == 38972389
+
+def test_threadlocal_abort_minor():
+    test_threadlocal_abort(case=1)
+
+def test_threadlocal_abort_major():
+    test_threadlocal_abort(case=2)

From noreply at buildbot.pypy.org  Thu Jun 27 15:29:41 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Thu, 27 Jun 2013 15:29:41 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: added test-running to the
	squeak image
Message-ID: <20130627132941.711E11C1067@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r486:28ca33de654f
Date: 2013-06-27 15:28 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/28ca33de654f/

Log:	added test-running to the squeak image

diff too long, truncating to 2000 out of 80726 lines

diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
--- a/images/Squeak4.5-12568.changes
+++ b/images/Squeak4.5-12568.changes
@@ -36,4 +36,4 @@
 Workspace allInstances do: [:w | w topView delete].
 ReleaseBuilderFor4dot4 prepareNewBuild.
 Smalltalk snapshot: true andQuit: true.
-!

----End fileIn of a stream----!

----SNAPSHOT----{31 March 2013 . 3:27:34 pm} Squeak4.5-12327.image priorSource: 7430688!
!Installer methodsFor: 'squeakmap' stamp: 'fbs 1/28/2013 19:25' prior: 57597950!
packageAndVersionFrom: pkg
	| p |
	p := ReadStream on: pkg .
	^{(p upTo: $(). p upTo: $)} collect: [:s | s withBlanksTrimmed].! !

"Installer-Core"!
!Categorizer methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 16:58'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!ClassCategoryReader methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:21'!
scanFrom: aStream environment: anEnvironment
	"File in methods from the stream, aStream."
	| methodText |
	[methodText := aStream nextChunkText.
	 methodText size > 0] whileTrue:
		[class 
			compile: methodText 
			environment: anEnvironment
			classified: category
			withStamp: changeStamp 
			notifying: nil]! !
!ClassCommentReader methodsFor: 'as yet unclassified' stamp: 'cwp 6/20/2012 17:22'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!Metaclass methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:29'!
bindingOf: varName environment: anEnvironment 
	^ thisClass classBindingOf: varName environment: anEnvironment! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 20:03' prior: 22505876!
\\ aNumber 
	"Primitive. Take the receiver modulo the argument. The result is the
	remainder rounded towards negative infinity, of the receiver divided
	by the argument. Fail if the argument is 0. Fail if either the argument
	or the result is not a SmallInteger or a LargePositiveInteger less than
	2-to-the-30th (1073741824). Optional. See Object documentation whatIsAPrimitive."

	<primitive: 31>
	aNumber isInteger
		ifTrue:
			[| neg qr q r |
			neg := self negative == aNumber negative == false.
			qr := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: neg).
			q := qr first normalize.
			r := qr last normalize.
			^(q negative
				ifTrue: [r isZero not]
				ifFalse: [q isZero and: [neg]])
					ifTrue: [r + aNumber]
					ifFalse: [r]].
	^super \\ aNumber
	! !
!LargePositiveInteger methodsFor: 'converting' stamp: 'nice 1/27/2012 22:41' prior: 37616324!
asFloat
	"Answer a Float that best approximates the value of the receiver.
	This algorithm is optimized to process only the significant digits of a LargeInteger.
	And it does honour IEEE 754 round to nearest even mode in case of excess precision (see details below)."
	
	"How numbers are rounded in IEEE 754 default rounding mode:
	A shift is applied so that the highest 53 bits are placed before the floating point to form a mantissa.
	The trailing bits form the fraction part placed after the floating point.
	This fractional number must be rounded to the nearest integer.
	If fraction part is 2r0.1, exactly between two consecutive integers, there is a tie.
	The nearest even integer is chosen in this case.
	Examples (First 52bits of mantissa are omitted for brevity):
	2r0.00001 is rounded downward to 2r0
	2r1.00001 is rounded downward to 2r1
	2r0.1 is a tie and rounded to 2r0 (nearest even)
	2r1.1 is a tie and rounded to 2r10 (nearest even)
	2r0.10001 is rounded upward to 2r1
	2r1.10001 is rounded upward to 2r10
	Thus, if the next bit after floating point is 0, the mantissa is left unchanged.
	If next bit after floating point is 1, an odd mantissa is always rounded upper.
	An even mantissa is rounded upper only if the fraction part is not a tie."
	
	"Algorihm details:
	The floating point hardware can perform the rounding correctly with several excess bits as long as there is a single inexact operation.
	This can be obtained by splitting the mantissa plus excess bits in two part with less bits than Float precision.
	Note 1: the inexact flag in floating point hardware must not be trusted because in some cases the operations would be exact but would not take into account some bits that were truncated before the Floating point operations.
	Note 2: the floating point hardware is presumed configured in default rounding mode."
	
	| mantissa shift excess result n |

	"Check how many bits excess the maximum precision of a Float mantissa."
	excess := self highBitOfMagnitude - Float precision.
	excess > 7
		ifTrue:
			["Remove the excess bits but seven."
			mantissa := self bitShiftMagnitude: 7 - excess.
			shift := excess - 7.
			"An even mantissa with a single excess bit immediately following would be truncated.
			But this would not be correct if above shift has truncated some extra bits.
			Check this case, and round excess bits upper manually."
			((mantissa digitAt: 1) = 2r01000000 and: [self anyBitOfMagnitudeFrom: 1 to: shift])
				ifTrue: [mantissa := mantissa + 1]]
		ifFalse:
			[mantissa := self.
			shift := 0].

	"There will be a single inexact round off at last iteration"
	result := (mantissa digitAt: (n := mantissa digitLength)) asFloat.
	[(n := n - 1) > 0] whileTrue: [
		result := 256.0 * result + (mantissa digitAt: n) asFloat].
	^result timesTwoPower: shift.! !
!LargePositiveInteger methodsFor: 'private' stamp: 'nice 12/30/2012 14:25'!
primitiveQuo: anInteger 
	"Primitive. Divide the receiver by the argument and return the result.
	Round the result down towards zero to make it a whole integer. Fail if
	the argument is 0. Fail if either the argument or the result is not a
	SmallInteger or a LargePositiveInteger less than 2-to-the-30th (1073741824). Optional. See
	Object documentation whatIsAPrimitive."

	<primitive: 33>
	^nil! !
!LargePositiveInteger methodsFor: 'arithmetic' stamp: 'nice 12/30/2012 14:34'!
rem: aNumber 
	"Remainder defined in terms of quo:. See super rem:.
	This is defined only to speed up case of very large integers."

	(self primitiveQuo: aNumber)
		ifNotNil: [:quo | ^self - (quo * aNumber)].
	 aNumber isInteger
		ifTrue:
			[| ng rem |
			ng := self negative == aNumber negative == false.
			rem := (self digitDiv:
				(aNumber class == SmallInteger
					ifTrue: [aNumber abs]
					ifFalse: [aNumber])
				neg: ng) at: 2.
			^ rem normalize].
	^super rem: aNumber! !
!LargeNegativeInteger methodsFor: 'converting' stamp: 'nice 1/1/2013 15:42' prior: 37616204!
asFloat
	^super asFloat negated! !
!UndefinedObject methodsFor: 'class hierarchy' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:32'!
bindingOf: varName environment: anEnvironment
	^superclass bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'testing method dictionary' stamp: 'cwp 6/20/2012 17:30'!
classBindingOf: varName environment: anEnvironment
	^self bindingOf: varName environment: anEnvironment! !
!Behavior methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:37'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor
	"Postprocesses a literal scanned by Scanner scanToken (esp. xLitQuote).
	If scannedLiteral is not an association, answer it.
	Else, if it is of the form:
		nil->#NameOfMetaclass
	answer nil->theMetaclass, if any has that name, else report an error.
	Else, if it is of the form:
		#NameOfGlobalVariable->anythiEng
	answer the global, class, or pool association with that nameE, if any, else
	add it to Undeclared a answer the new Association."

	| key value |
	(scannedLiteral isVariableBinding)
		ifFalse: [^ scannedLiteral].
	key := scannedLiteral key.
	value := scannedLiteral value.
	key ifNil: "###<metaclass soleInstance name>"
		[(self bindingOf: value environment: anEnvironment) ifNotNil:
			[:assoc|
			(assoc value isKindOf: Behavior) ifTrue: 
				[^ nil->assoc value class]].
			 requestor notify: 'No such metaclass'.
			 ^false].
	(key isSymbol) ifTrue: "##<global var name>"
		[(self bindingOf: key environment: anEnvironment) ifNotNil:
			[:assoc | ^assoc].
		^ anEnvironment undeclared: key].
	requestor notify: '## must be followed by a non-local variable name'.
	^false

"	Form literalScannedAs: 14 notifying: nil 14
	Form literalScannedAs: #OneBitForm notiEfying: nil  OneBitForm
	Form literalScannedAs: ##OneBitForm notifying: nil  OneBitForm->a Form
	Form literalScannedAs: ##Form notifying: nil   Form->Form
	Form literalScannedAs: ###Form notifying: nil   nilE->Form class
"! !
!Fraction methodsFor: 'converting' stamp: 'nice 11/21/2011 22:34' prior: 37619655!
asFloat
	"Answer a Float that closely approximates the value of the receiver.
	This implementation will answer the closest floating point number to the receiver.
	In case of a tie, it will use the IEEE 754 round to nearest even mode.
	In case of overflow, it will answer +/- Float infinity."

	| a b mantissa exponent hasTruncatedBits lostBit n ha hb hm |
	a := numerator abs.
	b := denominator.	"denominator is always positive"
	ha := a highBitOfMagnitude.
	hb := b highBitOfMagnitude.
	
	"Number of bits to keep in mantissa plus one to handle rounding."
	n := 1 + Float precision.

	"If both numerator and denominator are represented exactly in floating point number,
	then fastest thing to do is to use hardwired float division."
	(ha < n and: [hb < n]) ifTrue: [^numerator asFloat / denominator asFloat].

	"Shift the fraction by a power of two exponent so as to obtain a mantissa with n bits.
	First guess is rough, the mantissa might have n+1 bits."
	exponent := ha - hb - n.
	exponent >= 0
		ifTrue: [b := b bitShift: exponent]
		ifFalse: [a := a bitShift: exponent negated].
	mantissa := a quo: b.
	hasTruncatedBits := a > (mantissa * b).
	hm := mantissa highBit.
	
	"Check for gradual underflow, in which case the mantissa will loose bits.
	Keep at least one bit to let underflow preserve the sign of zero."
	lostBit := Float emin - (exponent + hm - 1).
	lostBit > 0 ifTrue: [n := n - lostBit max: 1].

	"Remove excess bits in the mantissa."
	hm > n
		ifTrue:
			[exponent := exponent + hm - n.
			hasTruncatedBits := hasTruncatedBits or: [mantissa anyBitOfMagnitudeFrom: 1 to: hm - n].
			mantissa := mantissa bitShift: n - hm].

	"Check if mantissa must be rounded upward.
	The case of tie (mantissa odd & hasTruncatedBits not)
	will be handled by Integer>>asFloat."
	(hasTruncatedBits and: [mantissa odd])
		ifTrue: [mantissa := mantissa + 1].

	^ (self positive
			ifTrue: [mantissa asFloat]
			ifFalse: [mantissa asFloat negated])
		timesTwoPower: exponent! !
!Float methodsFor: 'arithmetic' stamp: 'nice 12/20/2012 23:16' prior: 20878776!
negated
	"Answer a Number that is the negation of the receiver.
	Implementation note: this version cares of negativeZero."

	^-1.0 * self! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 6/20/2012 17:21'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor
	^ self 
		compile: text 
		environment: anEnvironment 
		classified: category 
		withStamp: changeStamp 
		notifying: requestor 
		logSource: self acceptsLoggingOfCompilation! !
!ClassDescription methodsFor: 'compiling' stamp: 'cwp 12/27/2012 13:17'!
compile: text environment: anEnvironment classified: category withStamp: changeStamp notifying: requestor logSource: logSource
	| methodAndNode context methodNode |
	context := CompilationCue
		source: text
		class: self
		environment: anEnvironment
		category: category
		requestor: requestor.
	methodNode := self newCompiler compile: context ifFail: [^ nil].
	methodAndNode := CompiledMethodWithNode 
		generateMethodFromNode: methodNode 
		trailer: self defaultMethodTrailer.

	logSource ifTrue: [
		self logMethodSource: text forMethodWithNode: methodAndNode 
			inCategory: category withStamp: changeStamp notifying: requestor.
	].
	self addAndClassifySelector: methodAndNode selector withMethod: methodAndNode 
		method inProtocol: category notifying: requestor.
	self instanceSide noteCompilationOf: methodAndNode selector meta: self isClassSide.
	^ methodAndNode selector! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:47'!
bindingOf: varName environment: anEnvironment
	"Answer the binding of some variable resolved in the scope of the receiver"
	| aSymbol binding |
	aSymbol := varName asSymbol.

	"First look in classVar dictionary."
	binding := self classPool bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Next look in shared pools."
	self sharedPools do:[:pool | 
		binding := pool bindingOf: aSymbol.
		binding ifNotNil:[^binding].
	].

	"Next look in declared environment."
	binding := anEnvironment bindingOf: aSymbol.
	binding ifNotNil:[^binding].

	"Finally look higher up the superclass chain and fail at the end."
	superclass == nil
		ifTrue: [^ nil]
		ifFalse: [^ superclass bindingOf: aSymbol].

! !

"Kernel"!

ParseNode subclass: #Encoder
	instanceVariableNames: 'scopeTable nTemps supered requestor class selector literalStream selectorSet litIndSet litSet sourceRanges globalSourceRanges addedSelectorAndMethodClassLiterals optimizedSelectors cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Encoder commentStamp: 'cwp 12/26/2012 23:29' prior: 36323851!
I encode names and literals into tree nodes with byte codes for the compiler. Byte codes for literals are not assigned until the tree-sizing pass of the compiler, because only then is it known which literals are actually needed. I also keep track of sourceCode ranges during parsing and code generation so I can provide an inverse map for the debugger.!

Scanner subclass: #Parser
	instanceVariableNames: 'here hereType hereMark hereEnd prevMark prevEnd encoder requestor parseNode failBlock requestorOffset tempsMark doitFlag properties category queriedUnusedTemporaries cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Parser commentStamp: 'cwp 12/26/2012 23:34' prior: 38557958!
I parse Smalltalk syntax and create a MethodNode that is the root of the parse tree. I look one token ahead.!

Object subclass: #CompilationCue
	instanceVariableNames: 'source context receiver class environment category requestor'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

Object subclass: #Compiler
	instanceVariableNames: 'sourceStream requestor class category context parser cue'
	classVariableNames: ''
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
!Compiler commentStamp: 'cwp 12/26/2012 23:17' prior: 59257505!
The compiler accepts Smalltalk source code and compiles it with respect to a given class. The user of the compiler supplies a context so that temporary variables are accessible during compilation. If there is an error, a requestor (usually a kind of StringHolderController) is sent the message notify:at:in: so that the error message can be displayed. If there is no error, then the result of compilation is a MethodNode, which is the root of a parse tree whose nodes are kinds of ParseNodes. The parse tree can be sent messages to (1) generate code for a CompiledMethod (this is done for compiling methods or evaluating expressions); (2) pretty-print the code (for formatting); or (3) produce a map from object code back to source code (used by debugger program-counter selection). See also Parser, Encoder, ParseNode.!
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:34'!
init: aCue notifying: anObject
	"The use of the variable requestor is a bit confusing here. This is
	*not* the original requestor, which is available through the cue.
	It's the Parser instance that is using the encoder."

	self setCue: aCue.
	requestor := anObject.
	nTemps := 0.
	supered := false.
	self initScopeAndLiteralTables.
	cue getClass variablesAndOffsetsDo:
		[:variable "<String|CFieldDefinition>" :offset "<Integer|nil>" |
		offset isNil
			ifTrue: [scopeTable at: variable name put: (FieldNode new fieldDefinition: variable)]
			ifFalse: [scopeTable
						at: variable
						put: (offset >= 0
								ifTrue: [InstanceVariableNode new
											name: variable index: offset]
								ifFalse: [MaybeContextInstanceVariableNode new
											name: variable index: offset negated])]].
	cue context ~~ nil ifTrue:
		[| homeNode |
		 homeNode := self bindTemp: self doItInContextName.
		 "0th temp = aContext passed as arg"
		 cue context tempNames withIndexDo:
			[:variable :index|
			scopeTable
				at: variable
				put: (MessageAsTempNode new
						receiver: homeNode
						selector: #namedTempAt:
						arguments: (Array with: (self encodeLiteral: index))
						precedence: 3
						from: self)]].
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/26/2012 23:30'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy instance variables for methods that
	don't use cue yet"
	class := cue getClass.! !
!Dictionary methodsFor: '*Compiler' stamp: 'cwp 6/22/2012 09:17'!
bindingOf: varName ifAbsent: aBlock

	^self associationAt: varName ifAbsent: aBlock! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:37'!
init: sourceStream cue: aCue failBlock: aBlock

	self setCue: aCue.
	failBlock := aBlock.
	requestorOffset := 0.
	super scan: sourceStream.
	prevMark := hereMark := mark.
	self advance
! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/26/2012 23:41'!
parse: sourceStream cue: aCue noPattern: noPattern ifFail: aBlock 
	"Answer a MethodNode for the argument, sourceStream, that is the root of
	 a parse tree. Parsing is done with respect to the CompilationCue to 
	 resolve variables. Errors in parsing are reported to the cue's requestor; 
	 otherwise aBlock is evaluated. The argument noPattern is a Boolean that is
	 true if the the sourceStream does not contain a method header (i.e., for DoIts)."

	| methNode repeatNeeded myStream s p subSelection |
	myStream := sourceStream.
	[repeatNeeded := false.
	 p := myStream position.
	 s := myStream upToEnd.
	 myStream position: p.
	 subSelection := aCue requestor notNil and: [aCue requestor selectionInterval = (p + 1 to: p + s size)].
	 self encoder init: aCue notifying: self.
	 self init: myStream cue: aCue failBlock: [^ aBlock value].
	 doitFlag := noPattern.
	 failBlock:= aBlock.
	 [methNode := self method: noPattern context: cue context] 
		on: ReparseAfterSourceEditing 
		do:	[ :ex |
			repeatNeeded := true.
			myStream := subSelection
							ifTrue:
								[ReadStream
									on: cue requestor text string
									from: cue requestor selectionInterval first
									to: cue requestor selectionInterval last]
							ifFalse:
								[ReadStream on: cue requestor text string]].
	 repeatNeeded] whileTrue:
		[encoder := self encoder class new].
	methNode sourceText: s.
	^methNode
! !
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:35'!
setCue: aCue
	cue := aCue.
	
	"Also set legacy variables for methods that don't use cue yet."
	requestor := cue requestor.
	category := cue category.! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
class: aClass
	^ self 
		context: nil
		class: aClass
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
context: aContext class: aClass requestor: anObject
	^ self
		source: nil
		context: aContext
		receiver: nil
		class: aClass
		environment: (aClass ifNotNil: [aClass environment])
		category: nil
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aTextOrStream class: aClass environment: anEnvironment category: aString requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		receiver: nil
		class: aClass
		environment: anEnvironment
		category: aString
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:53'!
source: aTextOrStream context: aContext class: aClass category: aString requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		receiver: (aContext ifNotNil: [aContext receiver])
		class: aClass 
		environment: (aClass ifNotNil: [aClass environment])
		category: aString 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream context: aContext class: aClass requestor: anObject
	^ self 
		source: aTextOrStream 
		context: aContext 
		class: aClass 
		category: nil 
		requestor: anObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:55'!
source: aTextOrStream context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	^ self basicNew
		initializeWithSource: aTextOrStream 
		context: aContext 
		receiver: recObject 
		class: aClass 
		environment: anEnvironment 
		category: aString 
		requestor: reqObject! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:16'!
source: aString environment: anEnvironment
	^ self 
		source: aString
		context: nil
		receiver: nil
		class: UndefinedObject
		environment: anEnvironment
		category: nil
		requestor: nil! !
!CompilationCue class methodsFor: 'instance creation' stamp: 'cwp 12/26/2012 23:54'!
source: aTextOrStream requestor: anObject
	^ self
		source: aTextOrStream
		context: nil
		class: nil
		requestor: anObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/20/2012 09:39'!
bindingOf: aSymbol
	^ class bindingOf: aSymbol environment: environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
category
	^ category! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 12/26/2012 23:19'!
context
	^ context! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
environment
	^ environment! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
getClass
	^ class! !
!CompilationCue methodsFor: 'initialization' stamp: 'cwp 12/26/2012 23:16'!
initializeWithSource: aTextOrString context: aContext receiver: recObject class: aClass environment: anEnvironment category: aString requestor: reqObject
	self initialize.
	source := aTextOrString isStream ifTrue: [aTextOrString contents] ifFalse: [aTextOrString].
	context := aContext.
	receiver := recObject.
	class := aClass.
	environment := anEnvironment.
	category := aString.
	requestor := reqObject! !
!CompilationCue methodsFor: 'binding' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: anObject notifying: anEncoder
	^ class literalScannedAs: anObject environment: environment notifying: anEncoder! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
receiver
	^ receiver! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:16'!
requestor
	^ requestor! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:15'!
source
	^ source! !
!CompilationCue methodsFor: 'accessing' stamp: 'cwp 6/19/2012 11:44'!
sourceStream
	^ source readStream! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 6/20/2012 17:25'!
evaluate: aString environment: anEnvironment
	^ self 
		evaluate: aString 
		environment: anEnvironment 
		logged: false! !
!Compiler class methodsFor: 'evaluating' stamp: 'cwp 12/27/2012 12:36'!
evaluate: aString environment: anEnvironment logged: aBoolean
	| cue |
	cue := CompilationCue
		source: aString
		environment: anEnvironment.
		
	^ self new
		evaluate: aString
		cue: cue
		ifFail: [^ nil]
		logged: aBoolean! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 13:18'!
compile: aCue ifFail: failBlock 
	"Answer a MethodNode. If the MethodNode can not be created, notify 
	the requestor in the contxt. If the requestor is nil, evaluate failBlock 
	instead. The MethodNode is the root  of a parse tree. It can be told 
	to generate a CompiledMethod to be installed in the method dictionary 
	of the class specified by the context."
	
	self setCue: aCue.
	self source: cue source.
	^self
		translate: sourceStream
		noPattern: false
		ifFail: failBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:06'!
evaluate: textOrStream cue: aCue ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	a method. Finally, the compiled method is invoked from here via 	withArgs:executeMethod:, hence the system no longer creates Doit method
	litter on errors."

	| methodNode method value toLog itsSelection itsSelectionString |
	self setCue: aCue.
	self source: textOrStream.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].

	method := self interactive
				ifTrue: [methodNode generateWithTempNames]
				ifFalse: [methodNode generate].

	value := cue receiver
				withArgs: (cue context ifNil: [#()] ifNotNil: [{cue context}])
				executeMethod: method.

	logFlag ifTrue:
		[toLog := ((cue requestor respondsTo: #selection)  
			and:[(itsSelection := cue requestor selection) notNil
			and:[(itsSelectionString := itsSelection asString) isEmptyOrNil not]])
				ifTrue:[itsSelectionString]
				ifFalse:[sourceStream contents].
		SystemChangeNotifier uniqueInstance evaluated: toLog context: cue context].
	^ value
! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:20'!
setCue: aCue
	cue := aCue.
	
	"Set legacy instance variables for methods that don't use cue yet."
	requestor := cue requestor.
	class := cue getClass.
	category := cue category.
	context := cue context.! !
!Compiler methodsFor: 'private' stamp: 'cwp 6/19/2012 21:58'!
source: textOrStream
	sourceStream := (textOrStream isKindOf: PositionableStream)
		ifTrue: [ textOrStream ]
		ifFalse: [ ReadStream on: textOrStream asString ]! !

"Compiler"!
!SmartRefStream class methodsFor: 'i/o' stamp: 'cwp 6/20/2012 17:42'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!SmartRefStream methodsFor: 'read write' stamp: 'cwp 6/20/2012 17:41'!
scanFrom: aByteStream environment: anEnvironment
	^ self scanFrom: aByteStream! !
!ImageSegment methodsFor: 'fileIn/Out' stamp: 'cwp 6/20/2012 17:23'!
scanFrom: aStream environment: anEnvironment
	^ self scanFrom: aStream! !
!PseudoClass methodsFor: 'printing' stamp: 'cwp 6/22/2012 15:39'!
literalScannedAs: scannedLiteral environment: anEnvironment notifying: requestor 
	^ scannedLiteral! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| aString newTranslations assoc currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[Locale
		currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:26'!
scanFrom: aStream environment: anEnvironment
	"Read a definition of dictionary.  
	Make sure current locale corresponds my locale id"
	| newTranslations currentPlatform |
	newTranslations := Dictionary new.
	currentPlatform := Locale currentPlatform.
	[| aString assoc |
	Locale currentPlatform: (Locale localeID: id).
	[aString := aStream nextChunk withSqueakLineEndings.
	aString size > 0] whileTrue: 
		[assoc := Compiler evaluate: aString environment: anEnvironment.
		assoc value = ''
			ifTrue: [self class registerPhrase: assoc key]
			ifFalse: [newTranslations add: assoc]]]
		ensure: [Locale currentPlatform: currentPlatform].
	self mergeTranslations: newTranslations! !
!ObjectScanner methodsFor: 'scanning' stamp: 'cwp 6/20/2012 17:39'!
scanFrom: aByteStream environment: anEnvironment
	"This should probably be reimplemented using an environment
	for compilation. For now, don't change anything"
	^ self scanFrom: aByteStream! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 09:16'!
bindingOf: varName ifAbsent: aBlock
	"SystemDictionary includes Symbols only"
	^super bindingOf: varName asSymbol ifAbsent: aBlock! !
!SystemDictionary methodsFor: 'accessing' stamp: 'cwp 6/22/2012 15:48'!
undeclared
	^ self at: #Undeclared! !

"System"!
!ExceptionTests methodsFor: 'testing-outer' stamp: 'fbs 1/1/2013 22:14' prior: 40840955!
expectedFailures
	^ #().! !

"Tests"!

ReleaseBuilder subclass: #ReleaseBuilderFor4dot5
	instanceVariableNames: ''
	classVariableNames: ''
	poolDictionaries: ''
	category: 'ReleaseBuilder'!
!ReleaseBuilderFor4dot5 commentStamp: 'fbs 1/1/2013 20:25' prior: 0!
The release builder for Squeak 4.5!
!ReleaseBuilder class methodsFor: 'scripts' stamp: 'fbs 12/31/2012 20:43'!
transferCurrentPackagesAsUser: username password: password
	"Copy the packages currently loaded in the image from the trunk repository to my releaseRepository."
	| trunkRep releaseRep |
	trunkRep := self trunkRepository.
	releaseRep := self releaseRepository
		user: username;
		password: password;
		yourself.
	MCWorkingCopy allManagers do:
		[ : eachWorkingCopy | eachWorkingCopy ancestors do:
			[ : eachVersionInfo | (releaseRep includesVersionNamed: eachVersionInfo versionName) ifFalse:
				[ (trunkRep versionWithInfo: eachVersionInfo)
					ifNil: [ Warning signal: eachVersionInfo name , ' not found in ', trunkRep ]
					ifNotNilDo: [ : ver | releaseRep storeVersion: ver ] ] ] ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
openWelcomeWorkspaces
	TheWorldMainDockingBar instance
		showWelcomeText: #squeakUserInterface
		label: 'Squeak User Interface'
		in: (40 @ 40 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #workingWithSqueak
		label: 'Working With Squeak'
		in: (80 @ 80 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #licenseInformation
		label: 'License Information'
		in: (120 @ 120 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeFutureDirections
		label: 'Future Directions'
		in: (160 @ 160 extent: 500 @ 300).
	TheWorldMainDockingBar instance
		showWelcomeText: #welcomeToSqueak
		label: 'Welcome to Squeak 4.5'
		in: (200 @ 200 extent: 500 @ 300)! !
!ReleaseBuilderFor4dot5 class methodsFor: 'scripts' stamp: 'fbs 1/1/2013 20:22'!
prepareNewBuild
	super prepareNewBuild.
	MCMockPackageInfo initialize.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:24'!
releaseRepository
	"At release time, change 'trunk' to 'squeak45'."
	^ MCHttpRepository
		location: 'http://source.squeak.org/trunk'
		user: 'squeak'
		password: 'squeak'! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:22'!
setDisplayExtent: extent
"Uncomment next line when the primitives become available in the Squeak VM."
"	DisplayScreen hostWindowSize: extent."
	Display extent = extent ifFalse: [ Warning signal: 'Display extent not set to ', extent ]! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
setPreferences
	Preferences 
		installBrightWindowColors ;
		setPreference: #scrollBarsWithoutMenuButton toValue: true ;
		setPreference: #swapMouseButtons toValue: true ;
		setPreference: #annotationPanes toValue: true ;
		setPreference: #showSplitterHandles toValue: false ;
		setPreference: #showBoundsInHalo toValue: true ;
		setPreference: #alternateHandlesLook toValue: false ;
		setPreference: #roundedMenuCorners toValue: false ;
		setPreference: #roundedWindowCorners toValue: false.
	PluggableButtonMorph roundedButtonCorners: false.
	FillInTheBlankMorph roundedDialogCorners: false.
	Workspace shouldStyle: false.
	NetNameResolver enableIPv6: true.! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
switchToNewRepository
	| old44Repository |
	MCMcmUpdater defaultUpdateURL: self releaseRepository description.
	old44Repository := MCRepositoryGroup default repositories
				detect: [:each | each description includesSubString: 'squeak44'] ifNone: [nil].
	old44Repository
		ifNotNil: [MCRepositoryGroup default removeRepository: old44Repository].
	MCRepositoryGroup default addRepository: self releaseRepository! !
!ReleaseBuilderFor4dot5 class methodsFor: 'private' stamp: 'fbs 1/1/2013 20:23'!
versionString
	^ 'Squeak4.5'.! !

ReleaseBuilder class removeSelector: #transferCurrentPackages!

"ReleaseBuilder"!
!Environment class methodsFor: 'as yet unclassified' stamp: 'cwp 1/1/2013 18:52' prior: 40834114!
initialize
	self install! !

"Environments"!
!Parser methodsFor: 'private' stamp: 'cwp 12/26/2012 23:59' prior: 52081878!
initPattern: aString notifying: req return: aBlock

	| result |
	self
		init: (ReadStream on: aString asString)
		cue: (CompilationCue source: aString requestor: req)
		failBlock: [^nil].
	encoder := self.
	result := aBlock value: (self pattern: false inContext: nil).
	encoder := failBlock := nil.  "break cycles"
	^result! !
!Parser methodsFor: 'public access' stamp: 'cwp 12/27/2012 00:01' prior: 34175471!
parse: sourceStream class: class category: aCategory noPattern: noPattern context: aContext notifying: req ifFail: aBlock 
	| c |
	c := CompilationCue
			source: sourceStream
			context: aContext
			class: class
			category: aCategory
			requestor: req.
	^ self 
		parse: sourceStream 
		cue: c 
		noPattern: noPattern 
		ifFail: aBlock! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:11' prior: 34183963!
evaluate: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code into
	 a method. If aContext is not nil, the text can refer to temporaries in that
	 context (the Debugger uses this). If aRequestor is not nil, then it will receive
	 a notify:at: message before the attempt to evaluate is aborted. Finally, the 
	 compiled method is invoked from here via withArgs:executeMethod:, hence
	 the system no longer creates Doit method litter on errors."
	
	| theClass |
	theClass := ((aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class).
	self setCue: (CompilationCue
		source: textOrStream
		context: aContext
		receiver: receiver
		class: theClass
		environment: theClass environment
		category: nil
		requestor: aRequestor).
	^ self evaluate: textOrStream cue: cue ifFail: failBlock logged: logFlag! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 09:17' prior: 34185488!
from: textOrStream class: aClass classified: aCategory context: aContext notifying: req
	self source: textOrStream.
	self setCue: 
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			category: aCategory
			requestor: req)! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/26/2012 23:55' prior: 50781309!
from: textOrStream class: aClass context: aContext notifying: req
	self source: textOrStream.
	self setCue:
		(CompilationCue
			source: textOrStream
			context: aContext
			class: aClass
			requestor: req)
! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/27/2012 09:41' prior: 50996506!
init: aClass context: aContext notifying: anObject
	| c |
	c := CompilationCue
		context: aContext 
		class: aClass 
		requestor: nil.
	self init: c notifying: anObject! !
!Encoder methodsFor: 'initialize-release' stamp: 'cwp 12/26/2012 23:58' prior: 39061698!
temps: tempVars literals: lits class: cl 
	"Initialize this encoder for decompilation."

	self setCue: (CompilationCue class: cl).
	supered := false.
	nTemps := tempVars size.
	tempVars do: [:node | scopeTable at: node name put: node].
	literalStream := WriteStream on: (Array new: lits size).
	literalStream nextPutAll: lits.
	sourceRanges := Dictionary new: 32.
	globalSourceRanges := OrderedCollection new: 32.! !

"Compiler"!
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:48' prior: 36026010!
addClassVarName: aString 
	"Add the argument, aString, as a class variable of the receiver.
	Signal an error if the first character of aString is not capitalized,
	or if it is already a variable named in the class."
	| symbol oldState |
	oldState := self copy.
	aString first canBeGlobalVarInitial
		ifFalse: [^self error: aString, ' class variable name should be capitalized; proceed to include anyway.'].
	symbol := aString asSymbol.
	self withAllSubclasses do: 
		[:subclass | 
		(self canFindWithoutEnvironment: symbol) ifTrue: [
			(DuplicateVariableError new)
				superclass: superclass; "fake!!!!!!"
				variable: aString;
				signal: aString, ' is already defined']].
	classPool == nil ifTrue: [classPool := Dictionary new].
	(classPool includesKey: symbol) ifFalse: 
		["Pick up any refs in Undeclared"
		classPool declare: symbol from: environment undeclared.
		SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: oldState to: self]! !
!Class methodsFor: 'compiling' stamp: 'cwp 6/20/2012 09:48' prior: 54782024!
bindingOf: varName
	^ self bindingOf: varName environment: self environment! !
!Class methodsFor: 'organization' stamp: 'cwp 6/25/2012 18:25' prior: 54785804!
category
	"Answer the system organization category for the receiver. First check whether the
	category name stored in the ivar is still correct and only if this fails look it up
	(latter is much more expensive)"

	category ifNotNil: [ :symbol |
		((self environment organization listAtCategoryNamed: symbol) includes: self name)
			ifTrue: [ ^symbol ] ].
	category := self environment organization categoryOfElement: self name.
	^category! !
!Class methodsFor: 'initialize-release' stamp: 'cwp 6/22/2012 15:49' prior: 36027730!
declare: varString 
	"Declare class variables common to all instances. Answer whether 
	recompilation is advisable."

	| newVars conflicts |
	
	newVars := 
		(Scanner new scanFieldNames: varString)
			collect: [:x | x asSymbol].
	newVars do:
		[:var | var first canBeGlobalVarInitial
			ifFalse: [self error: var, ' class variable name should be capitalized; proceed to include anyway.']].
	conflicts := false.
	classPool == nil 
		ifFalse: [(classPool keys reject: [:x | newVars includes: x]) do: 
					[:var | self removeClassVarName: var]].
	(newVars reject: [:var | self classPool includesKey: var])
		do: [:var | "adding"
			"check if new vars defined elsewhere"
			(self canFindWithoutEnvironment: var) ifTrue: [
					(DuplicateVariableError new)
						superclass: superclass; "fake!!!!!!"
						variable: var;
						signal: var, ' is already defined'.
					conflicts := true]].
	newVars size > 0
		ifTrue: 
			[classPool := self classPool.
			"in case it was nil"
			newVars do: [:var | classPool declare: var from: environment undeclared]].
	^conflicts! !
!Class methodsFor: 'class variables' stamp: 'cwp 6/22/2012 15:49' prior: 54802475!
removeClassVarName: aString 
	"Remove the class variable whose name is the argument, aString, from 
	the names defined in the receiver, a class. Create an error notification if 
	aString is not a class variable or if it is still being used in the code of 
	the class."

	| aSymbol |
	aSymbol := aString asSymbol.
	(classPool includesKey: aSymbol)
		ifFalse: [^self error: aString, ' is not a class variable'].
	self withAllSubclasses do:[:subclass |
		(Array with: subclass with: subclass class) do:[:classOrMeta |
			(classOrMeta whichSelectorsReferTo: (classPool associationAt: aSymbol))
				isEmpty ifFalse: [
					InMidstOfFileinNotification signal ifTrue: [
						Transcript cr; show: self name, ' (' , aString , ' is Undeclared) '.
						^ environment undeclared declare: aSymbol from: classPool].
					(self confirm: (aString,' is still used in code of class ', classOrMeta name,
						'.\Is it okay to move it to Undeclared?') withCRs)
						ifTrue:[^Undeclared declare: aSymbol from: classPool]
						ifFalse:[^self]]]].
	classPool removeKey: aSymbol.
	classPool isEmpty ifTrue: [classPool := nil].
! !
!Class methodsFor: 'class name' stamp: 'cwp 6/22/2012 15:49' prior: 54796206!
rename: aString 
	"The new name of the receiver is the argument, aString."

	| oldName newName |
	(newName := aString asSymbol) = (oldName := self name)
		ifTrue: [^ self].
	(self environment includesKey: newName)
		ifTrue: [^ self error: newName , ' already exists'].
	(environment undeclared includesKey: newName)
		ifTrue: [self inform: 'There are references to, ' , aString printString , '
from Undeclared. Check them after this change.'].
	name := newName.
	self environment renameClass: self from: oldName! !
!ClassBuilder methodsFor: 'class definition' stamp: 'cwp 6/22/2012 01:05' prior: 39054430!
name: className inEnvironment: env subclassOf: newSuper type: type instanceVariableNames: instVarString classVariableNames: classVarString poolDictionaries: poolString category: category unsafe: unsafe
	"Define a new class in the given environment.
	If unsafe is true do not run any validation checks.
	This facility is provided to implement important system changes."
	| oldClass instVars classVars copyOfOldClass newClass |
 
	environ := env.
	instVars := Scanner new scanFieldNames: instVarString.
	classVars := (Scanner new scanFieldNames: classVarString) collect: [:x | x asSymbol].

	"Validate the proposed name"
	unsafe ifFalse:[(self validateClassName: className) ifFalse:[^nil]].
	oldClass := env at: className ifAbsent:[nil].
	oldClass isBehavior 
		ifFalse: [oldClass := nil]  "Already checked in #validateClassName:"
		ifTrue: [
			copyOfOldClass := oldClass copy.
			copyOfOldClass superclass addSubclass: copyOfOldClass].
	
	
	[ | newCategory needNew force organization oldCategory |
	unsafe ifFalse:[
		"Run validation checks so we know that we have a good chance for recompilation"
		(self validateSuperclass: newSuper forSubclass: oldClass) ifFalse:[^nil].
		(self validateInstvars: instVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateClassvars: classVars from: oldClass forSuper: newSuper) ifFalse:[^nil].
		(self validateSubclassFormat: type from: oldClass forSuper: newSuper extra: instVars size) ifFalse:[^nil]].

	"See if we need a new subclass"
	needNew := self needsSubclassOf: newSuper type: type instanceVariables: instVars from: oldClass.
	needNew == nil ifTrue:[^nil]. "some error"

	(needNew and:[unsafe not]) ifTrue:[
		"Make sure we don't redefine any dangerous classes"
		(self tooDangerousClasses includes: oldClass name) ifTrue:[
			self error: oldClass name, ' cannot be changed'.
		].
		"Check if the receiver should not be redefined"
		(oldClass ~~ nil and:[oldClass shouldNotBeRedefined]) ifTrue:[
			self notify: oldClass name asText allBold, 
						' should not be redefined. \Proceed to store over it.' withCRs]].

	needNew ifTrue:[
		"Create the new class"
		newClass := self 
			newSubclassOf: newSuper 
			type: type 
			instanceVariables: instVars
			from: oldClass.
		newClass == nil ifTrue:[^nil]. "Some error"
		newClass setName: className.
		newClass environment: environ.
	] ifFalse:[
		"Reuse the old class"
		newClass := oldClass.
	].

	"Install the class variables and pool dictionaries... "
	force := (newClass declare: classVarString) | (newClass sharing: poolString).

	"... classify ..."
	newCategory := category asSymbol.
	organization := environ ifNotNil:[environ organization].
	oldClass isNil ifFalse: [oldCategory := (organization categoryOfElement: oldClass name) asSymbol].
	organization classify: newClass name under: newCategory suppressIfDefault: true.
	
	"... recompile ..."
	newClass := self recompile: force from: oldClass to: newClass mutate: false.

	"... export if not yet done ..."
	(environ at: newClass name ifAbsent:[nil]) == newClass ifFalse:[
		[environ at: newClass name put: newClass]
			on: AttemptToWriteReadOnlyGlobal do:[:ex| ex resume: true].
		environ flushClassNameCache.
	].


	newClass doneCompiling.
	"... notify interested clients ..."
	oldClass isNil ifTrue: [
		SystemChangeNotifier uniqueInstance classAdded: newClass inCategory: newCategory.
		^ newClass].
	newCategory ~= oldCategory 
		ifTrue: [SystemChangeNotifier uniqueInstance class: newClass recategorizedFrom: oldCategory to: category]
		ifFalse: [SystemChangeNotifier uniqueInstance classDefinitionChangedFrom: copyOfOldClass to: newClass.].
] ensure: 
		[copyOfOldClass ifNotNil: [copyOfOldClass superclass removeSubclass: copyOfOldClass].
		Behavior flushObsoleteSubclasses.
		].
	^newClass! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 22:57' prior: 18572019!
superclass: newSuper
	subclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat 
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class."
	| env |
	env := EnvironmentRequest signal ifNil: [newSuper environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: newSuper
		type: newSuper typeOfClass
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:01' prior: 50629912!
superclass: aClass
	variableByteSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable byte-sized nonpointer variables."
	| oldClassOrNil actualType env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a byte subclass of a class with named fields'].
	(aClass isVariable and: [aClass isWords])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with word fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a byte subclass of a class with pointer fields'].
	oldClassOrNil := aClass environment at: t ifAbsent:[nil].
	actualType := (oldClassOrNil notNil
				   and: [oldClassOrNil typeOfClass == #compiledMethod])
					ifTrue: [#compiledMethod]
					ifFalse: [#bytes].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: actualType
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:03' prior: 18573442!
superclass: aClass
	variableSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable pointer variables."
	
	| env |
	aClass isBits ifTrue: 
		[^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #variable
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18574098!
superclass: aClass
	variableWordSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class in which the subclass is to 
	have indexable word-sized nonpointer variables."
	| env |
	(aClass instSize > 0)
		ifTrue: [^self error: 'cannot make a word subclass of a class with named fields'].
	(aClass isVariable and: [aClass isBytes])
		ifTrue: [^self error: 'cannot make a word subclass of a class with byte fields'].
	(aClass isVariable and: [aClass isPointers])
		ifTrue: [^self error: 'cannot make a word subclass of a class with pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #words
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !
!ClassBuilder methodsFor: 'public' stamp: 'cwp 6/19/2012 23:04' prior: 18575028!
superclass: aClass
	weakSubclass: t instanceVariableNames: f 
	classVariableNames: d poolDictionaries: s category: cat
	"This is the standard initialization message for creating a new class as a 
	subclass of an existing class (the receiver) in which the subclass is to 
	have weak indexable pointer variables."
	| env |
	aClass isBits 
		ifTrue: [^self error: 'cannot make a pointer subclass of a class with non-pointer fields'].
	env := EnvironmentRequest signal ifNil: [aClass environment].
	^self 
		name: t
		inEnvironment: env
		subclassOf: aClass
		type: #weak
		instanceVariableNames: f
		classVariableNames: d
		poolDictionaries: s
		category: cat! !

"Kernel"!
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 59135029!
ambiguousSelector: aString inRange: anInterval
	| correctedSelector userSelection offset intervalWithOffset |
	
	self interactive ifFalse: [
		"In non interactive mode, compile with backward comapatibility: $- is part of literal argument"
		Transcript cr; store: encoder classEncoding; nextPutAll:#'>>';store: encoder selector; show: ' would send ' , token , '-'.
		^super ambiguousSelector: aString inRange: anInterval].
	
	"handle the text selection"
	userSelection := cue requestor selectionInterval.
	intervalWithOffset := anInterval first + requestorOffset to: anInterval last + requestorOffset.
	cue requestor selectFrom: intervalWithOffset first to: intervalWithOffset last.
	cue requestor select.

	"Build the menu with alternatives"
	correctedSelector := AmbiguousSelector 
			signalName: aString
			inRange: intervalWithOffset.
	correctedSelector ifNil: [^self fail].

	"Execute the selected action"
	offset := self substituteWord: correctedSelector wordInterval: intervalWithOffset offset: 0.
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + offset.
	token := (correctedSelector readStream upTo: Character space) asSymbol! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:21' prior: 38558136!
collectTemporaryDeclarationsFrom: methodNode
	| tempsMarks str |
	tempsMarks := OrderedCollection new.
	str := cue requestor text asString.
	methodNode accept: (ParseNodeEnumerator
		ofBlock: [ :aNode | 
			| mark |
			(aNode class canUnderstand: #tempsMark) 
				ifTrue: 
					[mark := aNode tempsMark.
					(mark notNil and: [ mark between: 1 and: str size ] and: [ (str at: mark) = $| ])
						ifTrue: [ tempsMarks addLast: aNode ]]]).
	(tempsMark notNil and: [ tempsMark between: 1 and: str size ] and: [ (str at: tempsMark) = $| ])
						ifTrue: [ tempsMarks addLast: self ].
	^ tempsMarks sorted: [ :a :b | a tempsMark > b tempsMark ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 52096606!
correctSelector: proposedKeyword wordIntervals: spots exprInterval: expInt ifAbort: abortAction
	"Correct the proposedKeyword to some selector symbol, correcting the original text if such action is indicated.  abortAction is invoked if the proposedKeyword couldn't be converted into a valid selector.  Spots is an ordered collection of intervals within the test stream of the for each of the keyword parts."

	| correctSelector userSelection |
	"If we can't ask the user, assume that the keyword will be defined later"
	self interactive ifFalse: [^proposedKeyword asSymbol].

	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spots first first to: spots last last.
	cue requestor select.

	correctSelector := UnknownSelector name: proposedKeyword.
	correctSelector ifNil: [^abortAction value].

	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.

	self substituteSelector: correctSelector keywords wordIntervals: spots.
	^(proposedKeyword last ~~ $:
	   and: [correctSelector last == $:])
		ifTrue: [abortAction value]
		ifFalse: [correctSelector]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:20' prior: 33907242!
correctVariable: proposedVariable interval: spot 
	"Correct the proposedVariable to a known variable, or declare it as a new
	variable if such action is requested.  We support declaring lowercase
	variables as temps or inst-vars, and uppercase variables as Globals or 
	ClassVars, depending on whether the context is nil (class=UndefinedObject).
	Spot is the interval within the test stream of the variable.
	rr 3/4/2004 10:26 : adds the option to define a new class. "

	"Check if this is an i-var, that has been corrected already (ugly)"

	"Display the pop-up menu"

	| binding userSelection action |
	(encoder classEncoding instVarNames includes: proposedVariable) ifTrue: 
		[^InstanceVariableNode new 
			name: proposedVariable
			index: (encoder classEncoding allInstVarNames indexOf: proposedVariable)].

	"If we can't ask the user for correction, make it undeclared"
	self interactive ifFalse: [^encoder undeclared: proposedVariable].

	"First check to see if the requestor knows anything about the variable"
	(binding := cue requestor bindingOf: proposedVariable)
		ifNotNil: [^encoder global: binding name: proposedVariable].
	userSelection := cue requestor selectionInterval.
	cue requestor selectFrom: spot first to: spot last.
	cue requestor select.

	"Build the menu with alternatives"
	action := UndeclaredVariable 
				signalFor: self
				name: proposedVariable
				inRange: spot.
	action ifNil: [^self fail].

	"Execute the selected action"
	cue requestor deselect.
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last.
	^action value! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:19' prior: 34172921!
declareUndeclaredTemps: methodNode
	"Declare any undeclared temps, declaring them at the smallest enclosing scope."

	| undeclared userSelection blocksToVars |
	(undeclared := encoder undeclaredTemps) isEmpty ifTrue:
		[^self].
	userSelection := cue requestor selectionInterval.
	blocksToVars := IdentityDictionary new.
	undeclared do:
		[:var|
		(blocksToVars
			at: (var tag == #method
					ifTrue: [methodNode block]
					ifFalse: [methodNode accept: (VariableScopeFinder new ofVariable: var)])
			ifAbsentPut: [SortedCollection new]) add: var name].
	(blocksToVars removeKey: methodNode block ifAbsent: []) ifNotNil:
		[:rootVars|
		rootVars do: [:varName| self pasteTempAtMethodLevel: varName]].
	(blocksToVars keys sorted: [:a :b| a tempsMark < b tempsMark]) do:
		[:block| | decl |
		decl := (blocksToVars at: block) reduce: [:a :b| a, ' ', b].
		block temporaries isEmpty
			ifTrue:
				[self substituteWord: ' | ', decl, ' |'
					wordInterval: (block tempsMark + 1 to: block tempsMark)
					offset: requestorOffset]
			ifFalse:
				[self substituteWord: decl, ' '
					wordInterval: (block tempsMark to: block tempsMark - 1)
					offset: requestorOffset]].
	cue requestor selectInvisiblyFrom: userSelection first to: userSelection last + requestorOffset.
	ReparseAfterSourceEditing signal! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 11:45' prior: 37183770!
defineClass: className 
	"prompts the user to define a new class,  
	asks for it's category, and lets the users edit further  
	the definition"
	| sym cat def d2 |
	sym := className asSymbol.
	cat := UIManager default request: 'Enter class category : ' initialAnswer: self encoder classEncoding theNonMetaClass category.
	cat
		ifEmpty: [cat := 'Unknown'].
	def := 'Object subclass: #' , sym , '
		instanceVariableNames: '''' 
		classVariableNames: ''''
		poolDictionaries: ''''
		category: ''' , cat , ''''.
	d2 := UIManager default request: 'Edit class definition : ' initialAnswer: def.
	d2
		ifEmpty: [d2 := def].
	Compiler evaluate: d2.
	^ encoder
		global: (cue environment bindingOf: sym)
		name: sym! !
!Parser methodsFor: 'primitives' stamp: 'cwp 12/27/2012 11:46' prior: 37184567!
externalFunctionDeclaration
	"Parse the function declaration for a call to an external library."
	| descriptorClass callType modifier retType externalName args argType module fn |
	descriptorClass := cue environment
		valueOf: #ExternalFunction 
		ifAbsent: [^ false].
	callType := descriptorClass callingConventionFor: here.
	callType == nil ifTrue:[^false].
	[modifier := descriptorClass callingConventionModifierFor: token.
	 modifier notNil] whileTrue:
		[self advance.
		 callType := callType bitOr: modifier].
	"Parse return type"
	self advance.
	retType := self externalType: descriptorClass.
	retType == nil ifTrue:[^self expected:'return type'].
	"Parse function name or index"
	externalName := here.
	(self match: #string) 
		ifTrue:[externalName := externalName asSymbol]
		ifFalse:[(self match:#number) ifFalse:[^self expected:'function name or index']].
	(self matchToken: #'(') ifFalse:[^self expected:'argument list'].
	args := WriteStream on: Array new.
	[here == #')'] whileFalse:[
		argType := self externalType: descriptorClass.
		argType == nil ifTrue:[^self expected:'argument'].
		argType isVoid & argType isPointerType not ifFalse:[args nextPut: argType].
	].
	(self matchToken: #')') ifFalse:[^self expected:')'].
	(self matchToken: 'module:') ifTrue:[
		module := here.
		(self match: #string) ifFalse:[^self expected: 'String'].
		module := module asSymbol].
	Smalltalk at: #ExternalLibraryFunction ifPresent:[:xfn|
		fn := xfn name: externalName 
				module: module 
				callType: callType
				returnType: retType
				argumentTypes: args contents.
		self allocateLiteral: fn.
	].
	(self matchToken: 'error:')
		ifTrue:
			[| errorCodeVariable |
			 errorCodeVariable := here.
			(hereType == #string
			 or: [hereType == #word]) ifFalse:[^self expected: 'error code (a variable or string)'].
			 self advance.
			 self addPragma: (Pragma keyword: #primitive:error: arguments: (Array with: 120 with: errorCodeVariable)).
			 fn ifNotNil: [fn setErrorCodeName: errorCodeVariable]]
		ifFalse:
			[self addPragma: (Pragma keyword: #primitive: arguments: #(120))].
	^true
! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:19' prior: 58306169!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Parser methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:22' prior: 58137223!
notify: string at: location
	cue requestor isNil
		ifTrue: [(encoder == self or: [encoder isNil]) ifTrue: [^ self fail  "failure setting up syntax error"].
				SyntaxErrorNotification
					inClass: encoder classEncoding
					category: cue category
					withCode: 
						(source contents asText
							copyReplaceFrom: location
							to: location - 1
							with: ((string , ' ->') asText allBold 
								addAttribute: TextColor red; yourself))
					doitFlag: doitFlag
					errorMessage: string
					location: location]
		ifFalse: [cue requestor
					notify: string , ' ->'
					at: location
					in: source].
	^self fail! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:17' prior: 34177108!
pasteTempAtMethodLevel: name
	| insertion delta theTextString characterBeforeMark |

	theTextString := cue requestor text string.
	characterBeforeMark := theTextString at: tempsMark-1 ifAbsent: [$ ].
	(theTextString at: tempsMark) = $| ifTrue: [
  		"Paste it before the second vertical bar"
		insertion := name, ' '.
		characterBeforeMark isSeparator ifFalse: [insertion := ' ', insertion].
		delta := 0.
	] ifFalse: [
		"No bars - insert some with CR, tab"
		insertion := '| ' , name , ' |',String cr.
		delta := 2.	"the bar and CR"
		characterBeforeMark = Character tab ifTrue: [
			insertion := insertion , String tab.
			delta := delta + 1.	"the tab"
		].
	].
	tempsMark := tempsMark +
		(self substituteWord: insertion
			wordInterval: (tempsMark to: tempsMark-1)
			offset: 0) - delta! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:16' prior: 52095305!
queryUndefined
	| varStart varName | 
	varName := parseNode key.
	varStart := self endOfLastToken + requestorOffset - varName size + 1.
	cue requestor selectFrom: varStart to: varStart + varName size - 1; select.
	(UndefinedVariable name: varName) ifFalse: [^ self fail]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38599341!
removeEmptyTempDeclarationsFrom: methodNode

	| sourceCode madeChanges tempsMarkHolder |
	sourceCode := cue requestor text asString.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	madeChanges := false.
	tempsMarkHolder do: [ :currentBlock | | tempsMarkChar0 tempsMarkChar1 tempsMarkChar2 end start |
		tempsMarkChar0 := (sourceCode at: currentBlock tempsMark).
		tempsMarkChar1 := (sourceCode at: currentBlock tempsMark - 1).
		tempsMarkChar2 := (sourceCode at: currentBlock tempsMark - 2).
		tempsMarkChar0 = $| & tempsMarkChar1 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 1].
		tempsMarkChar0 = $| & tempsMarkChar1 = $  & tempsMarkChar2 = $| 
			ifTrue: 
				[ end := currentBlock tempsMark. 
				start := end - 2].
		
		start notNil & end notNil ifTrue: [
			| lineStart lineEnd |
			lineStart := 1 + (sourceCode 
				lastIndexOf: Character cr 
				startingAt: start - 1
				ifAbsent: [ 0 ]).
			lineEnd := sourceCode 
				indexOf: Character cr
				startingAt: end + 1
				ifAbsent: [ sourceCode size ].
			((sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: lineStart) >= start 
				and: [ (sourceCode indexOfAnyOf: CharacterSet nonSeparators startingAt: end + 1) > lineEnd ]) ifTrue: [
					start := lineStart.
					end := lineEnd ].
			cue requestor correctFrom: start to: end with: ''.
			madeChanges := true.
			currentBlock tempsMark: nil ] ].
	madeChanges ifTrue: [ReparseAfterSourceEditing signal]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:15' prior: 38561281!
removeUnusedTemporaryNamed: temp from: str lookingAt: currentBlock movingTempMarksOf: someBlocks

	| start end |
	end := currentBlock tempsMark - 1.
	["Beginning at right temp marker..."
	start := end - temp size + 1.
	end < temp size or: [ (str at: start) = $| ]
		or: [ temp = (str copyFrom: start to: end) 
			and: [ ((str at: start - 1) = $| | (str at: start - 1) isSeparator) 
				& ((str at: end + 1) = $| | (str at: end + 1) isSeparator) ] ]]
		whileFalse: [ 
			"Search left for the unused temp"
			end := cue requestor nextTokenFrom: end direction: -1 ].
	(end < temp size or: [ (str at: start) = $| ])
		ifFalse: 
			[(str at: start - 1) = $ 
				ifTrue: [ start := start - 1 ].
			cue requestor correctFrom: start to: end with: ''.
			someBlocks do: [ :aBlock | aBlock tempsMark: aBlock tempsMark - (end - start + 1)].
			^true ].
	^false! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 38562194!
removeUnusedTemps: methodNode
	"Scan for unused temp names, and prompt the user about the prospect of removing each one found"

	| madeChanges tempsMarkHolder unusedTempNames tempMarkHoldersToChange |
	madeChanges := false.
	tempMarkHoldersToChange := OrderedCollection new.
	tempsMarkHolder := self collectTemporaryDeclarationsFrom: methodNode.
	unusedTempNames := encoder unusedTempNames select: 
		[ :temp | (encoder lookupVariable: temp ifAbsent: [ ]) isUndefTemp 
				and: [ self queriedUnusedTemporaries at: temp ifAbsentPut: [UnusedVariable name: temp] ]].
	tempsMarkHolder do: [ :currentBlock | 
		tempMarkHoldersToChange add: currentBlock.
		unusedTempNames do: 
			[ :temp |
			(self 
				removeUnusedTemporaryNamed: temp 
				from: cue requestor text asString 
				lookingAt: currentBlock
				movingTempMarksOf: tempMarkHoldersToChange) ifTrue: [ madeChanges := true ]]].
	madeChanges
		ifTrue: [ self removeEmptyTempDeclarationsFrom: methodNode.
			ReparseAfterSourceEditing signal ]! !
!Parser methodsFor: 'error correction' stamp: 'cwp 12/27/2012 10:14' prior: 34179326!
substituteWord: correctWord wordInterval: spot offset: o
	"Substitute the correctSelector into the (presumed interactive) receiver.
	 Update requestorOffset based on the delta size and answer the updated offset."

	cue requestor correctFrom: spot first + o to: spot last + o with: correctWord.
	requestorOffset := requestorOffset + correctWord size - spot size.
	^o + correctWord size - spot size! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34179807!
temporaries
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'
! !
!Parser methodsFor: 'expression types' stamp: 'cwp 12/27/2012 10:14' prior: 34180638!
temporariesIn: methodSelector
	" [ '|' (variable)* '|' ]"
	| vars theActualText |
	(self match: #verticalBar) ifFalse: 
		["no temps"
		doitFlag ifTrue:
			[tempsMark := self interactive
								ifTrue: [cue requestor selectionInterval first]
								ifFalse: [1].
			^ #()].
		tempsMark := hereMark	"formerly --> prevMark + prevToken".
		tempsMark > 0 ifTrue:
			[theActualText := source contents.
			[tempsMark < theActualText size and: [(theActualText at: tempsMark) isSeparator]]
				whileTrue: [tempsMark := tempsMark + 1]].
			^ #()].
	vars := OrderedCollection new.
	[hereType == #word]
		whileTrue: [vars addLast: (encoder bindTemp: self advance in: methodSelector)].
	(self match: #verticalBar) ifTrue: 
		[tempsMark := prevMark.
		^ vars].
	^ self expected: 'Vertical bar'! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:11' prior: 53971863!
compiledMethodFor: textOrStream in: aContext to: receiver notifying: aRequestor ifFail: failBlock logged: logFlag
	"Compiles the sourceStream into a parse tree, then generates code
	 into a method, and answers it.  If receiver is not nil, then the text can
	 refer to instance variables of that receiver (the Inspector uses this).
	 If aContext is not nil, the text can refer to temporaries in that context
	 (the Debugger uses this). If aRequestor is not nil, then it will receive a 
	 notify:at: message before the attempt to evaluate is aborted."

	| methodNode method theClass |
	theClass := (aContext == nil ifTrue: [receiver] ifFalse: [aContext receiver]) class.
	self from: textOrStream class: theClass context: aContext notifying: aRequestor.
	methodNode := self translate: sourceStream noPattern: true ifFail: [^failBlock value].
	method := self interactive ifTrue: [ 	methodNode generateWithTempNames ] 
		ifFalse: [methodNode generate].
		
	logFlag ifTrue:
		[SystemChangeNotifier uniqueInstance evaluated: sourceStream contents context: aContext].
	^method! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:33' prior: 34363593!
format: aStream noPattern: noPattern ifFail: failBlock
	^(self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]) preen! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 10:08' prior: 58306325!
interactive
	"Answer true if compilation is interactive"

	^ cue requestor notNil! !
!Compiler methodsFor: 'error handling' stamp: 'cwp 12/27/2012 10:10' prior: 50779387!
notify: aString at: location
	"Refer to the comment in Object|notify:."

	^ cue requestor == nil
		ifTrue: [SyntaxErrorNotification
					inClass: cue getClass
					category: cue category
					withCode: 
						(sourceStream contents
							copyReplaceFrom: location
							to: location - 1
							with: aString)
					doitFlag: false
					errorMessage: aString
					location: location]
		ifFalse: [cue requestor
					notify: aString
					at: location
					in: sourceStream]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:34' prior: 50777201!
parse: textOrStream in: aClass notifying: req
	"Compile the argument, textOrStream, with respect to the class, aClass, and
	 answer the MethodNode that is the root of the resulting parse tree.  Notify the
	 argument, req, if an error occurs. The failBlock is defaulted to an empty block."

	self from: textOrStream class: aClass context: nil notifying: req.
	^self parser
		parse: sourceStream
		cue: cue
		noPattern: false
		ifFail: []! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 10:09' prior: 36332471!
parser

	parser ifNil: [parser := (cue getClass ifNil: [self class]) newParser].
	^parser! !
!Compiler methodsFor: 'private' stamp: 'cwp 12/27/2012 11:37' prior: 50780779!
translate: aStream noPattern: noPattern ifFail: failBlock
	^self parser
		parse: aStream
		cue: cue 
		noPattern: noPattern
		ifFail: [^failBlock value]! !
!Compiler methodsFor: 'public access' stamp: 'cwp 12/27/2012 11:37' prior: 19124095!
translate: aStream noPattern: noPattern ifFail: failBlock parser: parser
	| tree |
	tree := parser
			parse: aStream
			cue: cue 
			noPattern: noPattern
			ifFail: [^ failBlock value].
	^ tree! !
!Encoder methodsFor: 'results' stamp: 'cwp 12/27/2012 10:26' prior: 50999892!
associationForClass
	| assoc |
	assoc := self environment associationAt: cue getClass name ifAbsent: [nil].
	^assoc value == cue getClass
		ifTrue: [assoc]
		ifFalse: [Association new value: cue getClass]! !
!Encoder methodsFor: 'temps' stamp: 'cwp 12/27/2012 10:25' prior: 20148386!
bindTemp: name in: methodSelector
	"Declare a temporary; error not if a field or class variable."
	scopeTable at: name ifPresent:[:node|
		"When non-interactive raise the error only if its a duplicate"
		(node isTemp or:[requestor interactive])
			ifTrue:[^self notify:'Name is already defined']
			ifFalse:[Transcript 
				show: '(', name, ' is shadowed in "' , cue getClass printString , '>>' , methodSelector printString , '")']].
	^self reallyBind: name! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:25' prior: 20149084!
classEncoding
	"This is a hack so that the parser may findout what class it was parsing for when it wants to create a syntax error view."
	^ cue getClass! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:39' prior: 20138819!
encodeLiteral: object

	^self
		name: object
		key: (cue  literalScannedAs: object notifying: self)
		class: LiteralNode
		type: LdLitType
		set: litSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 20139010!
encodeSelector: aSelector

	^self
		name: aSelector
		key: aSelector
		class: SelectorNode
		type: SendType
		set: selectorSet! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:40' prior: 58545123!
environment
	"Answer the environment of the current compilation context,
	 be it in a class or global (e.g. a workspace)"
	^cue environment! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 11:41' prior: 50994497!
lookupInPools: varName ifFound: assocBlock

	^Symbol
		hasInterned: varName
		ifTrue:
			[:sym|
			(cue bindingOf: sym)
				ifNil: [^false]
				ifNotNil: [:assoc| assocBlock value: assoc]]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 51004306!
possibleNamesFor: proposedName
	| results |
	results := cue getClass 
		possibleVariablesFor: proposedName 
		continuedFrom: nil.
	^ proposedName correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:24' prior: 50995012!
possibleVariablesFor: proposedVariable

	| results |
	results := proposedVariable correctAgainstDictionary: scopeTable
								continuedFrom: nil.
	proposedVariable first canBeGlobalVarInitial ifTrue:
		[ results := cue getClass possibleVariablesFor: proposedVariable
						continuedFrom: results ].
	^ proposedVariable correctAgainst: nil continuedFrom: results.
! !
!Encoder methodsFor: 'encoding' stamp: 'cwp 12/27/2012 11:42' prior: 51002830!
undeclared: name
	| sym |
	requestor interactive ifTrue:
		[requestor requestor == #error: ifTrue:
			[requestor error: 'Undeclared'].
		 ^self notify: 'Undeclared'].
	"Allow knowlegeable clients to squash the undeclared warning if they want (e.g.
	 Diffing pretty printers that are simply formatting text).  As this breaks
	 compilation it should only be used by clients that want to discard the result
	 of the compilation.  To squash the warning use e.g.
		[Compiler format: code in: class notifying: nil decorated: false]
			on: UndeclaredVariableWarning
			do: [:ex| ex resume: false]"
	sym := name asSymbol.
	^(UndeclaredVariableWarning new name: name selector: selector class: cue getClass) signal
		ifTrue:
			[| undeclared |
			undeclared := cue environment undeclared.
			undeclared at: sym put: nil.
			self global: (undeclared associationAt: sym) name: sym]
		ifFalse:
			[self global: (Association key: sym) name: sym]! !
!Encoder methodsFor: 'private' stamp: 'cwp 12/27/2012 10:23' prior: 51006007!
warnAboutShadowed: name
	requestor addWarning: name,' is shadowed'.
	selector ifNotNil:
		[Transcript cr; show: cue getClass name,'>>', selector, '(', name,' is shadowed)']! !

"Compiler"!
!SmalltalkImage methodsFor: 'housekeeping' stamp: 'cwp 6/22/2012 15:56' prior: 58497062!
cleanOutUndeclared 
	globals undeclared removeUnreferencedKeys! !
!SmalltalkImage methodsFor: 'special objects' stamp: 'cwp 6/22/2012 09:01' prior: 40515090!
recreateSpecialObjectsArray
	"Smalltalk recreateSpecialObjectsArray"
	
	"To external package developers:
	**** DO NOT OVERRIDE THIS METHOD.  *****
	If you are writing a plugin and need additional special object(s) for your own use, 
	use addGCRoot() function and use own, separate special objects registry "
	
	"The Special Objects Array is an array of objects used by the Squeak virtual machine.
	 Its contents are critical and accesses to it by the VM are unchecked, so don't even
	 think of playing here unless you know what you are doing."
	| newArray |
	newArray := Array new: 56.
	"Nil false and true get used throughout the interpreter"
	newArray at: 1 put: nil.
	newArray at: 2 put: false.
	newArray at: 3 put: true.
	"This association holds the active process (a ProcessScheduler)"
	newArray at: 4 put: (self bindingOf: #Processor).
	"Numerous classes below used for type checking and instantiation"
	newArray at: 5 put: Bitmap.
	newArray at: 6 put: SmallInteger.
	newArray at: 7 put: ByteString.
	newArray at: 8 put: Array.
	newArray at: 9 put: Smalltalk.
	newArray at: 10 put: Float.
	newArray at: 11 put: MethodContext.
	newArray at: 12 put: BlockContext.
	newArray at: 13 put: Point.
	newArray at: 14 put: LargePositiveInteger.
	newArray at: 15 put: Display.
	newArray at: 16 put: Message.
	newArray at: 17 put: CompiledMethod.
	newArray at: 18 put: (self specialObjectsArray at: 18).
	"(low space Semaphore)"
	newArray at: 19 put: Semaphore.
	newArray at: 20 put: Character.
	newArray at: 21 put: #doesNotUnderstand:.
	newArray at: 22 put: #cannotReturn:.
	newArray at: 23 put: nil. "This is the process signalling low space."
	"An array of the 32 selectors that are compiled as special bytecodes,
	 paired alternately with the number of arguments each takes."
	newArray at: 24 put: #(	#+ 1 #- 1 #< 1 #> 1 #<= 1 #>= 1 #= 1 #~= 1
							#* 1 #/ 1 #\\ 1 #@ 1 #bitShift: 1 #// 1 #bitAnd: 1 #bitOr: 1
							#at: 1 #at:put: 2 #size 0 #next 0 #nextPut: 1 #atEnd 0 #== 1 #class 0
							#blockCopy: 1 #value 0 #value: 1 #do: 1 #new 0 #new: 1 #x 0 #y 0 ).
	"An array of the 255 Characters in ascii order.
	 Cog inlines table into machine code at: prim so do not regenerate it."
	newArray at: 25 put: (self specialObjectsArray at: 25).
	newArray at: 26 put: #mustBeBoolean.
	newArray at: 27 put: ByteArray.
	newArray at: 28 put: Process.
	"An array of up to 31 classes whose instances will have compact headers"
	newArray at: 29 put: self compactClassesArray.
	newArray at: 30 put: (self specialObjectsArray at: 30). "(delay Semaphore)"
	newArray at: 31 put: (self specialObjectsArray at: 31). "(user interrupt Semaphore)"
	"Entries 32 - 34 unreferenced. Previously these contained prototype instances to be copied for fast initialization"
	newArray at: 32 put: nil. "was (Float new: 2)"
	newArray at: 33 put: nil. "was (LargePositiveInteger new: 4)"
	newArray at: 34 put: nil. "was Point new"
	newArray at: 35 put: #cannotInterpret:.
	"Note: This must be fixed once we start using context prototypes (yeah, right)"
	"(MethodContext new: CompiledMethod fullFrameSize)."
	newArray at: 36 put: (self specialObjectsArray at: 36). "Is the prototype MethodContext (unused by the VM)"
	newArray at: 37 put: BlockClosure.
	"(BlockContext new: CompiledMethod fullFrameSize)."
	newArray at: 38 put: (self specialObjectsArray at: 38). "Is the prototype BlockContext (unused by the VM)"
	"array of objects referred to by external code"
	newArray at: 39 put: (self specialObjectsArray at: 39).	"preserve external semaphores"
	newArray at: 40 put: nil. "Reserved for Mutex in Cog VMs"
	newArray at: 41 put: nil. "Reserved for a LinkedList instance for overlapped calls in CogMT"
	"finalization Semaphore"
	newArray at: 42 put: ((self specialObjectsArray at: 42) ifNil: [Semaphore new]).
	newArray at: 43 put: LargeNegativeInteger.
	"External objects for callout.
	 Note: Written so that one can actually completely remove the FFI."
	newArray at: 44 put: (self at: #ExternalAddress ifAbsent: []).
	newArray at: 45 put: (self at: #ExternalStructure ifAbsent: []).
	newArray at: 46 put: (self at: #ExternalData ifAbsent: []).
	newArray at: 47 put: (self at: #ExternalFunction ifAbsent: []).
	newArray at: 48 put: (self at: #ExternalLibrary ifAbsent: []).
	newArray at: 49 put: #aboutToReturn:through:.
	newArray at: 50 put: #run:with:in:.
	"51 reserved for immutability message"
	"newArray at: 51 put: #attemptToAssign:withIndex:."
	newArray at: 52 put: #(nil "nil => generic error" #'bad receiver'
							#'bad argument' #'bad index'
							#'bad number of arguments'
							#'inappropriate operation'  #'unsupported operation'
							#'no modification' #'insufficient object memory'
							#'insufficient C memory' #'not found' #'bad method'
							#'internal error in named primitive machinery'
							#'object may move').
	"53 to 55 are for Alien"
	newArray at: 53 put: (self at: #Alien ifAbsent: []).
	newArray at: 54 put: #invokeCallback:stack:registers:jmpbuf:.
	newArray at: 55 put: (self at: #UnsafeAlien ifAbsent: []).

	"Weak reference finalization"
	newArray at: 56 put: (self at: #WeakFinalizationList ifAbsent: []).

	"Now replace the interpreter's reference in one atomic operation"
	self specialObjectsArray becomeForward: newArray
	! !
!SmalltalkImage methodsFor: 'shrinking' stamp: 'cwp 6/22/2012 15:57' prior: 37288071!
unloadAllKnownPackages
	"Unload all packages we know how to unload and reload"

	"Prepare unloading"
	Smalltalk zapMVCprojects.
	Flaps disableGlobalFlaps: false.
	StandardScriptingSystem removeUnreferencedPlayers.
	Project removeAllButCurrent.
	#('Morphic-UserObjects' 'EToy-UserObjects' 'Morphic-Imported' )
		do: [:each | SystemOrganization removeSystemCategory: each].
	Smalltalk at: #ServiceRegistry ifPresent:[:aClass|
		SystemChangeNotifier uniqueInstance
			noMoreNotificationsFor: aClass.
	].
	World removeAllMorphs.

	"Go unloading"
	#(	'ReleaseBuilder' 'ScriptLoader'
		'311Deprecated' '39Deprecated'
		'Universes' 'SMLoader' 'SMBase' 'Installer-Core'
		'VersionNumberTests' 'VersionNumber'
		'Services-Base' 'PreferenceBrowser' 'Nebraska'
		'ToolBuilder-MVC' 'ST80'
		'CollectionsTests' 'GraphicsTests' 'KernelTests'  'MorphicTests' 
		'MultilingualTests' 'NetworkTests' 'ToolsTests' 'TraitsTests'
		'SystemChangeNotification-Tests' 'FlexibleVocabularies' 
		'EToys' 'Protocols' 'XML-Parser' 'Tests' 'SUnitGUI'
		'Help-Squeak' 'HelpSystem' 'SystemReporter'
	) do: [:pkgName| 
			(MCPackage named: pkgName) unload.
			MCMcmUpdater disableUpdatesOfPackage: pkgName.
			].
	"Traits use custom unload"
	Smalltalk at: #Trait ifPresent:[:aClass| aClass unloadTraits].

	"Post-unload cleanup"
	MCWorkingCopy flushObsoletePackageInfos.
	SystemOrganization removeSystemCategory: 'UserObjects'.
	Presenter defaultPresenterClass: nil.
	World dumpPresenter.
	ScheduledControllers := nil.
	Preferences removePreference: #allowEtoyUserCustomEvents.
	SystemOrganization removeEmptyCategories.
	ChangeSet removeChangeSetsNamedSuchThat:[:cs | (cs == ChangeSet current) not].
	globals undeclared removeUnreferencedKeys.
	StandardScriptingSystem initialize.
	MCFileBasedRepository flushAllCaches.
	MCDefinition clearInstances.
	Behavior flushObsoleteSubclasses.
	ChangeSet current clear.
	ChangeSet current name: 'Unnamed1'.
	Smalltalk flushClassNameCache.
	Smalltalk at: #Browser ifPresent:[:br| br initialize].
	DebuggerMethodMap voidMapCache.
	DataStream initialize.
	AppRegistry removeObsolete.
	FileServices removeObsolete.
	Preferences removeObsolete.
	TheWorldMenu removeObsolete.
	Smalltalk garbageCollect.
	Symbol compactSymbolTable.
	TheWorldMainDockingBar updateInstances.
	MorphicProject defaultFill: (Color gray: 0.9).
	World color: (Color gray: 0.9).
! !
!InternalTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:34' prior: 40472775!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!NaturalLanguageTranslator methodsFor: 'fileIn/fileOut' stamp: 'cwp 6/20/2012 17:27' prior: 40496770!
scanFrom: aStream 
	^ self scanFrom: aStream environment: Environment default! !
!SystemDictionary methodsFor: 'dictionary access' stamp: 'cwp 6/22/2012 15:58' prior: 30574136!
at: aKey put: anObject 
	"Override from Dictionary to check Undeclared and fix up
	references to undeclared variables."
	| index element |
	(self includesKey: aKey) ifFalse: 
		[self declare: aKey from: (self at: #Undeclared).
		self flushClassNameCache].
	super at: aKey put: anObject.
	^ anObject! !

"System"!

CodeHolder subclass: #Browser
	instanceVariableNames: 'environment systemOrganizer classOrganizer metaClassOrganizer editSelection metaClassIndicated selectedSystemCategory selectedClassName selectedMessageName selectedMessageCategoryName'
	classVariableNames: 'ListClassesHierarchically RecentClasses'
	poolDictionaries: ''
	category: 'Tools-Browser'!
!Browser commentStamp: 'cwp 12/27/2012 11:09' prior: 36419432!
I represent a query path into the class descriptions, the software of the system.!
!Browser methodsFor: 'accessing' stamp: 'cwp 6/24/2012 23:20'!
selectEnvironment: anEnvironment 
	environment := anEnvironment.
	systemOrganizer := environment organization! !
!Browser methodsFor: 'system category list' stamp: 'cwp 6/24/2012 23:06' prior: 36467357!

From noreply at buildbot.pypy.org  Thu Jun 27 16:11:58 2013
From: noreply at buildbot.pypy.org (Manuel Jacob)
Date: Thu, 27 Jun 2013 16:11:58 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: hg merge default
Message-ID: <20130627141158.2B78C1C1067@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob
Branch: refactor-str-types
Changeset: r65037:4cfd5cc8928e
Date: 2013-06-27 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/4cfd5cc8928e/

Log:	hg merge default

diff too long, truncating to 2000 out of 4528 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -6,6 +6,7 @@
 .idea
 .project
 .pydevproject
+__pycache__
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -707,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -934,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
--- a/pypy/doc/how-to-contribute.rst
+++ b/pypy/doc/how-to-contribute.rst
@@ -76,4 +76,4 @@
 The rest of directories serve specific niche goal and are unlikely a good
 entry point.
 
-.. _`introduction to RPython`: getting-started-dev.rst
+.. _`introduction to RPython`: getting-started-dev.html
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -57,3 +57,7 @@
 Fix a segfault in argsort when sorting by chunks on multidim numpypy arrays (mikefc)
 
 .. branch: dtype-isnative
+.. branch: ndarray-round
+
+.. branch: faster-str-of-bigint
+Improve performance of str(long).
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -29,12 +29,12 @@
     _application_traceback = None
 
     def __init__(self, w_type, w_value, tb=None):
-        assert w_type is not None
         self.setup(w_type)
         self._w_value = w_value
         self._application_traceback = tb
 
     def setup(self, w_type):
+        assert w_type is not None
         self.w_type = w_type
         if not we_are_translated():
             self.debug_excs = []
@@ -347,7 +347,6 @@
                 self.xstrings = strings
                 for i, _, attr in entries:
                     setattr(self, attr, args[i])
-                assert w_type is not None
 
             def _compute_value(self, space):
                 lst = [None] * (len(formats) + len(formats) + 1)
@@ -369,6 +368,18 @@
         _fmtcache2[formats] = OpErrFmt
     return OpErrFmt, strings
 
+class OpErrFmtNoArgs(OperationError):
+
+    def __init__(self, w_type, value):
+        self.setup(w_type)
+        self._value = value
+
+    def get_w_value(self, space):
+        w_value = self._w_value
+        if w_value is None:
+            self._w_value = w_value = space.wrap(self._value)
+        return w_value
+
 def get_operationerr_class(valuefmt):
     try:
         result = _fmtcache[valuefmt]
@@ -389,6 +400,8 @@
     %T - The result of space.type(w_arg).getname(space)
 
     """
+    if not len(args):
+        return OpErrFmtNoArgs(w_type, valuefmt)
     OpErrFmt, strings = get_operationerr_class(valuefmt)
     return OpErrFmt(w_type, strings, *args)
 operationerrfmt._annspecialcase_ = 'specialize:arg(1)'
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -33,6 +33,14 @@
     operr3 = operationerrfmt("w_type2", "a %s b %s c", "bar", "4b")
     assert operr3.__class__ is not operr.__class__
 
+def test_operationerrfmt_noargs(space):
+    operr = operationerrfmt(space.w_AttributeError, "no attribute 'foo'")
+    operr.normalize_exception(space)
+    val = operr.get_w_value(space)
+    assert space.isinstance_w(val, space.w_AttributeError)
+    w_repr = space.repr(val)
+    assert space.str_w(w_repr) == "AttributeError(\"no attribute 'foo'\",)"
+
 def test_operationerrfmt_T(space):
     operr = operationerrfmt(space.w_AttributeError,
                             "'%T' object has no attribute '%s'",
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -5,6 +5,22 @@
 
 @unwrap_spec(type=str)
 def newdict(space, type):
+    """ newdict(type)
+
+    Create a normal dict with a special implementation strategy.
+
+    type is a string and can be:
+
+    * "module" - equivalent to some_module.__dict__
+
+    * "instance" - equivalent to an instance dict with a not-changing-much
+                   set of keys
+
+    * "kwargs" - keyword args dict equivalent of what you get from **kwargs
+                 in a function, optimized for passing around
+
+    * "strdict" - string-key only dict. This one should be chosen automatically
+    """
     if type == 'module':
         return space.newdict(module=True)
     elif type == 'instance':
@@ -18,6 +34,10 @@
                               type)
 
 def dictstrategy(space, w_obj):
+    """ dictstrategy(dict)
+
+    show the underlaying strategy used by a dict object
+    """
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError,
                              space.wrap("expecting dict object"))
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -13,7 +13,6 @@
     _mixin_ = True
 
     def __init__(self, space, name, size, align, enumerators, enumvalues):
-        name = "enum " + name
         self._super.__init__(self, space, size, name, len(name), align)
         self.enumerators2values = {}   # str -> int
         self.enumvalues2erators = {}   # int -> str
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
         else:
             extra = " *"
         self.is_file = (ctitem.name == "struct _IO_FILE" or
-                        ctitem.name == "struct $FILE")
+                        ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -25,7 +25,6 @@
     custom_field_pos = False
 
     def __init__(self, space, name):
-        name = '%s %s' % (self.kind, name)
         W_CType.__init__(self, space, -1, name, len(name))
 
     def check_complete(self, w_errorcls=None):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -641,6 +641,8 @@
 
 def test_new_struct_type():
     BStruct = new_struct_type("foo")
+    assert repr(BStruct) == "<ctype 'foo'>"
+    BStruct = new_struct_type("struct foo")
     assert repr(BStruct) == "<ctype 'struct foo'>"
     BPtr = new_pointer_type(BStruct)
     assert repr(BPtr) == "<ctype 'struct foo *'>"
@@ -648,7 +650,7 @@
     py.test.raises(ValueError, alignof, BStruct)
 
 def test_new_union_type():
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert repr(BUnion) == "<ctype 'union foo'>"
     BPtr = new_pointer_type(BUnion)
     assert repr(BPtr) == "<ctype 'union foo *'>"
@@ -657,7 +659,7 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     assert BStruct.kind == "struct"
     assert BStruct.cname == "struct foo"
     assert BStruct.fields is None
@@ -689,7 +691,7 @@
 def test_complete_union():
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
-    BUnion = new_union_type("foo")
+    BUnion = new_union_type("union foo")
     assert BUnion.kind == "union"
     assert BUnion.cname == "union foo"
     assert BUnion.fields is None
@@ -708,7 +710,7 @@
 
 def test_struct_instance():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     p = cast(BStructPtr, 0)
     py.test.raises(AttributeError, "p.a1")    # opaque
@@ -728,7 +730,7 @@
 def test_union_instance():
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, -1), ('a2', BUInt, -1)])
     p = newp(new_pointer_type(BUnion), [-42])
     bigval = -42 + (1 << (8*size_of_int()))
@@ -744,7 +746,7 @@
 
 def test_struct_pointer():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1)])
@@ -758,7 +760,7 @@
     BVoidP = new_pointer_type(new_void_type())
     BInt = new_primitive_type("int")
     BIntPtr = new_pointer_type(BInt)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1),
                                        ('a2', BInt, -1),
@@ -792,7 +794,7 @@
 
 def test_array_in_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BArrayInt5 = new_array_type(new_pointer_type(BInt), 5)
     complete_struct_or_union(BStruct, [('a1', BArrayInt5, -1)])
     s = newp(new_pointer_type(BStruct), [[20, 24, 27, 29, 30]])
@@ -803,7 +805,7 @@
     def offsetof(BType, fieldname):
         return typeoffsetof(BType, fieldname)[1]
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     py.test.raises(TypeError, offsetof, BInt, "abc")
     py.test.raises(TypeError, offsetof, BStruct, "abc")
     complete_struct_or_union(BStruct, [('abc', BInt, -1), ('def', BInt, -1)])
@@ -832,7 +834,7 @@
 def test_function_type_taking_struct():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc = new_function_type((BStruct,), BShort, False)
@@ -937,7 +939,7 @@
 def test_call_function_7():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -953,7 +955,7 @@
 def test_call_function_20():
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
@@ -965,7 +967,7 @@
 
 def test_call_function_21():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
                                        ('c', BInt, -1),
@@ -985,7 +987,7 @@
 def test_call_function_22():
     BInt = new_primitive_type("int")
     BArray10 = new_array_type(new_pointer_type(BInt), 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BArray10, -1)])
     BFunc22 = new_function_type((BStruct, BStruct), BStruct, False)
@@ -1023,7 +1025,7 @@
 def test_cannot_pass_struct_with_array_of_length_0():
     BInt = new_primitive_type("int")
     BArray0 = new_array_type(new_pointer_type(BInt), 0)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a', BArray0)])
     py.test.raises(NotImplementedError, new_function_type,
                    (BStruct,), BInt, False)
@@ -1048,7 +1050,7 @@
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('c', BDouble, -1, 8),
                                        ('a', BSChar, -1, 2),
@@ -1221,7 +1223,7 @@
     BSChar = new_primitive_type("signed char")
     BInt = new_primitive_type("int")
     BDouble = new_primitive_type("double")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BSChar, -1),
                                        ('b', BDouble, -1)])
@@ -1238,7 +1240,7 @@
 
 def test_callback_returning_big_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a', BInt, -1),
                                        ('b', BInt, -1),
@@ -1275,26 +1277,27 @@
 def test_enum_type():
     BUInt = new_primitive_type("unsigned int")
     BEnum = new_enum_type("foo", (), (), BUInt)
-    assert repr(BEnum) == "<ctype 'enum foo'>"
+    assert repr(BEnum) == "<ctype 'foo'>"
     assert BEnum.kind == "enum"
-    assert BEnum.cname == "enum foo"
+    assert BEnum.cname == "foo"
     assert BEnum.elements == {}
     #
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert BEnum.kind == "enum"
+    assert BEnum.cname == "enum foo"
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     # 'elements' is not the real dict, but merely a copy
     BEnum.elements[2] = '??'
     assert BEnum.elements == {-20: 'ab', 0: 'def', 1: 'c'}
     #
-    BEnum = new_enum_type("bar", ('ab', 'cd'), (5, 5), BUInt)
+    BEnum = new_enum_type("enum bar", ('ab', 'cd'), (5, 5), BUInt)
     assert BEnum.elements == {5: 'ab'}
     assert BEnum.relements == {'ab': 5, 'cd': 5}
 
 def test_cast_to_enum():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
     assert sizeof(BEnum) == sizeof(BInt)
     e = cast(BEnum, 0)
     assert repr(e) == "<cdata 'enum foo' 0: def>"
@@ -1308,27 +1311,27 @@
     assert string(cast(BEnum, -242 + 2**128)) == '-242'
     #
     BUInt = new_primitive_type("unsigned int")
-    BEnum = new_enum_type("bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
+    BEnum = new_enum_type("enum bar", ('def', 'c', 'ab'), (0, 1, 20), BUInt)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum bar' 4294967295>"     # unsigned int
     #
     BLong = new_primitive_type("long")
-    BEnum = new_enum_type("baz", (), (), BLong)
+    BEnum = new_enum_type("enum baz", (), (), BLong)
     assert sizeof(BEnum) == sizeof(BLong)
     e = cast(BEnum, -1)
     assert repr(e) == "<cdata 'enum baz' -1>"
 
 def test_enum_with_non_injective_mapping():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('ab', 'cd'), (7, 7), BInt)
+    BEnum = new_enum_type("enum foo", ('ab', 'cd'), (7, 7), BInt)
     e = cast(BEnum, 7)
     assert repr(e) == "<cdata 'enum foo' 7: ab>"
     assert string(e) == 'ab'
 
 def test_enum_in_struct():
     BInt = new_primitive_type("int")
-    BEnum = new_enum_type("foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
-    BStruct = new_struct_type("bar")
+    BEnum = new_enum_type("enum foo", ('def', 'c', 'ab'), (0, 1, -20), BInt)
+    BStruct = new_struct_type("struct bar")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BEnum, -1)])
     p = newp(BStructPtr, [-20])
@@ -1432,7 +1435,7 @@
 
 def test_struct_with_bitfields():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     LONGBITS = 8 * sizeof(BLong)
     complete_struct_or_union(BStruct, [('a1', BLong, 1),
                                        ('a2', BLong, 2),
@@ -1455,7 +1458,7 @@
 def test_bitfield_instance():
     BInt = new_primitive_type("int")
     BUnsignedInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1),
                                        ('a2', BUnsignedInt, 2),
                                        ('a3', BInt, 3)])
@@ -1485,14 +1488,14 @@
 
 def test_bitfield_instance_init():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BStruct), [-1])
     assert p.a1 == -1
     p = newp(new_pointer_type(BStruct), {'a1': -1})
     assert p.a1 == -1
     #
-    BUnion = new_union_type("bar")
+    BUnion = new_union_type("union bar")
     complete_struct_or_union(BUnion, [('a1', BInt, 1)])
     p = newp(new_pointer_type(BUnion), [-1])
     assert p.a1 == -1
@@ -1561,7 +1564,7 @@
     py.test.raises(IndexError, newp, BArray, tuple(b'123456'))
     py.test.raises(IndexError, newp, BArray, list(b'123456'))
     py.test.raises(IndexError, newp, BArray, b'123456')
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     py.test.raises(TypeError, newp, new_pointer_type(BStruct), b'')
     py.test.raises(ValueError, newp, new_pointer_type(BStruct), [b'1'])
@@ -1601,7 +1604,7 @@
     p = newp(new_pointer_type(BFloat), cast(BFloat, 12.25))
     assert p[0] == 12.25
     #
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BInt, -1)])
     s1 = newp(BStructPtr, [42])
@@ -1619,7 +1622,7 @@
     s2 = newp(BStructPtr, s1[0])
     assert s2.a1 == 42
     #
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     BUnionPtr = new_pointer_type(BUnion)
     complete_struct_or_union(BUnion, [('a1', BInt, -1)])
     u1 = newp(BUnionPtr, [42])
@@ -1697,7 +1700,7 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharArray10 = new_array_type(BCharP, 10)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BCharArray10, -1)])
     p = newp(BStructPtr, None)
@@ -1714,7 +1717,7 @@
     new_function_type((), BFunc)    # works
     new_function_type((), new_primitive_type("int"))
     new_function_type((), new_pointer_type(BFunc))
-    BUnion = new_union_type("foo_u")
+    BUnion = new_union_type("union foo_u")
     complete_struct_or_union(BUnion, [])
     py.test.raises(NotImplementedError, new_function_type, (), BUnion)
     py.test.raises(TypeError, new_function_type, (), BArray)
@@ -1725,7 +1728,7 @@
     BFloat = new_primitive_type("float")
     BDouble = new_primitive_type("double")
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BShort, -1)])
     BFunc10 = new_function_type((BInt,), BStruct)
@@ -1735,7 +1738,7 @@
     assert s.a1 == bytechr(40)
     assert s.a2 == 40 * 40
     #
-    BStruct11 = new_struct_type("test11")
+    BStruct11 = new_struct_type("struct test11")
     complete_struct_or_union(BStruct11, [('a1', BInt, -1),
                                          ('a2', BInt, -1)])
     BFunc11 = new_function_type((BInt,), BStruct11)
@@ -1745,7 +1748,7 @@
     assert s.a1 == 40
     assert s.a2 == 40 * 40
     #
-    BStruct12 = new_struct_type("test12")
+    BStruct12 = new_struct_type("struct test12")
     complete_struct_or_union(BStruct12, [('a1', BDouble, -1),
                                          ])
     BFunc12 = new_function_type((BInt,), BStruct12)
@@ -1754,7 +1757,7 @@
     assert repr(s) == "<cdata 'struct test12' owning 8 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct13 = new_struct_type("test13")
+    BStruct13 = new_struct_type("struct test13")
     complete_struct_or_union(BStruct13, [('a1', BInt, -1),
                                          ('a2', BInt, -1),
                                          ('a3', BInt, -1)])
@@ -1766,7 +1769,7 @@
     assert s.a2 == 40 * 40
     assert s.a3 == 40 * 40 * 40
     #
-    BStruct14 = new_struct_type("test14")
+    BStruct14 = new_struct_type("struct test14")
     complete_struct_or_union(BStruct14, [('a1', BFloat, -1),
                                          ])
     BFunc14 = new_function_type((BInt,), BStruct14)
@@ -1775,7 +1778,7 @@
     assert repr(s) == "<cdata 'struct test14' owning 4 bytes>"
     assert s.a1 == 40.0
     #
-    BStruct15 = new_struct_type("test15")
+    BStruct15 = new_struct_type("struct test15")
     complete_struct_or_union(BStruct15, [('a1', BFloat, -1),
                                          ('a2', BInt, -1)])
     BFunc15 = new_function_type((BInt,), BStruct15)
@@ -1785,7 +1788,7 @@
     assert s.a1 == 40.0
     assert s.a2 == 40 * 40
     #
-    BStruct16 = new_struct_type("test16")
+    BStruct16 = new_struct_type("struct test16")
     complete_struct_or_union(BStruct16, [('a1', BFloat, -1),
                                          ('a2', BFloat, -1)])
     BFunc16 = new_function_type((BInt,), BStruct16)
@@ -1795,7 +1798,7 @@
     assert s.a1 == 40.0
     assert s.a2 == -40.0
     #
-    BStruct17 = new_struct_type("test17")
+    BStruct17 = new_struct_type("struct test17")
     complete_struct_or_union(BStruct17, [('a1', BInt, -1),
                                          ('a2', BFloat, -1)])
     BFunc17 = new_function_type((BInt,), BStruct17)
@@ -1818,7 +1821,7 @@
     BFunc2 = new_function_type((), new_primitive_type("short"))
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BFunc, -1)])
     newp(BStructPtr, [cast(BFunc, 0)])
@@ -1845,7 +1848,7 @@
         assert not pyuni4
     #
     BWCharP = new_pointer_type(BWChar)
-    BStruct = new_struct_type("foo_s")
+    BStruct = new_struct_type("struct foo_s")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BWChar, -1),
                                        ('a2', BWCharP, -1)])
@@ -1957,7 +1960,7 @@
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
     # struct that *also* owns the memory
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1),
                                        ('a2', new_primitive_type("int"), -1),
@@ -1977,7 +1980,7 @@
     assert q.a1 == 123456
 
 def test_nokeepalive_struct():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     BStructPtrPtr = new_pointer_type(BStructPtr)
     complete_struct_or_union(BStruct, [('a1', new_primitive_type("int"), -1)])
@@ -2164,7 +2167,7 @@
     assert repr(x) == "<cdata 'int[3]' NULL>"
 
 def test_cast_invalid():
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [])
     p = cast(new_pointer_type(BStruct), 123456)
     s = p[0]
@@ -2183,7 +2186,7 @@
 
 def test_bug_delattr():
     BLong = new_primitive_type("long")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BLong, -1)])
     x = newp(new_pointer_type(BStruct))
     py.test.raises(AttributeError, "del x.a1")
@@ -2192,7 +2195,7 @@
     py.test.skip("later")
     BLong = new_primitive_type("long")
     BArray = new_array_type(new_pointer_type(BLong), None)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructP = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BLong, -1),
                                        ('a2', BArray, -1)])
@@ -2308,7 +2311,7 @@
         BLong = new_primitive_type("long")
         BLongP = new_pointer_type(BLong)
         BArray0 = new_array_type(BLongP, length)
-        BStruct = new_struct_type("foo")
+        BStruct = new_struct_type("struct foo")
         BStructPtr = new_pointer_type(BStruct)
         complete_struct_or_union(BStruct, [('a1', BArray0, -1)])
         p = newp(BStructPtr, None)
@@ -2320,8 +2323,8 @@
 def test_nested_anonymous_struct():
     BInt = new_primitive_type("int")
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
-    BInnerStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
+    BInnerStruct = new_struct_type("struct foo")
     complete_struct_or_union(BInnerStruct, [('a1', BInt, -1),
                                             ('a2', BChar, -1)])
     complete_struct_or_union(BStruct, [('', BInnerStruct, -1),
@@ -2353,12 +2356,12 @@
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
     assert sizeof(BShort) == alignof(BShort) == 2
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BChar),
                                        ('a2', BChar),
                                        ('a3', BChar)])
     assert sizeof(BStruct) == 3 and alignof(BStruct) == 1
-    BUnion = new_union_type("u")
+    BUnion = new_union_type("union u")
     complete_struct_or_union(BUnion, [('s', BStruct),
                                       ('i', BShort)])
     assert sizeof(BUnion) == 4
@@ -2366,7 +2369,7 @@
 
 def test_unaligned_struct():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('b', BInt, -1, 1)],
                              None, 5, 1)
 
@@ -2424,7 +2427,7 @@
 
 def test_typeoffsetof():
     BChar = new_primitive_type("char")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2440,14 +2443,14 @@
 
 def test_typeoffsetof_no_bitfield():
     BInt = new_primitive_type("int")
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     complete_struct_or_union(BStruct, [('a1', BInt, 4)])
     py.test.raises(TypeError, typeoffsetof, BStruct, 'a1')
 
 def test_rawaddressof():
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
-    BStruct = new_struct_type("foo")
+    BStruct = new_struct_type("struct foo")
     BStructPtr = new_pointer_type(BStruct)
     complete_struct_or_union(BStruct, [('a1', BChar, -1),
                                        ('a2', BChar, -1),
@@ -2508,7 +2511,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("_IO_FILE")
+    BFILE = new_struct_type("struct _IO_FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2540,7 +2543,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    B_NOT_FILE = new_struct_type("NOT_FILE")
+    B_NOT_FILE = new_struct_type("struct NOT_FILE")
     B_NOT_FILEP = new_pointer_type(B_NOT_FILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2563,7 +2566,7 @@
     if sys.platform == "win32":
         py.test.skip("testing FILE not implemented")
     #
-    BFILE = new_struct_type("$FILE")
+    BFILE = new_struct_type("FILE")
     BFILEP = new_pointer_type(BFILE)
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
@@ -2762,7 +2765,7 @@
     BShort = new_primitive_type("short")
     BInt = new_primitive_type("int")
     BUInt = new_primitive_type("unsigned int")
-    BStruct = new_struct_type("foo1")
+    BStruct = new_struct_type("struct foo1")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('b1', BInt, 9),
                                        ('b2', BUInt, 7),
@@ -2775,7 +2778,7 @@
         assert sizeof(BStruct) == 12
     assert alignof(BStruct) == 4
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BShort, 9),
                                        ('c', BChar, -1)], -1, -1, -1, flag)
@@ -2790,7 +2793,7 @@
         assert sizeof(BStruct) == 6
         assert alignof(BStruct) == 2
     #
-    BStruct = new_struct_type("foo2")
+    BStruct = new_struct_type("struct foo2")
     complete_struct_or_union(BStruct, [('a', BChar, -1),
                                        ('',  BInt, 0),
                                        ('',  BInt, 0),
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -13,19 +13,9 @@
 from pypy.module._rawffi.interp_rawffi import TYPEMAP
 from pypy.module._rawffi.interp_rawffi import size_alignment
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
 
-def push_elem(ll_array, pos, value):
-    TP = lltype.typeOf(value)
-    ll_array = rffi.cast(rffi.CArrayPtr(TP), ll_array)
-    ll_array[pos] = value
-push_elem._annspecialcase_ = 'specialize:argtype(2)'
-
-def get_elem(ll_array, pos, ll_t):
-    ll_array = rffi.cast(rffi.CArrayPtr(ll_t), ll_array)
-    return ll_array[pos]
-get_elem._annspecialcase_ = 'specialize:arg(2)'
-
 
 class W_Array(W_DataShape):
     def __init__(self, basicffitype, size):
@@ -57,7 +47,7 @@
                                                 " array length"))
             for num in range(iterlength):
                 w_item = items_w[num]
-                unwrap_value(space, push_elem, result.ll_buffer, num,
+                unwrap_value(space, write_ptr, result.ll_buffer, num,
                              self.itemcode, w_item)
         return space.wrap(result)
 
@@ -118,7 +108,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, push_elem, self.ll_buffer, num,
+        unwrap_value(space, write_ptr, self.ll_buffer, num,
                      self.shape.itemcode, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
@@ -136,7 +126,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, get_elem, self.ll_buffer, num,
+        return wrap_value(space, read_ptr, self.ll_buffer, num,
                           self.shape.itemcode)
 
     def descr_getitem(self, space, w_index):
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._rawffi.array import push_elem
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from pypy.module._rawffi.structure import W_Structure
 from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp,
      unwrap_value, unpack_argshapes, got_libffi_error)
@@ -40,7 +40,7 @@
                 args_w[i] = space.wrap(rffi.cast(rffi.ULONG, ll_args[i]))
         w_res = space.call(w_callable, space.newtuple(args_w))
         if callback_ptr.result is not None: # don't return void
-            unwrap_value(space, push_elem, ll_res, 0,
+            unwrap_value(space, write_ptr, ll_res, 0,
                          callback_ptr.result, w_res)
     except OperationError, e:
         tbprint(space, space.wrap(e.get_traceback()),
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,7 @@
 
 from rpython.rlib.clibffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
 import rpython.rlib.rposix as rposix
 
@@ -43,6 +44,8 @@
 }
 TYPEMAP_PTR_LETTERS = "POszZ"
 TYPEMAP_NUMBER_LETTERS = "bBhHiIlLqQ?"
+TYPEMAP_FLOAT_LETTERS = "fd" # XXX long doubles are not propperly supported in
+                             # rpython, so we ignore then here
 
 if _MS_WINDOWS:
     TYPEMAP['X'] = ffi_type_pointer
@@ -239,6 +242,52 @@
 )
 
 unroll_letters_for_numbers = unrolling_iterable(TYPEMAP_NUMBER_LETTERS)
+unroll_letters_for_floats = unrolling_iterable(TYPEMAP_FLOAT_LETTERS)
+
+_ARM = rffi_platform.getdefined('__arm__', '')
+
+def read_ptr(ptr, ofs, TP):
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as t_array:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, t_array),
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.sizeof(TP))
+                ptr_val = t_array[0]
+                return ptr_val
+    else:
+        return rffi.cast(T, ptr)[ofs]
+read_ptr._annspecialcase_ = 'specialize:arg(2)'
+
+def write_ptr(ptr, ofs, value):
+    TP = lltype.typeOf(value)
+    T = lltype.Ptr(rffi.CArray(TP))
+    for c in unroll_letters_for_floats:
+        # Note: if we are on ARM and have a float-ish value that is not word
+        # aligned accessing it directly causes a SIGBUS. Instead we use memcpy
+        # to avoid the problem
+        if (_ARM and LL_TYPEMAP[c] is TP
+                    and rffi.cast(lltype.Signed, ptr) & 3 != 0):
+            if ofs != 0:
+                ptr = rffi.ptradd(ptr, ofs*rffi.sizeof(TP))
+            with lltype.scoped_alloc(T.TO, 1) as s_array:
+                s_array[0] = value
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, ptr),
+                    rffi.cast(rffi.VOIDP, s_array),
+                    rffi.sizeof(TP))
+                return
+    else:
+        rffi.cast(T, ptr)[ofs] = value
+write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -12,10 +12,13 @@
 from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance
 from pypy.module._rawffi.interp_rawffi import wrap_value, unwrap_value
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
-from pypy.module._rawffi.interp_rawffi import size_alignment, LL_TYPEMAP
+from pypy.module._rawffi.interp_rawffi import LL_TYPEMAP
 from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers
+from pypy.module._rawffi.interp_rawffi import size_alignment
+from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib import clibffi
-from rpython.rlib.rarithmetic import intmask, r_uint, signedtype, widen
+from rpython.rlib.rarithmetic import intmask, signedtype, widen
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong
 
 def unpack_fields(space, w_fields):
     fields_w = space.unpackiterable(w_fields)
@@ -260,10 +263,17 @@
 
 def LOW_BIT(x):
     return x & 0xFFFF
+
 def NUM_BITS(x):
     return x >> 16
-def BIT_MASK(x):
-    return (1 << x) - 1
+
+def BIT_MASK(x, ll_t):
+    if ll_t is lltype.SignedLongLong:
+        return (r_longlong(1) << x) - 1
+    elif ll_t is lltype.UnsignedLongLong:
+        return (r_ulonglong(1) << x) - 1
+    return (1  << x) -1
+BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
@@ -279,20 +289,20 @@
             lowbit = LOW_BIT(bitsize)
             if numbits:
                 value = widen(value)
-                current = widen(rffi.cast(T, ptr)[0])
-                bitmask = BIT_MASK(numbits)
+                bitmask = BIT_MASK(numbits, TP)
+                #
+                current = widen(read_ptr(ptr, 0, TP))
                 current &= ~ (bitmask << lowbit)
                 current |= (value & bitmask) << lowbit
                 value = rffi.cast(TP, current)
             break
-
-    rffi.cast(T, ptr)[0] = value
+    write_ptr(ptr, 0, value)
 push_field._annspecialcase_ = 'specialize:argtype(2)'
 
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     TP = lltype.Ptr(rffi.CArray(ll_t))
-    value = rffi.cast(TP, pos)[0]
+    value = read_ptr(pos, 0, ll_t)
 
     # Handle bitfields
     for c in unroll_letters_for_numbers:
@@ -301,13 +311,16 @@
             numbits = NUM_BITS(bitsize)
             lowbit = LOW_BIT(bitsize)
             if numbits:
-                value = widen(value)
+                value = widen(rffi.cast(ll_t, value))
+                bitmask = BIT_MASK(numbits, ll_t)
+                #
                 value >>= lowbit
-                value &= BIT_MASK(numbits)
+                value &= bitmask
                 if ll_t is lltype.Bool or signedtype(ll_t._type):
                     sign = (value >> (numbits - 1)) & 1
                     if sign:
-                        value = value - (1 << numbits)
+                        one = r_longlong(1) if ll_t is lltype.SignedLongLong else 1
+                        value = value - (one << numbits)
                 value = rffi.cast(ll_t, value)
             break
 
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1,6 +1,6 @@
 from rpython.translator.platform import platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.module._rawffi.interp_rawffi import TYPEMAP
+from pypy.module._rawffi.interp_rawffi import TYPEMAP, TYPEMAP_FLOAT_LETTERS
 from pypy.module._rawffi.tracker import Tracker
 
 import os, sys, py
@@ -48,12 +48,12 @@
             free(x1->next);
             free(x1);
         }
-        
+
         const char *static_str = "xxxxxx";
         long static_int = 42;
         double static_double = 42.42;
         long double static_longdouble = 42.42;
-        
+
         unsigned short add_shorts(short one, short two)
         {
            return one + two;
@@ -175,7 +175,7 @@
             inp.y = inp.x * 100;
             return inp;
         }
-        
+
         '''))
         symbols = """get_char char_check get_raw_pointer
                      add_shorts
@@ -194,7 +194,7 @@
         eci = ExternalCompilationInfo(export_symbols=symbols)
         return str(platform.compile([c_file], eci, 'x', standalone=False))
     prepare_c_example = staticmethod(prepare_c_example)
-    
+
     def setup_class(cls):
         space = cls.space
         from rpython.rlib.clibffi import get_libc_name
@@ -211,6 +211,7 @@
         cls.w_platform = space.wrap(platform.name)
         cls.w_sizes_and_alignments = space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
+        cls.w_float_typemap = space.wrap(TYPEMAP_FLOAT_LETTERS)
 
     def test_libload(self):
         import _rawffi
@@ -438,7 +439,7 @@
         arg.free()
         assert t.tm_year == 70
         assert t.tm_sec == 1
-        assert t.tm_min == 2      
+        assert t.tm_min == 2
         x.free()
 
     def test_nested_structures(self):
@@ -464,6 +465,20 @@
         free_double_struct = lib.ptr("free_double_struct", ['P'], None)
         free_double_struct(res)
 
+    def test_structure_bitfields_char(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'B', 1),
+                               ('B', 'B', 6),
+                               ('C', 'B', 1)])
+        x = X()
+        x.A = 0xf
+        x.B = 0xff
+        x.C = 0xf
+        assert x.A == 1
+        assert x.B == 63
+        assert x.C == 1
+        x.free()
+
     def test_structure_bitfields(self):
         import _rawffi
         X = _rawffi.Structure([('A', 'I', 1),
@@ -486,6 +501,60 @@
         assert (y.a, y.b, y.c) == (-1, -7, 0)
         y.free()
 
+    def test_structure_bitfields_longlong(self):
+        import _rawffi
+        Z = _rawffi.Structure([('a', 'Q', 1),
+                               ('b', 'Q', 62),
+                               ('c', 'Q', 1)])
+        z = Z()
+        z.a, z.b, z.c = 7, 0x1000000000000001, 7
+        assert (z.a, z.b, z.c) == (1, 0x1000000000000001, 1)
+        z.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
+    def test_structure_ulonglong_bitfields(self):
+        import _rawffi
+        X = _rawffi.Structure([('A', 'Q', 1),
+                               ('B', 'Q', 62),
+                               ('C', 'Q', 1)])
+        x = X()
+        x.A, x.B, x.C = 7, 0x1000000000000001, 7
+        assert x.A == 1
+        assert x.B == 0x1000000000000001
+        assert x.C == 1
+        x.free()
+
+    def test_structure_longlong_bitfields(self):
+        import _rawffi
+        Y = _rawffi.Structure([('a', 'q', 1),
+                               ('b', 'q', 61),
+                               ('c', 'q', 1)])
+        y = Y()
+        y.a, y.b, y.c = 0, -7, 0
+        assert (y.a, y.b, y.c) == (0, -7, 0)
+        y.free()
+
     def test_invalid_bitfields(self):
         import _rawffi
         raises(TypeError, _rawffi.Structure, [('A', 'c', 1)])
@@ -571,7 +640,7 @@
         res = pass_ll(arg1)
         assert res[0] == 1<<42
         arg1.free()
-    
+
     def test_callback(self):
         import _rawffi
         import struct
@@ -680,9 +749,20 @@
     def test_sizes_and_alignments(self):
         import _rawffi
         for k, (s, a) in self.sizes_and_alignments.iteritems():
+            print k,s,a
             assert _rawffi.sizeof(k) == s
             assert _rawffi.alignment(k) == a
 
+    def test_unaligned(self):
+        import _rawffi
+        for k in self.float_typemap:
+            S = _rawffi.Structure([('pad', 'c'), ('value', k)], pack=1)
+            s = S()
+            s.value = 4
+            assert s.value == 4
+            s.free()
+
+
     def test_array_addressof(self):
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
@@ -970,7 +1050,7 @@
         assert res.y == 33
         assert s2h.x == 7
         assert s2h.y == 11
-        
+
         s2h.free()
 
     def test_ret_struct_containing_array(self):
@@ -1029,7 +1109,7 @@
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
-    
+
     def setup_class(cls):
         cls.w_sizes_and_alignments = cls.space.wrap(dict(
             [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()]))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -13,7 +13,7 @@
 from pypy.module.sys.interp_encoding import setdefaultencoding
 from pypy.module._codecs.interp_codecs import CodecState
 from pypy.objspace.std import unicodeobject
-from rpython.rlib import runicode
+from rpython.rlib import rstring, runicode
 from rpython.tool.sourcetools import func_renamer
 import sys
 
@@ -684,12 +684,9 @@
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
     if rffi.cast(lltype.Signed, direction) <= 0:
-        w_ret = space.call_method(w_str, "startswith", w_substr,
-                                  space.wrap(start), space.wrap(end))
+        return rstring.startswith(str, substr, start, end)
     else:
-        w_ret = space.call_method(w_str, "endswith", w_substr,
-                                  space.wrap(start), space.wrap(end))
-    return space.int_w(w_ret)
+        return rstring.endswith(str, substr, start, end)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_Count(space, w_str, w_substr, start, end):
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -235,6 +235,11 @@
         w_values = space.newtuple([self])
         return convert_to_array(space, w_values)
 
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0):
+        v = self.convert_to(self.get_dtype(space))
+        return self.get_dtype(space).itemtype.round(v, decimals)
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -501,6 +506,7 @@
     any = interp2app(W_GenericBox.descr_any),
     all = interp2app(W_GenericBox.descr_all),
     ravel = interp2app(W_GenericBox.descr_ravel),
+    round = interp2app(W_GenericBox.descr_round),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -558,9 +558,27 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "resize not implemented yet"))
 
-    def descr_round(self, space, w_decimals=0, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "round not implemented yet"))
+    @unwrap_spec(decimals=int)
+    def descr_round(self, space, decimals=0, w_out=None):
+        if space.is_none(w_out):
+            if self.get_dtype().is_bool_type():
+                #numpy promotes bool.round() to float16. Go figure.
+                w_out = W_NDimArray.from_shape(self.get_shape(),
+                       interp_dtype.get_dtype_cache(space).w_float16dtype)
+            else:
+                w_out = None
+        elif not isinstance(w_out, W_NDimArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                "return arrays must be of ArrayType"))
+        out = interp_dtype.dtype_agreement(space, [self], self.get_shape(),
+                                           w_out)
+        if out.get_dtype().is_bool_type() and self.get_dtype().is_bool_type():
+            calc_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
+        else:
+            calc_dtype = out.get_dtype()
+
+        loop.round(space, self, calc_dtype, self.get_shape(), decimals, out)
+        return out
 
     def descr_searchsorted(self, space, w_v, w_side='left'):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -975,6 +993,7 @@
     byteswap = interp2app(W_NDimArray.descr_byteswap),
     choose   = interp2app(W_NDimArray.descr_choose),
     clip     = interp2app(W_NDimArray.descr_clip),
+    round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
 
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -173,7 +173,7 @@
         iter = x_iter
     shapelen = len(shape)
     while not iter.done():
-        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, 
+        where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
                                         arr_dtype=arr_dtype)
         w_cond = arr_iter.getitem()
         if arr_dtype.itemtype.bool(w_cond):
@@ -188,7 +188,7 @@
     return out
 
 axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce',
-                                    greens=['shapelen', 
+                                    greens=['shapelen',
                                             'func', 'dtype',
                                             'identity'],
                                     reds='auto')
@@ -228,7 +228,7 @@
     arg_driver = jit.JitDriver(name='numpy_' + op_name,
                                greens = ['shapelen', 'dtype'],
                                reds = 'auto')
-    
+
     def argmin_argmax(arr):
         result = 0
         idx = 1
@@ -265,7 +265,7 @@
      result.shape == [3, 5, 2, 4]
      broadcast shape should be [3, 5, 2, 7, 4]
      result should skip dims 3 which is len(result_shape) - 1
-        (note that if right is 1d, result should 
+        (note that if right is 1d, result should
                   skip len(result_shape))
      left should skip 2, 4 which is a.ndims-1 + range(right.ndims)
           except where it==(right.ndims-2)
@@ -283,9 +283,9 @@
     righti = right.create_dot_iter(broadcast_shape, right_skip)
     while not outi.done():
         dot_driver.jit_merge_point(dtype=dtype)
-        lval = lefti.getitem().convert_to(dtype) 
-        rval = righti.getitem().convert_to(dtype) 
-        outval = outi.getitem().convert_to(dtype) 
+        lval = lefti.getitem().convert_to(dtype)
+        rval = righti.getitem().convert_to(dtype)
+        outval = outi.getitem().convert_to(dtype)
         v = dtype.itemtype.mul(lval, rval)
         value = dtype.itemtype.add(v, outval).convert_to(dtype)
         outi.setitem(value)
@@ -355,7 +355,7 @@
         setitem_filter_driver.jit_merge_point(shapelen=shapelen,
                                               index_dtype=index_dtype,
                                               arr_dtype=arr_dtype,
-                                             ) 
+                                             )
         if index_iter.getitem_bool():
             arr_iter.setitem(value_iter.getitem())
             value_iter.next()
@@ -572,6 +572,21 @@
         out_iter.next()
         min_iter.next()
 
+round_driver = jit.JitDriver(greens = ['shapelen', 'dtype'],
+                                    reds = 'auto')
+
+def round(space, arr, dtype, shape, decimals, out):
+    arr_iter = arr.create_iter(shape)
+    shapelen = len(shape)
+    out_iter = out.create_iter(shape)
+    while not arr_iter.done():
+        round_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+        w_v = dtype.itemtype.round(arr_iter.getitem().convert_to(dtype),
+                     decimals)
+        out_iter.setitem(w_v)
+        arr_iter.next()
+        out_iter.next()
+
 diagonal_simple_driver = jit.JitDriver(greens = ['axis1', 'axis2'],
                                        reds = 'auto')
 
@@ -613,4 +628,4 @@
         out_iter.setitem(arr.getitem_index(space, indexes))
         iter.next()
         out_iter.next()
-       
+
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -21,3 +21,19 @@
 
         a = zeros(3)
         assert loads(dumps(sum(a))) == sum(a)
+
+    def test_round(self):
+        from numpypy import int32, float64, complex128, bool
+        i = int32(1337)
+        f = float64(13.37)
+        c = complex128(13 + 37.j)
+        b = bool(0)
+        assert i.round(decimals=-2) == 1300
+        assert i.round(decimals=1) == 1337
+        assert c.round() == c
+        assert f.round() == 13.
+        assert f.round(decimals=-1) == 10.
+        assert f.round(decimals=1) == 13.4
+        exc = raises(AttributeError, 'b.round()')
+        assert exc.value[0] == "'bool' object has no attribute 'round'"
+
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -286,7 +286,7 @@
 
         skip('sign of nan is non-determinant')
         assert (signbit([float('nan'), float('-nan'), -float('nan')]) ==
-            [False, True, True]).all()    
+            [False, True, True]).all()
 
     def test_reciprocal(self):
         from numpypy import array, reciprocal, complex64, complex128
@@ -334,6 +334,23 @@
         assert all([math.copysign(1, f(abs(float("nan")))) == 1 for f in floor, ceil, trunc])
         assert all([math.copysign(1, f(-abs(float("nan")))) == -1 for f in floor, ceil, trunc])
 
+    def test_round(self):
+        from numpypy import array, dtype
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -1.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == a.round()).all()
+        i = array([-1000, -100, -1, 0, 1, 111, 1111, 11111], dtype=int)
+        assert (i == i.round()).all()
+        assert (i.round(decimals=4) == i).all()
+        assert (i.round(decimals=-4) == [0, 0, 0, 0, 0, 0, 0, 10000]).all()
+        b = array([True, False], dtype=bool)
+        bround = b.round()
+        assert (bround == [1., 0.]).all()
+        assert bround.dtype is dtype('float16')
+        c = array([10.5+11.5j, -15.2-100.3456j, 0.2343+11.123456j])
+        assert (c.round(0) == [10.+12.j, -15-100j, 0+11j]).all()
+
+
     def test_copysign(self):
         from numpypy import array, copysign
 
@@ -364,7 +381,7 @@
             assert b[i] == res
 
     def test_exp2(self):
-        import math 
+        import math
         from numpypy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
@@ -759,8 +776,8 @@
              complex(inf, inf), complex(inf, ninf), complex(0, inf),
              complex(ninf, ninf), complex(nan, 0), complex(0, nan),
              complex(nan, nan)]
-        assert (isfinite(a) == [True, True, False, False, False, 
-                        False, False, False, False, False]).all() 
+        assert (isfinite(a) == [True, True, False, False, False,
+                        False, False, False, False, False]).all()
 
     def test_logical_ops(self):
         from numpypy import logical_and, logical_or, logical_xor, logical_not
@@ -864,7 +881,7 @@
         #numpy returns (a.real*b.real + a.imag*b.imag) / abs(b)**2
         expect = [3., -23., 1.]
         for i in range(len(a)):
-            assert b[i] == expect[i] 
+            assert b[i] == expect[i]
         b = floor_divide(a[0], 0.)
         assert math.isnan(b.real)
         assert b.imag == 0.
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -37,7 +37,7 @@
         return self.box(
             func(
                 self,
-                self.for_computation(raw)
+                self.for_computation(raw),
             )
         )
     return dispatcher
@@ -521,6 +521,23 @@
             return v
         return 0
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.for_computation(self.unbox(v))
+        if decimals < 0:
+            # No ** in rpython
+            factor = 1
+            for i in xrange(-decimals):
+                factor *=10
+            #int does floor division, we want toward zero
+            if raw < 0:
+                ans = - (-raw / factor * factor)
+            else:
+                ans = raw / factor * factor
+        else:
+            ans = raw
+        return self.box(ans)
+
     @raw_unary_op
     def signbit(self, v):
         return v < 0
@@ -798,6 +815,16 @@
     def ceil(self, v):
         return math.ceil(v)
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        raw = self.for_computation(self.unbox(v))
+        if rfloat.isinf(raw):
+            return v
+        elif rfloat.isnan(raw):
+            return v
+        ans = rfloat.round_double(raw, decimals, half_even=True)
+        return self.box(ans)
+
     @simple_unary_op
     def trunc(self, v):
         if v < 0:
@@ -1354,6 +1381,15 @@
         except ZeroDivisionError:
             return rfloat.NAN, rfloat.NAN
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        ans = list(self.for_computation(self.unbox(v)))
+        if isfinite(ans[0]):
+            ans[0] = rfloat.round_double(ans[0], decimals,  half_even=True)
+        if isfinite(ans[1]):
+            ans[1] = rfloat.round_double(ans[1], decimals,  half_even=True)
+        return self.box_complex(ans[0], ans[1])
+
     # No floor, ceil, trunc in numpy for complex
     #@simple_unary_op
     #def floor(self, v):
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,8 +2,9 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.buffer import RWBuffer
 from rpython.rlib import rmmap, rarithmetic
-from rpython.rlib.rmmap import RValueError, RTypeError
+from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -20,13 +21,14 @@
         self.mmap.close()
 
     def read_byte(self):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.read_byte())
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def readline(self):
+        self.check_valid()
         return self.space.wrap(self.mmap.readline())
 
     @unwrap_spec(num=int)
@@ -36,6 +38,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def find(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -49,6 +52,7 @@
 
     @unwrap_spec(tofind='bufferstr')
     def rfind(self, tofind, w_start=None, w_end=None):
+        self.check_valid()
         space = self.space
         if w_start is None:
             start = self.mmap.pos
@@ -62,16 +66,18 @@
 
     @unwrap_spec(pos=OFF_T, whence=int)
     def seek(self, pos, whence=0):
+        self.check_valid()
         try:
             self.mmap.seek(pos, whence)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def tell(self):
+        self.check_valid()
         return self.space.wrap(self.mmap.tell())
 
     def descr_size(self):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.file_size())
         except OSError, e:
@@ -79,41 +85,40 @@
 
     @unwrap_spec(data='bufferstr')
     def write(self, data):
+        self.check_valid()
         self.check_writeable()
         try:
             self.mmap.write(data)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(byte=str)
     def write_byte(self, byte):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.write_byte(byte)
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(offset=int, size=int)
     def flush(self, offset=0, size=0):
+        self.check_valid()
         try:
             return self.space.wrap(self.mmap.flush(offset, size))
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
         except OSError, e:
             raise mmap_error(self.space, e)
 
     @unwrap_spec(dest=int, src=int, count=int)
     def move(self, dest, src, count):
+        self.check_valid()
+        self.check_writeable()
         try:
             self.mmap.move(dest, src, count)
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     @unwrap_spec(newsize=int)
     def resize(self, newsize):
@@ -136,28 +141,19 @@
         try:
             self.mmap.check_valid()
         except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
+            raise mmap_error(self.space, v)
 
     def check_writeable(self):
         try:
             self.mmap.check_writeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def check_resizeable(self):
         try:
             self.mmap.check_resizeable()
-        except RValueError, v:
-            raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(v.message))
-        except RTypeError, v:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(v.message))
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
 
     def descr_getitem(self, w_index):
         self.check_valid()
@@ -166,6 +162,8 @@
         start, stop, step = space.decode_index(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(self.mmap.getitem(start))
+        elif step == 1:
+            return space.wrap(self.mmap.getslice(start, stop - start))
         else:
             res = "".join([self.mmap.getitem(i)
                            for i in range(start, stop, step)])
@@ -189,15 +187,16 @@
             if len(value) != length:
                 raise OperationError(space.w_ValueError,
                           space.wrap("mmap slice assignment is wrong size"))
-            for i in range(length):
-                self.mmap.setitem(start, value[i])
-                start += step
+            if step == 1:
+                self.mmap.setslice(start, value)
+            else:
+                for i in range(length):
+                    self.mmap.setitem(start, value[i])
+                    start += step
 
     def descr_buffer(self):
-        # XXX improve to work directly on the low-level address
-        from pypy.interpreter.buffer import StringLikeBuffer
-        space = self.space
-        return space.wrap(StringLikeBuffer(space, space.wrap(self)))
+        self.check_valid()
+        return self.space.wrap(MMapBuffer(self.space, self.mmap))
 
 if rmmap._POSIX:
 
@@ -213,10 +212,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 elif rmmap._MS_WINDOWS:
@@ -232,10 +229,8 @@
                                        offset))
         except OSError, e:
             raise mmap_error(space, e)
-        except RValueError, e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
-        except RTypeError, e:
-            raise OperationError(space.w_TypeError, space.wrap(e.message))
+        except RMMapError, e:
+            raise mmap_error(space, e)
         return space.wrap(self)
 
 W_MMap.typedef = TypeDef("mmap",
@@ -275,5 +270,64 @@
                                                  space.w_EnvironmentError)
 
 def mmap_error(space, e):
-    w_error = space.fromcache(Cache).w_error
-    return wrap_oserror(space, e, w_exception_class=w_error)
+    if isinstance(e, RValueError):
+        return OperationError(space.w_ValueError,
+                              space.wrap(e.message))
+    elif isinstance(e, RTypeError):
+        return OperationError(space.w_TypeError,
+                              space.wrap(e.message))
+    elif isinstance(e, OSError):
+        w_error = space.fromcache(Cache).w_error
+        return wrap_oserror(space, e, w_exception_class=w_error)
+    else:
+        # bogus 'e'?
+        return OperationError(space.w_SystemError, space.wrap('%s' % e))
+mmap_error._dont_inline_ = True
+
+
+class MMapBuffer(RWBuffer):
+    def __init__(self, space, mmap):
+        self.space = space
+        self.mmap = mmap
+
+    def get_raw_address(self):
+        return self.mmap.data
+
+    def getlength(self):
+        return self.mmap.size
+
+    def getitem(self, index):
+        self.check_valid()
+        return self.mmap.data[index]
+
+    def getslice(self, start, stop, step, size):
+        self.check_valid()
+        if step == 1:
+            return self.mmap.getslice(start, size)
+        else:
+            return RWBuffer.getslice(self, start, stop, step, size)
+
+    def setitem(self, index, char):
+        self.check_valid_writeable()
+        self.mmap.data[index] = char
+
+    def setslice(self, start, string):
+        self.check_valid_writeable()
+        self.mmap.setslice(start, string)
+
+    def get_raw_address(self):
+        self.check_valid()
+        return self.mmap.data
+
+    def check_valid(self):
+        try:
+            self.mmap.check_valid()
+        except RValueError, v:
+            raise mmap_error(self.space, v)
+
+    def check_valid_writeable(self):
+        try:
+            self.mmap.check_valid()
+            self.mmap.check_writeable()
+        except RMMapError, v:
+            raise mmap_error(self.space, v)
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -550,6 +550,24 @@
         m.close()
         f.close()
 
+    def test_buffer_write(self):
+        from mmap import mmap
+        f = open(self.tmpname + "y", "w+")
+        f.write("foobar")
+        f.flush()
+        m = mmap(f.fileno(), 6)
+        m[5] = '?'
+        b = buffer(m)
+        try:
+            b[:3] = "FOO"
+        except TypeError:     # on CPython: "buffer is read-only" :-/
+            skip("on CPython: buffer is read-only")
+        m.close()
+        f.seek(0)
+        got = f.read()
+        assert got == "FOOba?"
+        f.close()
+
     def test_offset(self):
         from mmap import mmap, ALLOCATIONGRANULARITY
         f = open(self.tmpname + "y", "w+")
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -45,10 +45,7 @@
     raise NotImplementedError
 
 def resolvedirof(filename):
-    try:
-        filename = rpath.rabspath(filename)
-    except OSError:
-        pass
+    filename = rpath.rabspath(filename)
     dirname = rpath.rabspath(os.path.join(filename, '..'))
     if os.path.islink(filename):
         try:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py
@@ -990,7 +990,7 @@
         assert f.a == 12345
         assert b.b == b"B"
         assert b.c == b"C"
-        assert repr(b).startswith("<cdata 'struct $bar_t *'")
+        assert repr(b).startswith("<cdata 'bar_t *'")
 
     def test_struct_with_two_usages(self):
         for name in ['foo_s', '']:    # anonymous or not
@@ -1300,9 +1300,9 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("typedef enum { Value0 = 0 } e, *pe;\n"
                  "typedef enum { Value1 = 1 } e1;")
-        assert ffi.getctype("e*") == 'enum $e *'
-        assert ffi.getctype("pe") == 'enum $e *'
-        assert ffi.getctype("e1*") == 'enum $e1 *'
+        assert ffi.getctype("e*") == 'e *'
+        assert ffi.getctype("pe") == 'e *'
+        assert ffi.getctype("e1*") == 'e1 *'
 
     def test_new_ctype(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py
@@ -179,3 +179,9 @@
             'ffi.cdef("struct s2 { char x:0; };"); ffi.new("struct s2 *")')
         py.test.raises(TypeError,
             'ffi.cdef("struct s3 { char x:9; };"); ffi.new("struct s3 *")')
+
+    def test_struct_with_typedef(self):
+        ffi = FFI()
+        ffi.cdef("typedef struct { float x; } foo_t;")
+        p = ffi.new("foo_t *", [5.2])
+        assert repr(p).startswith("<cdata 'foo_t *' ")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py
@@ -1442,7 +1442,7 @@
                        "foo_t myfunc(void) { foo_t x = { 42 }; return x; }")
     assert str(e.value) in [
         "function myfunc: 'foo_t' is used as result type, but is opaque",
-        "function myfunc: result type 'struct $foo_t' is opaque"]
+        "function myfunc: result type 'foo_t' is opaque"]
 
 def test_include():
     ffi1 = FFI()
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py
@@ -66,6 +66,10 @@
         x.a, x.b, x.c = -1, 7, -1
         assert (x.a, x.b, x.c) == (-1, 7, -1)
 
+        x = X()
+        x.a, x.b, x.c = -1, -7, -1
+        assert (x.a, x.b, x.c) == (-1, -7, -1)
+
     def test_ulonglong(self):
         class X(Structure):
             _fields_ = [("a", c_ulonglong, 1),
@@ -75,8 +79,8 @@
         assert sizeof(X) == sizeof(c_longlong)
         x = X()
         assert (x.a, x.b, x.c) == (0, 0, 0)
-        x.a, x.b, x.c = 7, 7, 7
-        assert (x.a, x.b, x.c) == (1, 7, 1)
+        x.a, x.b, x.c = 7, 2305843009213693953, 7
+        assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1)
 
     def test_signed(self):
         for c_typ in signed_int_types:
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -50,10 +50,6 @@
 def delegate_Bool2IntObject(space, w_bool):
     return W_IntObject(int(w_bool.boolval))
 
-def delegate_Bool2SmallInt(space, w_bool):
-    from pypy.objspace.std.smallintobject import W_SmallIntObject
-    return W_SmallIntObject(int(w_bool.boolval))   # cannot overflow
-
 

From noreply at buildbot.pypy.org  Thu Jun 27 16:44:53 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Jun 2013 16:44:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: move code from strutil to rstring,
 rarithmetic, rfloat and rbigint.
Message-ID: <20130627144453.338681C0351@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r65038:c1fe611caf63
Date: 2013-06-27 14:26 +0200
http://bitbucket.org/pypy/pypy/changeset/c1fe611caf63/

Log:	move code from strutil to rstring, rarithmetic, rfloat and rbigint.

diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -41,8 +41,8 @@
     def save_field(self, field_builder):
         field = field_builder.build()
         if self.numeric_field:
-            from pypy.objspace.std.strutil import ParseStringError
-            from pypy.objspace.std.strutil import string_to_float
+            from rpython.rlib.rstring import ParseStringError
+            from rpython.rlib.rfloat import string_to_float
             self.numeric_field = False
             try:
                 ff = string_to_float(field)
diff --git a/pypy/module/micronumpy/interp_support.py b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.micronumpy import interp_dtype, loop
-from pypy.objspace.std.strutil import strip_spaces
+from rpython.rlib.rstring import strip_spaces
 from rpython.rlib.rarithmetic import maxint
 from pypy.module.micronumpy.base import W_NDimArray
 
diff --git a/pypy/objspace/std/complextype.py b/pypy/objspace/std/complextype.py
--- a/pypy/objspace/std/complextype.py
+++ b/pypy/objspace/std/complextype.py
@@ -1,10 +1,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std.register_all import register_all
-from pypy.objspace.std.strutil import string_to_float, ParseStringError
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef
 from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod
+from rpython.rlib.rfloat import string_to_float
+from rpython.rlib.rstring import ParseStringError
 
 # ERRORCODES
 
diff --git a/pypy/objspace/std/floattype.py b/pypy/objspace/std/floattype.py
--- a/pypy/objspace/std/floattype.py
+++ b/pypy/objspace/std/floattype.py
@@ -8,10 +8,9 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
-from pypy.objspace.std.strutil import ParseStringError
-from pypy.objspace.std.strutil import string_to_float
 from pypy.objspace.std.model import W_Object
 from rpython.rlib.rbigint import rbigint
+from rpython.rlib.rstring import ParseStringError
 
 
 float_as_integer_ratio = SMM("as_integer_ratio", 1)
@@ -41,7 +40,7 @@
           space.isinstance_w(w_value, space.w_bytearray)):
         strvalue = space.bufferstr_w(w_value)
         try:
-            value = string_to_float(strvalue)
+            value = rfloat.string_to_float(strvalue)
         except ParseStringError, e:
             raise OperationError(space.w_ValueError,
                                  space.wrap(e.msg))
@@ -49,7 +48,7 @@
         from unicodeobject import unicode_to_decimal_w
         strvalue = unicode_to_decimal_w(space, w_value)
         try:
-            value = string_to_float(strvalue)
+            value = rfloat.string_to_float(strvalue)
         except ParseStringError, e:
             raise OperationError(space.w_ValueError,
                                  space.wrap(e.msg))
diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py
--- a/pypy/objspace/std/inttype.py
+++ b/pypy/objspace/std/inttype.py
@@ -5,13 +5,11 @@
 from pypy.interpreter.buffer import Buffer
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
-from pypy.objspace.std.strutil import (string_to_int, string_to_bigint,
-                                       ParseStringError,
-                                       ParseStringOverflowError)
 from pypy.objspace.std.model import W_Object
-from rpython.rlib.rarithmetic import r_uint
+from rpython.rlib.rarithmetic import r_uint, string_to_int
 from rpython.rlib.objectmodel import instantiate
 from rpython.rlib.rbigint import rbigint
+from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
 
 # ____________________________________________________________
 
@@ -78,12 +76,11 @@
 def retry_to_w_long(space, parser, base=0):
     parser.rewind()
     try:
-        bigint = string_to_bigint(None, base=base, parser=parser)
+        bigint = rbigint.fromstr(None, base=base, parser=parser)
     except ParseStringError, e:
         raise OperationError(space.w_ValueError,
                              space.wrap(e.msg))
-    from pypy.objspace.std.longobject import newlong
-    return newlong(space, bigint)
+    return space.newlong_from_rbigint(bigint)
 
 @unwrap_spec(w_x = WrappedDefault(0))
 def descr__new__(space, w_inttype, w_x, w_base=None):
diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py
--- a/pypy/objspace/std/longtype.py
+++ b/pypy/objspace/std/longtype.py
@@ -4,7 +4,7 @@
      interpindirect2app
 from pypy.objspace.std.model import W_Object
 from pypy.objspace.std.stdtypedef import StdTypeDef
-from pypy.objspace.std.strutil import string_to_bigint, ParseStringError
+from rpython.rlib.rstring import ParseStringError
 from rpython.rlib.rbigint import rbigint
 
 def descr_conjugate(space, w_int):
@@ -66,7 +66,7 @@
 
 def string_to_w_long(space, w_longtype, s, base=10):
     try:
-        bigint = string_to_bigint(s, base)
+        bigint = rbigint.fromstr(s, base)
     except ParseStringError, e:
         raise OperationError(space.w_ValueError,
                              space.wrap(e.msg))
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
deleted file mode 100644
--- a/pypy/objspace/std/strutil.py
+++ /dev/null
@@ -1,188 +0,0 @@
-"""
-Pure Python implementation of string utilities.
-"""
-
-from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rfloat import rstring_to_float, INFINITY, NAN
-from rpython.rlib.rbigint import rbigint, parse_digit_string
-from pypy.interpreter.error import OperationError
-import math
-
-# XXX factor more functions out of stringobject.py.
-# This module is independent from PyPy.
-
-def strip_spaces(s):
-    # XXX this is not locale-dependent
-    p = 0
-    q = len(s)
-    while p < q and s[p] in ' \f\n\r\t\v':
-        p += 1
-    while p < q and s[q-1] in ' \f\n\r\t\v':
-        q -= 1
-    assert q >= p     # annotator hint, don't remove
-    return s[p:q]
-
-class ParseStringError(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-
-class ParseStringOverflowError(Exception):
-    def __init__(self, parser):
-        self.parser = parser
-
-# iterator-like class
-class NumberStringParser:
-
-    def error(self):
-        raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.original_base, self.literal))
-
-    def __init__(self, s, literal, base, fname):
-        self.literal = literal
-        self.fname = fname
-        sign = 1
-        if s.startswith('-'):
-            sign = -1
-            s = strip_spaces(s[1:])
-        elif s.startswith('+'):
-            s = strip_spaces(s[1:])
-        self.sign = sign
-        self.original_base = base
-
-        if base == 0:
-            if s.startswith('0x') or s.startswith('0X'):
-                base = 16
-            elif s.startswith('0b') or s.startswith('0B'):
-                base = 2
-            elif s.startswith('0'): # also covers the '0o' case
-                base = 8
-            else:
-                base = 10
-        elif base < 2 or base > 36:
-            raise ParseStringError, "%s() base must be >= 2 and <= 36" % (fname,)
-        self.base = base
-
-        if base == 16 and (s.startswith('0x') or s.startswith('0X')):
-            s = s[2:]
-        if base == 8 and (s.startswith('0o') or s.startswith('0O')):
-            s = s[2:]
-        if base == 2 and (s.startswith('0b') or s.startswith('0B')):
-            s = s[2:]
-        if not s:
-            self.error()
-        self.s = s
-        self.n = len(s)
-        self.i = 0
-
-    def rewind(self):
-        self.i = 0
-
-    def next_digit(self): # -1 => exhausted
-        if self.i < self.n:
-            c = self.s[self.i]
-            digit = ord(c)
-            if '0' <= c <= '9':
-                digit -= ord('0')
-            elif 'A' <= c <= 'Z':
-                digit = (digit - ord('A')) + 10
-            elif 'a' <= c <= 'z':
-                digit = (digit - ord('a')) + 10
-            else:
-                self.error()
-            if digit >= self.base:
-                self.error()
-            self.i += 1
-            return digit
-        else:
-            return -1
-
-def string_to_int(s, base=10):
-    """Utility to converts a string to an integer.
-    If base is 0, the proper base is guessed based on the leading
-    characters of 's'.  Raises ParseStringError in case of error.
-    Raises ParseStringOverflowError in case the result does not fit.
-    """
-    s = literal = strip_spaces(s)
-    p = NumberStringParser(s, literal, base, 'int')
-    base = p.base
-    result = 0
-    while True:
-        digit = p.next_digit()
-        if digit == -1:
-            return result
-
-        if p.sign == -1:
-            digit = -digit
-
-        try:
-            result = ovfcheck(result * base)
-            result = ovfcheck(result + digit)
-        except OverflowError:
-            raise ParseStringOverflowError(p)
-
-def string_to_bigint(s, base=10, parser=None):
-    """As string_to_int(), but ignores an optional 'l' or 'L' suffix
-    and returns an rbigint."""
-    if parser is None:
-        s = literal = strip_spaces(s)
-        if (s.endswith('l') or s.endswith('L')) and base < 22:
-            # in base 22 and above, 'L' is a valid digit!  try: long('L',22)
-            s = s[:-1]
-        p = NumberStringParser(s, literal, base, 'long')
-    else:
-        p = parser
-    return parse_digit_string(p)
-
-# Tim's comment:
-# 57 bits are more than needed in any case.
-# to allow for some rounding, we take one
-# digit more.
-
-# In the PyPy case, we can compute everything at compile time:
-# XXX move this stuff to some central place, it is now also
-# in _float_formatting.
-
-def calc_mantissa_bits():
-    bits = 1 # I know it is almost always 53, but let it compute...
-    while 1:
-        pattern = (1L << bits) - 1
-        comp = long(float(pattern))
-        if comp != pattern:
-            return bits - 1
-        bits += 1
-
-MANTISSA_BITS = calc_mantissa_bits()
-del calc_mantissa_bits
-MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1
-
-def string_to_float(s):
-    """
-    Conversion of string to float.
-    This version tries to only raise on invalid literals.
-    Overflows should be converted to infinity whenever possible.
-
-    Expects an unwrapped string and return an unwrapped float.
-    """
-
-    s = strip_spaces(s)
-
-    if not s:
-        raise ParseStringError("empty string for float()")
-
-
-    low = s.lower()
-    if low == "-inf" or low == "-infinity":
-        return -INFINITY
-    elif low == "inf" or low == "+inf":
-        return INFINITY
-    elif low == "infinity" or low == "+infinity":
-        return INFINITY
-    elif low == "nan" or low == "+nan":
-        return NAN
-    elif low == "-nan":
-        return -NAN
-
-    try:
-        return rstring_to_float(s)
-    except ValueError:
-        raise ParseStringError("invalid literal for float(): '%s'" % s)
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_strutil.py
+++ /dev/null
@@ -1,189 +0,0 @@
-import py, random
-from pypy.objspace.std.strutil import *
-from pypy.interpreter.error import OperationError
-
-
-class TestStrUtil:
-
-    def test_string_to_int(self):
-        space = self.space
-        cases = [('0', 0),
-                 ('1', 1),
-                 ('9', 9),
-                 ('10', 10),
-                 ('09', 9),
-                 ('0000101', 101),    # not octal unless base 0 or 8
-                 ('5123', 5123),
-                 (' 0', 0),
-                 ('0  ', 0),
-                 (' \t \n   32313  \f  \v   \r  \n\r    ', 32313),
-                 ('+12', 12),
-                 ('-5', -5),
-                 ('- 5', -5),
-                 ('+ 5', 5),
-                 ('  -123456789 ', -123456789),
-                 ]
-        for s, expected in cases:
-            assert string_to_int(s) == expected
-            assert string_to_bigint(s).tolong() == expected
-
-    def test_string_to_int_base(self):
-        space = self.space        
-        cases = [('111', 2, 7),
-                 ('010', 2, 2),
-                 ('102', 3, 11),
-                 ('103', 4, 19),
-                 ('107', 8, 71),
-                 ('109', 10, 109),
-                 ('10A', 11, 131),
-                 ('10a', 11, 131),
-                 ('10f', 16, 271),
-                 ('10F', 16, 271),
-                 ('0x10f', 16, 271),
-                 ('0x10F', 16, 271),
-                 ('10z', 36, 1331),
-                 ('10Z', 36, 1331),
-                 ('12',   0, 12),
-                 ('015',  0, 13),
-                 ('0x10', 0, 16),
-                 ('0XE',  0, 14),
-                 ('0',    0, 0),
-                 ('0b11', 2, 3),
-                 ('0B10', 2, 2),
-                 ('0o77', 8, 63),
-                 ]
-        for s, base, expected in cases:
-            assert string_to_int(s, base) == expected
-            assert string_to_int('+'+s, base) == expected
-            assert string_to_int('-'+s, base) == -expected
-            assert string_to_int(s+'\n', base) == expected
-            assert string_to_int('  +'+s, base) == expected
-            assert string_to_int('-'+s+'  ', base) == -expected
-
-    def test_string_to_int_error(self):
-        space = self.space
-        cases = ['0x123',    # must use base 0 or 16
-                 ' 0X12 ',
-                 '0b01',
-                 '0o01',
-                 '',
-                 '++12',
-                 '+-12',
-                 '-+12',
-                 '--12',
-                 '12a6',
-                 '12A6',
-                 'f',
-                 'Z',
-                 '.',
-                 '@',
-                 ]
-        for s in cases:
-            raises(ParseStringError, string_to_int, s)
-            raises(ParseStringError, string_to_int, '  '+s)
-            raises(ParseStringError, string_to_int, s+'  ')
-            raises(ParseStringError, string_to_int, '+'+s)
-            raises(ParseStringError, string_to_int, '-'+s)
-        raises(ParseStringError, string_to_int, '0x', 16)
-        raises(ParseStringError, string_to_int, '-0x', 16)
-
-        exc = raises(ParseStringError, string_to_int, '')
-        assert exc.value.msg == "invalid literal for int() with base 10: ''"
-        exc = raises(ParseStringError, string_to_int, '', 0)
-        assert exc.value.msg == "invalid literal for int() with base 0: ''"
-
-    def test_string_to_int_overflow(self):
-        import sys
-        space = self.space
-        raises(ParseStringOverflowError, string_to_int,
-               str(sys.maxint*17))
-
-    def test_string_to_int_not_overflow(self):
-        import sys
-        for x in [-sys.maxint-1, sys.maxint]:
-            y = string_to_int(str(x))
-            assert y == x
-
-    def test_string_to_int_base_error(self):
-        space = self.space
-        cases = [('1', 1),
-                 ('1', 37),
-                 ('a', 0),
-                 ('9', 9),
-                 ('0x123', 7),
-                 ('145cdf', 15),
-                 ('12', 37),
-                 ('12', 98172),
-                 ('12', -1),
-                 ('12', -908),
-                 ('12.3', 10),
-                 ('12.3', 13),
-                 ('12.3', 16),
-                 ]
-        for s, base in cases:
-            raises(ParseStringError, string_to_int, s, base)
-            raises(ParseStringError, string_to_int, '  '+s, base)
-            raises(ParseStringError, string_to_int, s+'  ', base)
-            raises(ParseStringError, string_to_int, '+'+s, base)
-            raises(ParseStringError, string_to_int, '-'+s, base)
-
-    def test_string_to_bigint(self):
-        assert string_to_bigint('123L').tolong() == 123
-        assert string_to_bigint('123L  ').tolong() == 123
-        raises(ParseStringError, string_to_bigint, 'L')
-        raises(ParseStringError, string_to_bigint, 'L  ')
-        assert string_to_bigint('123L', 4).tolong() == 27
-        assert string_to_bigint('123L', 30).tolong() == 27000 + 1800 + 90 + 21
-        assert string_to_bigint('123L', 22).tolong() == 10648 + 968 + 66 + 21
-        assert string_to_bigint('123L', 21).tolong() == 441 + 42 + 3
-        assert string_to_bigint('1891234174197319').tolong() == 1891234174197319
-
-    def test_string_to_float(self):
-        assert string_to_float('0') == 0.0
-        assert string_to_float('1') == 1.0
-        assert string_to_float('-1.5') == -1.5
-        assert string_to_float('1.5E2') == 150.0
-        assert string_to_float('2.5E-1') == 0.25
-        assert string_to_float('1e1111111111111') == float('1e1111111111111')
-        assert string_to_float('1e-1111111111111') == float('1e-1111111111111')
-        assert string_to_float('-1e1111111111111') == float('-1e1111111111111')
-        assert string_to_float('-1e-1111111111111') == float('-1e-1111111111111')
-        assert string_to_float('1e111111111111111111111') == float('1e111111111111111111111')
-        assert string_to_float('1e-111111111111111111111') == float('1e-111111111111111111111')
-        assert string_to_float('-1e111111111111111111111') == float('-1e111111111111111111111')
-        assert string_to_float('-1e-111111111111111111111') == float('-1e-111111111111111111111')
-
-        valid_parts = [['', '  ', ' \f\n\r\t\v'],
-                       ['', '+', '-'],
-                       ['00', '90', '.5', '2.4', '3.', '0.07',
-                        '12.3489749871982471987198371293717398256187563298638726'
-                        '2187362820947193247129871083561249818451804287437824015'
-                        '013816418758104762348932657836583048761487632840726386'],
-                       ['', 'e0', 'E+1', 'E-01', 'E42'],
-                       ['', '  ', ' \f\n\r\t\v'],
-                       ]
-        invalid_parts = [['#'],
-                         ['++', '+-', '-+', '--'],
-                         ['', '1.2.3', '.', '5..6'],
-                         ['E+', 'E-', 'e', 'e++', 'E++2'],
-                         ['#'],
-                         ]
-        for part0 in valid_parts[0]:
-            for part1 in valid_parts[1]:
-                for part2 in valid_parts[2]:
-                    for part3 in valid_parts[3]:
-                        for part4 in valid_parts[4]:
-                            s = part0+part1+part2+part3+part4
-                            assert (abs(string_to_float(s) - float(s)) <=
-                                    1E-13 * abs(float(s)))
-
-        for j in range(len(invalid_parts)):
-            for invalid in invalid_parts[j]:
-                for i in range(20):
-                    parts = [random.choice(lst) for lst in valid_parts]
-                    parts[j] = invalid
-                    s = ''.join(parts)
-                    print repr(s)
-                    if s.strip(): # empty s raises OperationError directly
-                        py.test.raises(ParseStringError, string_to_float, s)
-        py.test.raises(ParseStringError, string_to_float, "")
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -668,3 +668,36 @@
     if T == lltype.Float:
         return longlong2float(rffi.cast(rffi.LONGLONG, res))
     return rffi.cast(T, res)
+
+
+# String parsing support
+# ---------------------------
+
+def string_to_int(s, base=10):
+    """Utility to converts a string to an integer.
+    If base is 0, the proper base is guessed based on the leading
+    characters of 's'.  Raises ParseStringError in case of error.
+    Raises ParseStringOverflowError in case the result does not fit.
+    """
+    from rpython.rlib.rstring import NumberStringParser, \
+        ParseStringOverflowError, \
+        ParseStringError, strip_spaces
+    s = literal = strip_spaces(s)
+    p = NumberStringParser(s, literal, base, 'int')
+    base = p.base
+    result = 0
+    while True:
+        digit = p.next_digit()
+        if digit == -1:
+            return result
+
+        if p.sign == -1:
+            digit = -digit
+
+        try:
+            result = ovfcheck(result * base)
+            result = ovfcheck(result + digit)
+        except OverflowError:
+            raise ParseStringOverflowError(p)
+
+
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -252,6 +252,21 @@
 
     @staticmethod
     @jit.elidable
+    def fromstr(s, base=0, parser=None):
+        """As string_to_int(), but ignores an optional 'l' or 'L' suffix
+        and returns an rbigint."""
+        from rpython.rlib.rstring import NumberStringParser, \
+            strip_spaces
+        if parser is None:
+            s = literal = strip_spaces(s)
+            if (s.endswith('l') or s.endswith('L')) and base < 22:
+                # in base 22 and above, 'L' is a valid digit!  try: long('L',22)
+                s = s[:-1]
+            parser = NumberStringParser(s, literal, base, 'long')
+        return parse_digit_string(parser)
+
+    @staticmethod
+    @jit.elidable
     def frombytes(s, byteorder, signed):
         if byteorder not in ('big', 'little'):
             raise InvalidEndiannessError()
@@ -2289,7 +2304,7 @@
     return a
 
 def parse_digit_string(parser):
-    # helper for objspace.std.strutil
+    # helper for fromstr
     a = rbigint()
     base = parser.base
     digitmax = BASE_MAX[base]
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -27,6 +27,39 @@
 
 globals().update(rffi_platform.configure(CConfig))
 
+def string_to_float(s):
+    """
+    Conversion of string to float.
+    This version tries to only raise on invalid literals.
+    Overflows should be converted to infinity whenever possible.
+
+    Expects an unwrapped string and return an unwrapped float.
+    """
+    from rpython.rlib.rstring import strip_spaces, ParseStringError
+
+    s = strip_spaces(s)
+
+    if not s:
+        raise ParseStringError("empty string for float()")
+
+
+    low = s.lower()
+    if low == "-inf" or low == "-infinity":
+        return -INFINITY
+    elif low == "inf" or low == "+inf":
+        return INFINITY
+    elif low == "infinity" or low == "+infinity":
+        return INFINITY
+    elif low == "nan" or low == "+nan":
+        return NAN
+    elif low == "-nan":
+        return -NAN
+
+    try:
+        return rstring_to_float(s)
+    except ValueError:
+        raise ParseStringError("invalid literal for float(): '%s'" % s)
+
 def rstring_to_float(s):
     return rstring_to_float_impl(s)
 
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -186,6 +186,92 @@
             return False
     return True
 
+# -------------- numeric parsing support --------------------
+
+def strip_spaces(s):
+    # XXX this is not locale-dependent
+    p = 0
+    q = len(s)
+    while p < q and s[p] in ' \f\n\r\t\v':
+        p += 1
+    while p < q and s[q-1] in ' \f\n\r\t\v':
+        q -= 1
+    assert q >= p     # annotator hint, don't remove
+    return s[p:q]
+
+class ParseStringError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+
+class ParseStringOverflowError(Exception):
+    def __init__(self, parser):
+        self.parser = parser
+
+# iterator-like class
+class NumberStringParser:
+
+    def error(self):
+        raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
+                               (self.fname, self.original_base, self.literal))
+
+    def __init__(self, s, literal, base, fname):
+        self.literal = literal
+        self.fname = fname
+        sign = 1
+        if s.startswith('-'):
+            sign = -1
+            s = strip_spaces(s[1:])
+        elif s.startswith('+'):
+            s = strip_spaces(s[1:])
+        self.sign = sign
+        self.original_base = base
+
+        if base == 0:
+            if s.startswith('0x') or s.startswith('0X'):
+                base = 16
+            elif s.startswith('0b') or s.startswith('0B'):
+                base = 2
+            elif s.startswith('0'): # also covers the '0o' case
+                base = 8
+            else:
+                base = 10
+        elif base < 2 or base > 36:
+            raise ParseStringError, "%s() base must be >= 2 and <= 36" % (fname,)
+        self.base = base
+
+        if base == 16 and (s.startswith('0x') or s.startswith('0X')):
+            s = s[2:]
+        if base == 8 and (s.startswith('0o') or s.startswith('0O')):
+            s = s[2:]
+        if base == 2 and (s.startswith('0b') or s.startswith('0B')):
+            s = s[2:]
+        if not s:
+            self.error()
+        self.s = s
+        self.n = len(s)
+        self.i = 0
+
+    def rewind(self):
+        self.i = 0
+
+    def next_digit(self): # -1 => exhausted
+        if self.i < self.n:
+            c = self.s[self.i]
+            digit = ord(c)
+            if '0' <= c <= '9':
+                digit -= ord('0')
+            elif 'A' <= c <= 'Z':
+                digit = (digit - ord('A')) + 10
+            elif 'a' <= c <= 'z':
+                digit = (digit - ord('a')) + 10
+            else:
+                self.error()
+            if digit >= self.base:
+                self.error()
+            self.i += 1
+            return digit
+        else:
+            return -1
 
 # -------------- public API ---------------------------------
 
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.rlib.rarithmetic import *
+from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
 import sys
 import py
 
@@ -412,3 +413,123 @@
 
 def test_byteswap_interpret():
     interpret(test_byteswap, [])
+
+
+class TestStringToInt:
+
+    def test_string_to_int(self):
+        cases = [('0', 0),
+                 ('1', 1),
+                 ('9', 9),
+                 ('10', 10),
+                 ('09', 9),
+                 ('0000101', 101),    # not octal unless base 0 or 8
+                 ('5123', 5123),
+                 (' 0', 0),
+                 ('0  ', 0),
+                 (' \t \n   32313  \f  \v   \r  \n\r    ', 32313),
+                 ('+12', 12),
+                 ('-5', -5),
+                 ('- 5', -5),
+                 ('+ 5', 5),
+                 ('  -123456789 ', -123456789),
+                 ]
+        for s, expected in cases:
+            assert string_to_int(s) == expected
+            #assert string_to_bigint(s).tolong() == expected
+
+    def test_string_to_int_base(self):
+        cases = [('111', 2, 7),
+                 ('010', 2, 2),
+                 ('102', 3, 11),
+                 ('103', 4, 19),
+                 ('107', 8, 71),
+                 ('109', 10, 109),
+                 ('10A', 11, 131),
+                 ('10a', 11, 131),
+                 ('10f', 16, 271),
+                 ('10F', 16, 271),
+                 ('0x10f', 16, 271),
+                 ('0x10F', 16, 271),
+                 ('10z', 36, 1331),
+                 ('10Z', 36, 1331),
+                 ('12',   0, 12),
+                 ('015',  0, 13),
+                 ('0x10', 0, 16),
+                 ('0XE',  0, 14),
+                 ('0',    0, 0),
+                 ('0b11', 2, 3),
+                 ('0B10', 2, 2),
+                 ('0o77', 8, 63),
+                 ]
+        for s, base, expected in cases:
+            assert string_to_int(s, base) == expected
+            assert string_to_int('+'+s, base) == expected
+            assert string_to_int('-'+s, base) == -expected
+            assert string_to_int(s+'\n', base) == expected
+            assert string_to_int('  +'+s, base) == expected
+            assert string_to_int('-'+s+'  ', base) == -expected
+
+    def test_string_to_int_error(self):
+        cases = ['0x123',    # must use base 0 or 16
+                 ' 0X12 ',
+                 '0b01',
+                 '0o01',
+                 '',
+                 '++12',
+                 '+-12',
+                 '-+12',
+                 '--12',
+                 '12a6',
+                 '12A6',
+                 'f',
+                 'Z',
+                 '.',
+                 '@',
+                 ]
+        for s in cases:
+            py.test.raises(ParseStringError, string_to_int, s)
+            py.test.raises(ParseStringError, string_to_int, '  '+s)
+            py.test.raises(ParseStringError, string_to_int, s+'  ')
+            py.test.raises(ParseStringError, string_to_int, '+'+s)
+            py.test.raises(ParseStringError, string_to_int, '-'+s)
+        py.test.raises(ParseStringError, string_to_int, '0x', 16)
+        py.test.raises(ParseStringError, string_to_int, '-0x', 16)
+
+        exc = py.test.raises(ParseStringError, string_to_int, '')
+        assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = py.test.raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
+
+    def test_string_to_int_overflow(self):
+        import sys
+        py.test.raises(ParseStringOverflowError, string_to_int,
+               str(sys.maxint*17))
+
+    def test_string_to_int_not_overflow(self):
+        import sys
+        for x in [-sys.maxint-1, sys.maxint]:
+            y = string_to_int(str(x))
+            assert y == x
+
+    def test_string_to_int_base_error(self):
+        cases = [('1', 1),
+                 ('1', 37),
+                 ('a', 0),
+                 ('9', 9),
+                 ('0x123', 7),
+                 ('145cdf', 15),
+                 ('12', 37),
+                 ('12', 98172),
+                 ('12', -1),
+                 ('12', -908),
+                 ('12.3', 10),
+                 ('12.3', 13),
+                 ('12.3', 16),
+                 ]
+        for s, base in cases:
+            py.test.raises(ParseStringError, string_to_int, s, base)
+            py.test.raises(ParseStringError, string_to_int, '  '+s, base)
+            py.test.raises(ParseStringError, string_to_int, s+'  ', base)
+            py.test.raises(ParseStringError, string_to_int, '+'+s, base)
+            py.test.raises(ParseStringError, string_to_int, '-'+s, base)
diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -210,6 +210,18 @@
         assert x.tolong() == 0
         assert x.tobool() is False
 
+    def test_fromstr(self):
+        from rpython.rlib.rstring import ParseStringError
+        assert rbigint.fromstr('123L').tolong() == 123
+        assert rbigint.fromstr('123L  ').tolong() == 123
+        py.test.raises(ParseStringError, rbigint.fromstr, 'L')
+        py.test.raises(ParseStringError, rbigint.fromstr, 'L  ')
+        assert rbigint.fromstr('123L', 4).tolong() == 27
+        assert rbigint.fromstr('123L', 30).tolong() == 27000 + 1800 + 90 + 21
+        assert rbigint.fromstr('123L', 22).tolong() == 10648 + 968 + 66 + 21
+        assert rbigint.fromstr('123L', 21).tolong() == 441 + 42 + 3
+        assert rbigint.fromstr('1891234174197319').tolong() == 1891234174197319
+
     def test_add(self):
         x = 123456789123456789000000L
         y = 123858582373821923936744221L
diff --git a/rpython/rlib/test/test_rfloat.py b/rpython/rlib/test/test_rfloat.py
--- a/rpython/rlib/test/test_rfloat.py
+++ b/rpython/rlib/test/test_rfloat.py
@@ -7,6 +7,7 @@
 from rpython.rlib.rfloat import round_double
 from rpython.rlib.rfloat import erf, erfc, gamma, lgamma, isnan
 from rpython.rlib.rfloat import ulps_check, acc_check
+from rpython.rlib.rfloat import string_to_float
 from rpython.rlib.rbigint import rbigint
 
 def test_copysign():
@@ -232,3 +233,57 @@
     assert f(10.0) == 362880.0
     assert f(1720.0) == -42
     assert f(172.0) == -42
+
+
+
+def test_string_to_float():
+    from rpython.rlib.rstring import ParseStringError
+    import random
+    assert string_to_float('0') == 0.0
+    assert string_to_float('1') == 1.0
+    assert string_to_float('-1.5') == -1.5
+    assert string_to_float('1.5E2') == 150.0
+    assert string_to_float('2.5E-1') == 0.25
+    assert string_to_float('1e1111111111111') == float('1e1111111111111')
+    assert string_to_float('1e-1111111111111') == float('1e-1111111111111')
+    assert string_to_float('-1e1111111111111') == float('-1e1111111111111')
+    assert string_to_float('-1e-1111111111111') == float('-1e-1111111111111')
+    assert string_to_float('1e111111111111111111111') == float('1e111111111111111111111')
+    assert string_to_float('1e-111111111111111111111') == float('1e-111111111111111111111')
+    assert string_to_float('-1e111111111111111111111') == float('-1e111111111111111111111')
+    assert string_to_float('-1e-111111111111111111111') == float('-1e-111111111111111111111')
+
+    valid_parts = [['', '  ', ' \f\n\r\t\v'],
+                   ['', '+', '-'],
+                   ['00', '90', '.5', '2.4', '3.', '0.07',
+                    '12.3489749871982471987198371293717398256187563298638726'
+                    '2187362820947193247129871083561249818451804287437824015'
+                    '013816418758104762348932657836583048761487632840726386'],
+                   ['', 'e0', 'E+1', 'E-01', 'E42'],
+                   ['', '  ', ' \f\n\r\t\v'],
+                   ]
+    invalid_parts = [['#'],
+                     ['++', '+-', '-+', '--'],
+                     ['', '1.2.3', '.', '5..6'],
+                     ['E+', 'E-', 'e', 'e++', 'E++2'],
+                     ['#'],
+                     ]
+    for part0 in valid_parts[0]:
+        for part1 in valid_parts[1]:
+            for part2 in valid_parts[2]:
+                for part3 in valid_parts[3]:
+                    for part4 in valid_parts[4]:
+                        s = part0+part1+part2+part3+part4
+                        assert (abs(string_to_float(s) - float(s)) <=
+                                1E-13 * abs(float(s)))
+
+    for j in range(len(invalid_parts)):
+        for invalid in invalid_parts[j]:
+            for i in range(20):
+                parts = [random.choice(lst) for lst in valid_parts]
+                parts[j] = invalid
+                s = ''.join(parts)
+                print repr(s)
+                if s.strip(): # empty s raises OperationError directly
+                    py.test.raises(ParseStringError, string_to_float, s)
+    py.test.raises(ParseStringError, string_to_float, "")

From noreply at buildbot.pypy.org  Thu Jun 27 16:44:54 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Jun 2013 16:44:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20130627144454.9D7551C0351@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r65039:2fca073afc8e
Date: 2013-06-27 16:44 +0200
http://bitbucket.org/pypy/pypy/changeset/2fca073afc8e/

Log:	merge

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -29,12 +29,12 @@
     _application_traceback = None
 
     def __init__(self, w_type, w_value, tb=None):
-        assert w_type is not None
         self.setup(w_type)
         self._w_value = w_value
         self._application_traceback = tb
 
     def setup(self, w_type):
+        assert w_type is not None
         self.w_type = w_type
         if not we_are_translated():
             self.debug_excs = []
@@ -347,7 +347,6 @@
                 self.xstrings = strings
                 for i, _, attr in entries:
                     setattr(self, attr, args[i])
-                assert w_type is not None
 
             def _compute_value(self, space):
                 lst = [None] * (len(formats) + len(formats) + 1)
@@ -369,6 +368,18 @@
         _fmtcache2[formats] = OpErrFmt
     return OpErrFmt, strings
 
+class OpErrFmtNoArgs(OperationError):
+
+    def __init__(self, w_type, value):
+        self.setup(w_type)
+        self._value = value
+
+    def get_w_value(self, space):
+        w_value = self._w_value
+        if w_value is None:
+            self._w_value = w_value = space.wrap(self._value)
+        return w_value
+
 def get_operationerr_class(valuefmt):
     try:
         result = _fmtcache[valuefmt]
@@ -389,6 +400,8 @@
     %T - The result of space.type(w_arg).getname(space)
 
     """
+    if not len(args):
+        return OpErrFmtNoArgs(w_type, valuefmt)
     OpErrFmt, strings = get_operationerr_class(valuefmt)
     return OpErrFmt(w_type, strings, *args)
 operationerrfmt._annspecialcase_ = 'specialize:arg(1)'
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -33,6 +33,14 @@
     operr3 = operationerrfmt("w_type2", "a %s b %s c", "bar", "4b")
     assert operr3.__class__ is not operr.__class__
 
+def test_operationerrfmt_noargs(space):
+    operr = operationerrfmt(space.w_AttributeError, "no attribute 'foo'")
+    operr.normalize_exception(space)
+    val = operr.get_w_value(space)
+    assert space.isinstance_w(val, space.w_AttributeError)
+    w_repr = space.repr(val)
+    assert space.str_w(w_repr) == "AttributeError(\"no attribute 'foo'\",)"
+
 def test_operationerrfmt_T(space):
     operr = operationerrfmt(space.w_AttributeError,
                             "'%T' object has no attribute '%s'",
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -225,8 +225,8 @@
             space.raise_key_error(w_key)
 
     def descr_reversed(self, space):
-        raise OperationError(space.w_TypeError, space.wrap(
-                'argument to reversed() must be a sequence'))
+        raise operationerrfmt(space.w_TypeError,
+                              'argument to reversed() must be a sequence')
 
     def descr_copy(self, space):
         """D.copy() -> a shallow copy of D"""
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1222,7 +1222,8 @@
     def _safe_find(self, w_list, obj, start, stop):
         l = self.unerase(w_list.lstorage)
         for i in range(start, min(stop, len(l))):
-            if l[i] == obj:
+            val = l[i]
+            if val == obj:
                 return i
         raise ValueError
 
@@ -1542,18 +1543,6 @@
         if reverse:
             l.reverse()
 
-    def _safe_find(self, w_list, obj, start, stop):
-        from rpython.rlib.rfloat import isnan
-        if not isnan(obj):
-            return AbstractUnwrappedStrategy._safe_find(self, w_list, obj,
-                                                        start, stop)
-        # unwrapped nan != nan, finding it requires more effort
-        l = self.unerase(w_list.lstorage)
-        for i in range(start, min(stop, len(l))):
-            if isnan(l[i]):
-                return i
-        raise ValueError
-
 
 class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = None
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1314,10 +1314,6 @@
         non_list = NonList()
         assert [] != non_list
 
-    def test_nan_containment(self):
-        nan = float('nan')
-        assert nan in [nan]
-
 
 class AppTestForRangeLists(AppTestW_ListObject):
     spaceconfig = {"objspace.std.withrangelist": True}
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -3,35 +3,34 @@
 """
 
 import os
-from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
-                                            ConstFloat, BoxInt,
-                                            BoxFloat, INT, REF, FLOAT,
-                                            TargetToken)
-from rpython.jit.backend.x86.regloc import *
-from rpython.rtyper.lltypesystem import lltype, rffi, rstr
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rgc
 from rpython.jit.backend.llsupport import symbolic
+from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
+    unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
+from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc,
+     RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op)
+from rpython.jit.backend.x86 import rx86
+from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
+    IS_X86_64)
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
+from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
+    FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
+    ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
+    xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
+    X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG)
 from rpython.jit.codewriter import longlong
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
+    ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken)
 from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.backend.llsupport.descr import ArrayDescr
-from rpython.jit.backend.llsupport.descr import CallDescr
-from rpython.jit.backend.llsupport.descr import unpack_arraydescr
-from rpython.jit.backend.llsupport.descr import unpack_fielddescr
-from rpython.jit.backend.llsupport.descr import unpack_interiorfielddescr
-from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
-from rpython.jit.backend.llsupport.regalloc import FrameManager, BaseRegalloc,\
-     RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op
-from rpython.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE
-from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64
-from rpython.jit.backend.x86 import rx86
+from rpython.rlib import rgc
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
+
 
 class X86RegisterManager(RegisterManager):
-
     box_types = [INT, REF]
     all_regs = [ecx, eax, edx, ebx, esi, edi]
     no_lower_byte_regs = [esi, edi]
@@ -886,7 +885,6 @@
                 gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8))
         return gcmap
 
-
     def consider_setfield_gc(self, op):
         ofs, size, _ = unpack_fielddescr(op.getdescr())
         ofs_loc = imm(ofs)
@@ -950,7 +948,7 @@
     def consider_setarrayitem_gc(self, op):
         itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
         args = op.getarglist()
-        base_loc  = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
         if itemsize == 1:
             need_lower_byte = True
         else:
@@ -1311,7 +1309,6 @@
         #if jump_op is not None and jump_op.getdescr() is descr:
         #    self._compute_hint_frame_locations_from_descr(descr)
 
-
     def consider_keepalive(self, op):
         pass
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -351,7 +351,8 @@
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
         self.producer = {}
-        self.pendingfields = []
+        self.pendingfields = None # set temporarily to a list, normally by
+                                  # heap.py, as we're about to generate a guard
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.replaces_guard = {}
@@ -546,12 +547,14 @@
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
+            pendingfields = self.pendingfields
+            self.pendingfields = None
             if self.replaces_guard and op in self.replaces_guard:
                 self.replace_op(self.replaces_guard[op], op)
                 del self.replaces_guard[op]
                 return
             else:
-                op = self.store_final_boxes_in_guard(op)
+                op = self.store_final_boxes_in_guard(op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
         if op.result:
@@ -571,12 +574,13 @@
         else:
             assert False
 
-    def store_final_boxes_in_guard(self, op):
+    def store_final_boxes_in_guard(self, op, pendingfields):
+        assert pendingfields is not None
         descr = op.getdescr()
         assert isinstance(descr, compile.ResumeGuardDescr)
         modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
         try:
-            newboxes = modifier.finish(self, self.pendingfields)
+            newboxes = modifier.finish(self, pendingfields)
             if len(newboxes) > self.metainterp_sd.options.failargs_limit:
                 raise resume.TagOverflow
         except resume.TagOverflow:
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -7,7 +7,13 @@
     def __init__(self, unroll):
         self.last_label_descr = None
         self.unroll = unroll
-        
+
+    def emit_operation(self, op):
+        if op.is_guard():
+            if self.optimizer.pendingfields is None:
+                self.optimizer.pendingfields = []
+        Optimization.emit_operation(self, op)
+
     def optimize_CALL_PURE(self, op):
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -27,7 +27,7 @@
     snapshot0 = resume.Snapshot(None, [b0])
     fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
     #
-    opt.store_final_boxes_in_guard(op)
+    opt.store_final_boxes_in_guard(op, [])
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -61,7 +61,7 @@
     KARATSUBA_CUTOFF = 19
 else:
     KARATSUBA_CUTOFF = 38
-    
+
 KARATSUBA_SQUARE_CUTOFF = 2 * KARATSUBA_CUTOFF
 
 # For exponentiation, use the binary left-to-right algorithm
@@ -85,38 +85,41 @@
 
 def _load_unsigned_digit(x):
     return rffi.cast(UNSIGNED_TYPE, x)
-        
+
 _load_unsigned_digit._always_inline_ = True
 
 NULLDIGIT = _store_digit(0)
-ONEDIGIT  = _store_digit(1)
+ONEDIGIT = _store_digit(1)
 
 def _check_digits(l):
     for x in l:
         assert type(x) is type(NULLDIGIT)
         assert UDIGIT_MASK(x) & MASK == UDIGIT_MASK(x)
-            
+
 class InvalidEndiannessError(Exception):
     pass
 
 class InvalidSignednessError(Exception):
     pass
 
+
 class Entry(extregistry.ExtRegistryEntry):
     _about_ = _check_digits
+
     def compute_result_annotation(self, s_list):
         from rpython.annotator import model as annmodel
         assert isinstance(s_list, annmodel.SomeList)
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
+
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
 
+
 class rbigint(object):
     """This is a reimplementation of longs using a list of digits."""
     _immutable_ = True
     _immutable_fields_ = ["_digits"]
-    
 
     def __init__(self, digits=[NULLDIGIT], sign=0, size=0):
         if not we_are_translated():
@@ -164,7 +167,7 @@
     def numdigits(self):
         return self.size
     numdigits._always_inline_ = True
-    
+
     @staticmethod
     @jit.elidable
     def fromint(intval):
@@ -180,15 +183,14 @@
             ival = r_uint(intval)
         else:
             return NULLRBIGINT
-        
+
         carry = ival >> SHIFT
         if carry:
             return rbigint([_store_digit(ival & MASK),
                 _store_digit(carry)], sign, 2)
         else:
             return rbigint([_store_digit(ival & MASK)], sign, 1)
-            
-        
+
     @staticmethod
     @jit.elidable
     def frombool(b):
@@ -334,7 +336,7 @@
                 # Avoid bogus 0's
                 s = d ^ MASK if self.sign == -1 else d
                 while s:
-                    s >>=1
+                    s >>= 1
                     accumbits += 1
             else:
                 accumbits += SHIFT
@@ -355,7 +357,7 @@
 
             if self.sign == -1:
                 # Add a sign bit
-                accum |= (~_widen_digit(0)) << accumbits;
+                accum |= (~_widen_digit(0)) << accumbits
 
             result.append(chr(accum & 0xFF))
 
@@ -554,15 +556,15 @@
     def mul(self, b):
         asize = self.numdigits()
         bsize = b.numdigits()
-        
+
         a = self
-        
+
         if asize > bsize:
             a, b, asize, bsize = b, a, bsize, asize
 
         if a.sign == 0 or b.sign == 0:
             return NULLRBIGINT
-        
+
         if asize == 1:
             if a._digits[0] == NULLDIGIT:
                 return NULLRBIGINT
@@ -575,14 +577,14 @@
                     return rbigint([_store_digit(res & MASK), _store_digit(carry)], a.sign * b.sign, 2)
                 else:
                     return rbigint([_store_digit(res & MASK)], a.sign * b.sign, 1)
-                
-            result =  _x_mul(a, b, a.digit(0))
+
+            result = _x_mul(a, b, a.digit(0))
         elif USE_KARATSUBA:
             if a is b:
                 i = KARATSUBA_SQUARE_CUTOFF
             else:
                 i = KARATSUBA_CUTOFF
-                
+
             if asize <= i:
                 result = _x_mul(a, b)
                 """elif 2 * asize <= bsize:
@@ -608,13 +610,13 @@
                 return rbigint(self._digits[:self.size], 1, self.size)
             elif digit and digit & (digit - 1) == 0:
                 return self.rshift(ptwotable[digit])
-            
+
         div, mod = _divrem(self, other)
         if mod.sign * other.sign == -1:
             if div.sign == 0:
                 return ONENEGATIVERBIGINT
             div = div.sub(ONERBIGINT)
-            
+
         return div
 
     @jit.look_inside
@@ -625,7 +627,7 @@
     def mod(self, other):
         if self.sign == 0:
             return NULLRBIGINT
-        
+
         if other.sign != 0 and other.numdigits() == 1:
             digit = other.digit(0)
             if digit == 1:
@@ -648,7 +650,7 @@
                         size -= 1
                 else:
                     rem = self.digit(0) % digit
-                    
+
                 if rem == 0:
                     return NULLRBIGINT
                 mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1)
@@ -699,9 +701,9 @@
                     "cannot be negative when 3rd argument specified")
             # XXX failed to implement
             raise ValueError("bigint pow() too negative")
-        
+
         size_b = b.numdigits()
-        
+
         if c is not None:
             if c.sign == 0:
                 raise ValueError("pow() 3rd argument cannot be 0")
@@ -717,13 +719,13 @@
             #     return 0
             if c.numdigits() == 1 and c._digits[0] == ONEDIGIT:
                 return NULLRBIGINT
-   
+
             # if base < 0:
             #     base = base % modulus
             # Having the base positive just makes things easier.
             if a.sign < 0:
                 a = a.mod(c)
-            
+
         elif b.sign == 0:
             return ONERBIGINT
         elif a.sign == 0:
@@ -745,12 +747,12 @@
                     if a.sign == -1 and not digit % 2:
                         ret.sign = 1
                     return ret
-                
+
         # At this point a, b, and c are guaranteed non-negative UNLESS
         # c is NULL, in which case a may be negative. */
 
         z = rbigint([ONEDIGIT], 1, 1)
-        
+
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
         if size_b <= FIVEARY_CUTOFF:
@@ -766,7 +768,7 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 size_b -= 1
-                
+
         else:
             # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
             # This is only useful in the case where c != None.
@@ -799,7 +801,7 @@
                     # must get the next digit from 'b' in order to complete
                     if size_b == 0:
                         break # Done
-                        
+
                     size_b -= 1
                     assert size_b >= 0
                     bi = b.udigit(size_b)
@@ -813,7 +815,7 @@
                     z = _help_mult(z, table[index], c)
             #
             assert j == -5
-        
+
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
         return z
@@ -832,12 +834,12 @@
     def invert(self): #Implement ~x as -(x + 1)
         if self.sign == 0:
             return ONENEGATIVERBIGINT
-        
+
         ret = self.add(ONERBIGINT)
         ret.sign = -ret.sign
         return ret
-        
-    @jit.elidable    
+
+    @jit.elidable
     def lshift(self, int_other):
         if int_other < 0:
             raise ValueError("negative shift count")
@@ -846,14 +848,14 @@
 
         # wordshift, remshift = divmod(int_other, SHIFT)
         wordshift = int_other // SHIFT
-        remshift  = int_other - wordshift * SHIFT
+        remshift = int_other - wordshift * SHIFT
 
         if not remshift:
             # So we can avoid problems with eq, AND avoid the need for normalize.
             if self.sign == 0:
                 return self
             return rbigint([NULLDIGIT] * wordshift + self._digits, self.sign, self.size + wordshift)
-        
+
         oldsize = self.numdigits()
         newsize = oldsize + wordshift + 1
         z = rbigint([NULLDIGIT] * newsize, self.sign, newsize)
@@ -865,7 +867,7 @@
             accum >>= SHIFT
             wordshift += 1
             j += 1
-        
+
         newsize -= 1
         assert newsize >= 0
         z.setdigit(newsize, accum)
@@ -873,7 +875,7 @@
         z._normalize()
         return z
     lshift._always_inline_ = True # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def lqshift(self, int_other):
         " A quicker one with much less checks, int_other is valid and for the most part constant."
@@ -893,7 +895,7 @@
         z._normalize()
         return z
     lqshift._always_inline_ = True # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def rshift(self, int_other, dont_invert=False):
         if int_other < 0:
@@ -925,7 +927,7 @@
         z._normalize()
         return z
     rshift._always_inline_ = 'try' # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def and_(self, other):
         return _bitwise(self, '&', other)
@@ -983,7 +985,7 @@
             self._digits = [NULLDIGIT]
 
     _normalize._always_inline_ = True
-    
+
     @jit.elidable
     def bit_length(self):
         i = self.numdigits()
@@ -1033,7 +1035,7 @@
     # is NULL.
     if c is not None:
         res = res.mod(c)
-        
+
     return res
 
 def digits_from_nonneg_long(l):
@@ -1118,7 +1120,7 @@
 
 def _x_sub(a, b):
     """ Subtract the absolute values of two integers. """
-    
+
     size_a = a.numdigits()
     size_b = b.numdigits()
     sign = 1
@@ -1139,7 +1141,7 @@
             sign = -1
             a, b = b, a
         size_a = size_b = i+1
-        
+
     z = rbigint([NULLDIGIT] * size_a, sign, size_a)
     borrow = UDIGIT_TYPE(0)
     i = _load_unsigned_digit(0)
@@ -1157,7 +1159,7 @@
         borrow >>= SHIFT
         #borrow &= 1
         i += 1
-        
+
     assert borrow == 0
     z._normalize()
     return z
@@ -1167,7 +1169,7 @@
 for x in range(SHIFT-1):
     ptwotable[r_longlong(2 << x)] = x+1
     ptwotable[r_longlong(-2 << x)] = x+1
-    
+
 def _x_mul(a, b, digit=0):
     """
     Grade school multiplication, ignoring the signs.
@@ -1216,14 +1218,14 @@
             i += 1
         z._normalize()
         return z
-    
+
     elif digit:
         if digit & (digit - 1) == 0:
             return b.lqshift(ptwotable[digit])
-        
+
         # Even if it's not power of two it can still be useful.
         return _muladd1(b, digit)
-        
+
     z = rbigint([NULLDIGIT] * (size_a + size_b), 1)
     # gradeschool long mult
     i = UDIGIT_TYPE(0)
@@ -1273,7 +1275,7 @@
     """
     asize = a.numdigits()
     bsize = b.numdigits()
-    
+
     # (ah*X+al)(bh*X+bl) = ah*bh*X*X + (ah*bl + al*bh)*X + al*bl
     # Let k = (ah+al)*(bh+bl) = ah*bl + al*bh  + ah*bh + al*bl
     # Then the original product is
@@ -1356,7 +1358,7 @@
         t2 = _x_add(bh, bl)
 
     t3 = t1.mul(t2)
-    assert t3.sign >=0
+    assert t3.sign >= 0
 
     # Add t3.  It's not obvious why we can't run out of room here.
     # See the (*) comment after this function.
@@ -1436,9 +1438,9 @@
     #bslice = rbigint([0] * asize, 1)
     # XXX we cannot pre-allocate, see comments below!
     # XXX prevent one list from being created.
-    bslice = rbigint(sign = 1)
-    
-    nbdone = 0;
+    bslice = rbigint(sign=1)
+
+    nbdone = 0
     while bsize > 0:
         nbtouse = min(bsize, asize)
 
@@ -1487,7 +1489,7 @@
     The sign of a is ignored; n should not be zero.
     """
     assert n > 0 and n <= MASK
-        
+
     size = a.numdigits()
     z = rbigint([NULLDIGIT] * size, 1, size)
     rem = _inplace_divrem1(z, a, n)
@@ -1568,7 +1570,7 @@
     """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the top.
     """
-    
+
     carry = 0
     assert 0 <= d and d < SHIFT
     i = 0
@@ -1577,18 +1579,18 @@
         z.setdigit(i, acc)
         carry = acc >> SHIFT
         i += 1
-        
+
     return carry
 
 def _v_rshift(z, a, m, d):
     """ Shift digit vector a[0:m] d bits right, with 0 <= d < PyLong_SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the bottom.
     """
-    
+
     carry = _widen_digit(0)
     acc = _widen_digit(0)
     mask = (1 << d) - 1
-    
+
     assert 0 <= d and d < SHIFT
     i = m-1
     while i >= 0:
@@ -1596,7 +1598,7 @@
         carry = acc & mask
         z.setdigit(i, acc >> d)
         i -= 1
-        
+
     return carry
 
 def _x_divrem(v1, w1):
@@ -1604,13 +1606,13 @@
     size_v = v1.numdigits()
     size_w = w1.numdigits()
     assert size_v >= size_w and size_w > 1
-    
+
     v = rbigint([NULLDIGIT] * (size_v + 1), 1, size_v + 1)
     w = rbigint([NULLDIGIT] * size_w, 1, size_w)
-    
+
     """ normalize: shift w1 left so that its top digit is >= PyLong_BASE/2.
         shift v1 left by the same amount. Results go into w and v. """
-        
+
     d = SHIFT - bits_in_digit(w1.digit(abs(size_w-1)))
     carry = _v_lshift(w, w1, size_w, d)
     assert carry == 0
@@ -1618,7 +1620,7 @@
     if carry != 0 or v.digit(abs(size_v-1)) >= w.digit(abs(size_w-1)):
         v.setdigit(size_v, carry)
         size_v += 1
-        
+
     """ Now v->ob_digit[size_v-1] < w->ob_digit[size_w-1], so quotient has
         at most (and usually exactly) k = size_v - size_w digits. """
     k = size_v - size_w
@@ -1627,10 +1629,10 @@
         assert _v_rshift(w, v, size_w, d) == 0
         w._normalize()
         return rbigint([NULLDIGIT]), w
-    
+
     assert k > 0
     a = rbigint([NULLDIGIT] * k, 1, k)
-    
+
     wm1 = w.widedigit(abs(size_w-1))
     wm2 = w.widedigit(abs(size_w-2))
 
@@ -1640,7 +1642,7 @@
         assert j >= 0
         """ inner loop: divide vk[0:size_w+1] by w0[0:size_w], giving
             single-digit quotient q, remainder in vk[0:size_w]. """
-            
+
         # estimate quotient digit q; may overestimate by 1 (rare)
         if j >= size_v:
             vtop = 0
@@ -1653,9 +1655,9 @@
         while wm2 * q > ((r << SHIFT) | v.widedigit(abs(j-2))):
             q -= 1
             r += wm1
-            
+
         #assert q <= MASK+1, We need to compare to BASE <=, but ehm, it gives a buildin long error. So we ignore this.
-        
+
         # subtract q*w0[0:size_w] from vk[0:size_w+1]
         zhi = 0
         i = 0
@@ -1664,7 +1666,7 @@
             v.setdigit(k+i, z)
             zhi = z >> SHIFT
             i += 1
-        
+
         # add w back if q was too large (this branch taken rarely)
         if vtop + zhi < 0:
             carry = UDIGIT_TYPE(0)
@@ -1675,21 +1677,20 @@
                 carry >>= SHIFT
                 i += 1
             q -= 1
-            
+
         # store quotient digit
         a.setdigit(k, q)
         k -= 1
         j -= 1
-        
-        
+
     carry = _v_rshift(w, v, size_w, d)
     assert carry == 0
-    
+
     a._normalize()
     w._normalize()
-    
+
     return a, w
-        
+
 def _divrem(a, b):
     """ Long division with remainder, top-level routine """
     size_a = a.numdigits()
@@ -1906,14 +1907,16 @@
         da = float(a.digit(a_size))
         while True:
             a_size -= 1
-            if a_size < 0: break
+            if a_size < 0:
+                break
             da = da * BASE_AS_FLOAT + a.digit(a_size)
 
         b_size -= 1
         db = float(b.digit(b_size))
         while True:
             b_size -= 1
-            if b_size < 0: break
+            if b_size < 0:
+                break
             db = db * BASE_AS_FLOAT + b.digit(b_size)
 
         return _truediv_result(da / db, negate)
@@ -2048,7 +2051,6 @@
 
 def _format_int(val, digits):
     base = len(digits)
-    j = 0
     out = []
     while val:
         out.append(digits[val % base])
@@ -2101,7 +2103,7 @@
     if negative:
         output.append('-')
     output.append(prefix)
-    _format_recursive(x,len(pts)-1, output, pts, digits, output.getlength())
+    _format_recursive(x, len(pts)-1, output, pts, digits, output.getlength())
 
     output.append(suffix)
     return output.build()
@@ -2171,7 +2173,7 @@
             digb = b.digit(i) ^ maskb
         else:
             digb = maskb
-            
+
         if op == '&':
             z.setdigit(i, diga & digb)
         elif op == '|':
@@ -2179,11 +2181,11 @@
         elif op == '^':
             z.setdigit(i, diga ^ digb)
         i += 1
-        
+
     z._normalize()
     if negz == 0:
         return z
-    
+
     return z.invert()
 _bitwise._annspecialcase_ = "specialize:arg(1)"
 

From noreply at buildbot.pypy.org  Thu Jun 27 17:53:18 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Jun 2013 17:53:18 +0200 (CEST)
Subject: [pypy-commit] pypy kill-gen-store-back-in: Implement a hint to call
 gen_store_back by hand. useful for generators
Message-ID: <20130627155318.7D9A61C0351@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: kill-gen-store-back-in
Changeset: r65040:8001b03cceff
Date: 2013-06-27 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/8001b03cceff/

Log:	Implement a hint to call gen_store_back by hand. useful for
	generators

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -521,6 +521,8 @@
             op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
                                  op.result)
             return [SpaceOperation('-live-', [], None), op1, None]
+        if hints.get('force_virtualizable'):
+            return SpaceOperation('hint_force_virtualizable', [op.args[0]], None)
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1306,6 +1306,10 @@
     def bhimpl_force_virtualizable(cpu, v, descr):
         cpu.bh_force_virtualizable(v, descr)
 
+    @arguments("r")
+    def bhimpl_hint_force_virtualizable(r):
+        pass
+
     @arguments("cpu", "d", returns="r")
     def bhimpl_new(cpu, descr):
         return cpu.bh_new(descr)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -674,6 +674,10 @@
     opimpl_raw_load_i = _opimpl_raw_load
     opimpl_raw_load_f = _opimpl_raw_load
 
+    @arguments("box")
+    def opimpl_hint_force_virtualizable(self, box):
+        self.metainterp.gen_store_back_in_vable(box)
+
     @arguments("box", "descr", "descr", "orgpc")
     def opimpl_record_quasiimmut_field(self, box, fielddescr,
                                        mutatefielddescr, orgpc):
@@ -2509,6 +2513,33 @@
                                                         virtualizable)
             self.virtualizable_boxes.append(virtualizable_box)
 
+            vinfo = self.jitdriver_sd.virtualizable_info
+
+    def gen_store_back_in_vable(self, box):
+        vinfo = self.jitdriver_sd.virtualizable_info
+        if vinfo is not None:
+            # xxx only write back the fields really modified
+            vbox = self.virtualizable_boxes[-1]
+            if vbox is not box:
+                return # ignore the hint on non-standard virtualizable
+            for i in range(vinfo.num_static_extra_boxes):
+                fieldbox = self.virtualizable_boxes[i]
+                descr = vinfo.static_field_descrs[i]
+                self.execute_and_record(rop.SETFIELD_GC, descr, vbox, fieldbox)
+            i = vinfo.num_static_extra_boxes
+            virtualizable = vinfo.unwrap_virtualizable_box(vbox)
+            for k in range(vinfo.num_arrays):
+                descr = vinfo.array_field_descrs[k]
+                abox = self.execute_and_record(rop.GETFIELD_GC, descr, vbox)
+                descr = vinfo.array_descrs[k]
+                for j in range(vinfo.get_array_length(virtualizable, k)):
+                    itembox = self.virtualizable_boxes[i]
+                    i += 1
+                    self.execute_and_record(rop.SETARRAYITEM_GC, descr,
+                                            abox, ConstInt(j), itembox)
+            assert i + 1 == len(self.virtualizable_boxes)
+
+
     def replace_box(self, oldbox, newbox):
         assert isinstance(oldbox, Box)
         for frame in self.framestack:
diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -3,8 +3,9 @@
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
-from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin
-from rpython.jit.metainterp.warmspot import get_translator
+from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.warmspot import get_translator, get_stats
+from rpython.jit.metainterp.resoperation import rop
 from rpython.rlib.jit import JitDriver, hint, dont_look_inside, promote, virtual_ref
 from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.annlowlevel import hlstr
@@ -26,7 +27,6 @@
         return lltype_to_annotation(lltype.Void)
 
     def specialize_call(self, hop):
-        op = self.instance    # the LLOp object that was called
         args_v = [hop.inputarg(hop.args_r[0], 0),
                   hop.inputconst(lltype.Void, hop.args_v[1].value),
                   hop.inputconst(lltype.Void, {})]
@@ -1544,6 +1544,39 @@
         res = self.meta_interp(f, [])
         assert res == f()
 
+    def test_force_virtualizable_by_hint(self):
+        class Frame(object):
+            _virtualizable_ = ['x']
+
+        driver = JitDriver(greens = [], reds = ['i', 'frame'],
+                           virtualizables = ['frame'])
+
+        def f(frame, i):
+            while i > 0:
+                driver.jit_merge_point(i=i, frame=frame)
+                i -= 1
+                frame.x += 1
+            hint(frame, force_virtualizable=True)
+
+        def main():
+            frame = Frame()
+            frame.x = 0
+            s = 0
+            for i in range(20):
+                f(frame, 4)
+                s += frame.x
+            return s
+
+        r = self.meta_interp(main, [])
+        assert r == main()
+        # fish the bridge
+        loop = get_stats().get_all_loops()[0]
+        d = loop.operations[-3].getdescr()
+        bridge = getattr(d, '_llgraph_bridge', None)
+        if bridge is not None:
+            l = [op for op in
+                 bridge.operations if op.getopnum() == rop.SETFIELD_GC]
+            assert len(l) == 2
 
 class TestLLtype(ExplicitVirtualizableTests,
                  ImplicitVirtualizableTests,

From noreply at buildbot.pypy.org  Thu Jun 27 18:04:37 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 27 Jun 2013 18:04:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
	implement cfloat alias for complex64
Message-ID: <20130627160437.EACA61C2FB2@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65041:37ecba0fbf1c
Date: 2013-06-27 19:03 +0300
http://bitbucket.org/pypy/pypy/changeset/37ecba0fbf1c/

Log:	test, implement cfloat alias for complex64

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -71,6 +71,7 @@
         'complex_': 'interp_boxes.W_Complex128Box',
         'complex128': 'interp_boxes.W_Complex128Box',
         'complex64': 'interp_boxes.W_Complex64Box',
+        'cfloat': 'interp_boxes.W_Complex64Box',
     }
     if ENABLED_LONG_DOUBLE:
         long_double_dtypes = [
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -585,6 +585,7 @@
         import numpypy as numpy
 
         assert numpy.complex_ is numpy.complex128
+        assert numpy.cfloat is numpy.complex64
         assert numpy.complex64.__mro__ == (numpy.complex64,
             numpy.complexfloating, numpy.inexact, numpy.number, numpy.generic,
             object)

From noreply at buildbot.pypy.org  Thu Jun 27 18:33:12 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 18:33:12 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Implement transactions (incomplete so
	far)
Message-ID: <20130627163312.D02CB1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r306:6b07c127c560
Date: 2013-06-27 17:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/6b07c127c560/

Log:	Implement transactions (incomplete so far)

diff --git a/duhton/consobject.c b/duhton/consobject.c
--- a/duhton/consobject.c
+++ b/duhton/consobject.c
@@ -1,10 +1,5 @@
 #include "duhton.h"
 
-typedef struct {
-    DuOBJECT_HEAD
-    DuObject *car, *cdr;
-} DuConsObject;
-
 
 void cons_trace(DuConsObject *ob, void visit(gcptr *))
 {
diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -108,6 +108,11 @@
 DuObject *DuSymbol_FromString(const char *name);
 char *DuSymbol_AsString(DuObject *ob);
 
+typedef struct {
+    DuOBJECT_HEAD
+    DuObject *car, *cdr;
+} DuConsObject;
+
 DuObject *DuCons_New(DuObject *car, DuObject *cdr);
 DuObject *DuCons_Car(DuObject *cons);
 DuObject *DuCons_Cdr(DuObject *cons);
diff --git a/duhton/glob.c b/duhton/glob.c
--- a/duhton/glob.c
+++ b/duhton/glob.c
@@ -503,16 +503,14 @@
 
 DuObject *du_transaction(DuObject *cons, DuObject *locals)
 {
-    Du_FatalError("transaction: not implemented");
-#if 0
     if (cons == Du_None)
         Du_FatalError("transaction: expected at least one argument");
+
+    _du_read1(cons);
     DuObject *sym = _DuCons_CAR(cons);
     DuObject *rest = _DuCons_NEXT(cons);
     _DuFrame_EvalCall(locals, sym, rest, 0);
-    Du_INCREF(Du_None);
     return Du_None;
-#endif
 }
 
 DuObject *du_sleepms(DuObject *cons, DuObject *locals)
diff --git a/duhton/transaction.c b/duhton/transaction.c
--- a/duhton/transaction.c
+++ b/duhton/transaction.c
@@ -1,12 +1,140 @@
 #include "duhton.h"
+#include <pthread.h>
+#include <unistd.h>
 
+#define NUM_THREADS  4
+
+
+static DuConsObject du_pending_transactions = {
+    DuOBJECT_HEAD_INIT(DUTYPE_CONS),
+    NULL,
+    Du_None,
+};
+
+static pthread_mutex_t mutex_sleep = PTHREAD_MUTEX_INITIALIZER;
+static int thread_sleeping = 0;
+
+static void *run_thread(void *);   /* forward */
+
+static void run_all_threads(void)
+{
+    int i;
+    pthread_t th[NUM_THREADS];
+
+    for (i = 0; i < NUM_THREADS; i++) {
+        int status = pthread_create(&th[i], NULL, run_thread, NULL);
+        if (status != 0)
+            stm_fatalerror("status != 0\n");
+    }
+    for (i = 0; i < NUM_THREADS; i++) {
+        pthread_join(th[i], NULL);
+    }
+}
+
+/************************************************************/
 
 void Du_TransactionAdd(DuObject *code, DuObject *frame)
 {
-    /* XXX */
+    DuObject *cell = DuCons_New(code, frame);
+    DuObject *pending = (DuObject *)stm_thread_local_obj;
+
+    if (pending == NULL) {
+        pending = Du_None;
+    }
+    pending = DuCons_New(cell, pending);
+    stm_thread_local_obj = (gcptr)pending;
 }
 
 void Du_TransactionRun(void)
 {
-    /* XXX */
+    if (stm_thread_local_obj == NULL)
+        return;
+
+    DuConsObject *root = &du_pending_transactions;
+    _du_write1(root);
+    root->cdr = stm_thread_local_obj;
+
+    stm_commit_transaction();
+    run_all_threads();
+    stm_begin_inevitable_transaction();
 }
+
+/************************************************************/
+
+static DuObject *next_cell(void)
+{
+    DuObject *pending = (DuObject *)stm_thread_local_obj;
+
+    if (pending == NULL) {
+        /* fish from the global list of pending transactions */
+        DuConsObject *root;
+
+      restart:
+        root = &du_pending_transactions;
+        _du_read1(root);
+
+        if (root->cdr != Du_None) {
+            DuObject *cell = root->cdr;
+            _du_write1(root);
+
+            _du_read1(cell);
+            DuObject *result = _DuCons_CAR(cell);
+            root->cdr = _DuCons_NEXT(cell);
+
+            return result;
+        }
+        else {
+            /* nothing to do, wait */
+            thread_sleeping++;
+            if (thread_sleeping == NUM_THREADS) {
+                pthread_mutex_unlock(&mutex_sleep);
+            }
+            stm_commit_transaction();
+            pthread_mutex_lock(&mutex_sleep);
+            stm_begin_inevitable_transaction();
+            if (thread_sleeping == NUM_THREADS) {
+                pthread_mutex_unlock(&mutex_sleep);
+                return NULL;
+            }
+            thread_sleeping--;
+            goto restart;
+        }
+    }
+
+    /* we have at least one thread-local transaction pending */
+    _du_read1(pending);
+    DuObject *result = _DuCons_CAR(pending);
+    DuObject *next = _DuCons_NEXT(pending);
+
+    if (next != Du_None) {
+        /* we have more than one: add the others to the global list */
+        assert(!"XXX");
+        abort();
+    }
+
+    return result;
+}
+
+int run_transaction(gcptr cell, int retry_counter)
+{
+    DuObject *code  = DuCons_Car(cell);
+    DuObject *frame = DuCons_Cdr(cell);
+    Du_Progn(code, frame);
+    return 0;
+}
+
+void *run_thread(void *ignored)
+{
+    stm_initialize();
+
+    while (1) {
+        /* we are inevitable here */
+        DuObject *cell = next_cell();
+        if (cell == NULL)
+            break;
+        stm_perform_transaction(cell, run_transaction);
+    }
+
+    stm_finalize();
+    return NULL;
+}

From noreply at buildbot.pypy.org  Thu Jun 27 18:33:13 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 18:33:13 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20130627163313.E8E1E1C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r307:7543e6ddbf3a
Date: 2013-06-27 17:06 +0200
http://bitbucket.org/pypy/stmgc/changeset/7543e6ddbf3a/

Log:	Fix

diff --git a/duhton/transaction.c b/duhton/transaction.c
--- a/duhton/transaction.c
+++ b/duhton/transaction.c
@@ -102,6 +102,8 @@
     }
 
     /* we have at least one thread-local transaction pending */
+    stm_thread_local_obj = NULL;
+
     _du_read1(pending);
     DuObject *result = _DuCons_CAR(pending);
     DuObject *next = _DuCons_NEXT(pending);

From noreply at buildbot.pypy.org  Thu Jun 27 18:33:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 18:33:14 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Adapt the tests to the large amount of
	stderr output
Message-ID: <20130627163314.F3A751C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r308:c03eaca800d4
Date: 2013-06-27 17:32 +0200
http://bitbucket.org/pypy/stmgc/changeset/c03eaca800d4/

Log:	Adapt the tests to the large amount of stderr output

diff --git a/duhton/test/support.py b/duhton/test/support.py
--- a/duhton/test/support.py
+++ b/duhton/test/support.py
@@ -1,4 +1,4 @@
-import py, re
+import py, re, select
 import os, subprocess
 from cStringIO import StringIO
 
@@ -12,8 +12,21 @@
                              stderr = subprocess.PIPE)
     popen.stdin.write(stdin)
     popen.stdin.close()
-    result = popen.stdout.read()
-    error = popen.stderr.read()
+    #
+    result = []
+    error = []
+    owait = {popen.stdout: result, popen.stderr: error}
+    while owait:
+        iwtd, _, _ = select.select(owait.keys(), [], [])
+        for o in iwtd:
+            data = os.read(o.fileno(), 4096)
+            if data:
+                owait[o].append(data)
+            else:
+                del owait[o]
+    #
+    result = ''.join(result)
+    error = ''.join(error)
     exitcode = popen.wait()
     if exitcode:
         raise OSError("%r failed (exit code %r)\n" % (cmdargs[0], exitcode) +
diff --git a/duhton/test/test_transaction.py b/duhton/test/test_transaction.py
--- a/duhton/test/test_transaction.py
+++ b/duhton/test/test_transaction.py
@@ -16,11 +16,11 @@
 
             (defun g (thread n)
                 (set c (+ (get c) 1))
-                (if (> (get c) 2000)
+                (if (> (get c) 200)
                     (print (quote overflow) (get c))
-                  (if (< n 1000)
+                  (if (< n 100)
                       (transaction f thread (+ n 1))
-                    (if (< (get c) 2000)
+                    (if (< (get c) 200)
                         (print (quote not-enough))
                       (print (quote ok))))))
 
@@ -35,17 +35,18 @@
 
 def test_conflict_list():
     for i in range(20):
+        print 'test_conflict_list', i
         res = run("""
 
             (setq lst (list 0))
 
             (defun g (thread n)
                 (set lst 0 (+ (get lst 0) 1))
-                (if (> (get lst 0) 2000)
+                (if (> (get lst 0) 200)
                     (print (quote overflow) (get lst 0))
-                  (if (< n 1000)
+                  (if (< n 100)
                       (transaction f thread (+ n 1))
-                    (if (< (get lst 0) 2000)
+                    (if (< (get lst 0) 200)
                         (print (quote not-enough))
                       (print (quote ok))))))
 
@@ -63,11 +64,12 @@
 
 def test_list_length():
     for i in range(20):
+        print 'test_list_length', i
         res = run("""
             (setq lst (list))
             (defun f ()
                 (setq n (len lst))
-                (if (< n 1000)
+                (if (< n 100)
                     (transaction f)
                   (print (quote done)))
                 (append lst n)
@@ -79,9 +81,10 @@
 
 def test_list_pop():
     for i in range(20):
+        print 'test_list_pop', i
         res = run("""
             (setq lst (list))
-            (while (< (len lst) 1000)
+            (while (< (len lst) 100)
                 (append lst (len lst)))
             (defun f ()
                 (if (== (len lst) 0)

From noreply at buildbot.pypy.org  Thu Jun 27 18:33:16 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 18:33:16 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add tons of debugging support,
 found where I forgot the save/restore.
Message-ID: <20130627163316.114A31C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r309:92111cbf8669
Date: 2013-06-27 17:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/92111cbf8669/

Log:	Add tons of debugging support, found where I forgot the
	save/restore.

diff --git a/duhton/duhton.h b/duhton/duhton.h
--- a/duhton/duhton.h
+++ b/duhton/duhton.h
@@ -146,22 +146,41 @@
 void Du_TransactionRun(void);
 
 
-#define _du_save1(p1)           (stm_push_root((DuObject *)(p1)))
-#define _du_save2(p1,p2)        (stm_push_root((DuObject *)(p1)),  \
-                                 stm_push_root((DuObject *)(p2)))
-#define _du_save3(p1,p2,p3)     (stm_push_root((DuObject *)(p1)),  \
-                                 stm_push_root((DuObject *)(p2)),  \
-                                 stm_push_root((DuObject *)(p3)))
+#define _du_save1(p1)           (_push_root((DuObject *)(p1)))
+#define _du_save2(p1,p2)        (_push_root((DuObject *)(p1)),  \
+                                 _push_root((DuObject *)(p2)))
+#define _du_save3(p1,p2,p3)     (_push_root((DuObject *)(p1)),  \
+                                 _push_root((DuObject *)(p2)),  \
+                                 _push_root((DuObject *)(p3)))
 
-#define _du_restore1(p1)        (p1 = (typeof(p1))stm_pop_root())
-#define _du_restore2(p1,p2)     (p2 = (typeof(p2))stm_pop_root(),  \
-                                 p1 = (typeof(p1))stm_pop_root())
-#define _du_restore3(p1,p2,p3)  (p3 = (typeof(p3))stm_pop_root(),  \
-                                 p2 = (typeof(p2))stm_pop_root(),  \
-                                 p1 = (typeof(p1))stm_pop_root())
+#define _du_restore1(p1)        (p1 = (typeof(p1))_pop_root())
+#define _du_restore2(p1,p2)     (p2 = (typeof(p2))_pop_root(),  \
+                                 p1 = (typeof(p1))_pop_root())
+#define _du_restore3(p1,p2,p3)  (p3 = (typeof(p3))_pop_root(),  \
+                                 p2 = (typeof(p2))_pop_root(),  \
+                                 p1 = (typeof(p1))_pop_root())
 
 #define _du_read1(p1)    (p1 = (typeof(p1))stm_read_barrier((DuObject *)(p1)))
 #define _du_write1(p1)   (p1 = (typeof(p1))stm_write_barrier((DuObject *)(p1)))
 
 
+#ifdef NDEBUG
+# define _push_root(ob)     stm_push_root(ob)
+# define _pop_root()        stm_pop_root()
+#else
+# define _check_not_free(ob)                                    \
+    assert(stm_get_tid((DuObject *)(ob)) > DUTYPE_INVALID &&    \
+           stm_get_tid((DuObject *)(ob)) < _DUTYPE_TOTAL)
+static inline void _push_root(gcptr ob) {
+    if (ob) _check_not_free(ob);
+    stm_push_root(ob);
+}
+static inline gcptr _pop_root(void) {
+    gcptr ob = stm_pop_root();
+    if (ob) _check_not_free(ob);
+    return ob;
+}
+#endif
+
+
 #endif  /* _DUHTON_H_ */
diff --git a/duhton/glob.c b/duhton/glob.c
--- a/duhton/glob.c
+++ b/duhton/glob.c
@@ -102,7 +102,9 @@
 
 DuObject *du_print(DuObject *cons, DuObject *locals)
 {
+    _du_save2(cons, locals);
     DuObject *lst = DuList_New();
+    _du_restore2(cons, locals);
 
     while (cons != Du_None) {
         _du_read1(cons);
diff --git a/duhton/test/test_gc.py b/duhton/test/test_gc.py
new file mode 100644
--- /dev/null
+++ b/duhton/test/test_gc.py
@@ -0,0 +1,15 @@
+from support import run
+
+
+def test_long1():
+    got = run("""
+        (defun g (n)
+         (if (>= n 7)
+             (print n)
+           (g (+ n 1))
+           (g (+ n 2))))
+        (g 0)
+    """)
+    pieces = got.splitlines()
+    assert len(pieces) == xxx
+    assert 0

From noreply at buildbot.pypy.org  Thu Jun 27 18:33:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 18:33:21 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Finish the tests,
	which all pass now.  Yay!
Message-ID: <20130627163321.A4B9C1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r310:102472b49110
Date: 2013-06-27 17:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/102472b49110/

Log:	Finish the tests, which all pass now. Yay!

diff --git a/duhton/test/test_gc.py b/duhton/test/test_gc.py
--- a/duhton/test/test_gc.py
+++ b/duhton/test/test_gc.py
@@ -4,12 +4,19 @@
 def test_long1():
     got = run("""
         (defun g (n)
-         (if (>= n 7)
+         (if (>= n 12)
              (print n)
            (g (+ n 1))
            (g (+ n 2))))
         (g 0)
     """)
     pieces = got.splitlines()
-    assert len(pieces) == xxx
-    assert 0
+    expected = []
+    def g(n):
+        if n >= 12:
+            expected.append(str(n))
+        else:
+            g(n + 1)
+            g(n + 2)
+    g(0)
+    assert pieces == expected
diff --git a/duhton/test/test_transaction.py b/duhton/test/test_transaction.py
--- a/duhton/test/test_transaction.py
+++ b/duhton/test/test_transaction.py
@@ -8,6 +8,20 @@
     assert run("(defun f(x) (if (< x 20) (transaction f (+ x 1)) (print x))) "
                "(print (f 0))") == "None\n20\n"
 
+def test_multiple_starts():
+    got = run("""
+        (defun g (n)
+         (if (>= n 12)
+             (print n)
+           (g (+ n 1))
+           (g (+ n 2))))
+        (transaction g 0)
+    """)
+    pieces = got.splitlines()
+    assert len(pieces) == 377
+    assert pieces.count('12') == 233
+    assert pieces.count('13') == 144
+
 def test_conflict_container():
     for i in range(20):
         res = run("""

From noreply at buildbot.pypy.org  Thu Jun 27 18:33:22 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 18:33:22 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add a longer version
Message-ID: <20130627163322.A42F71C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r311:003fcd027fc5
Date: 2013-06-27 17:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/003fcd027fc5/

Log:	Add a longer version

diff --git a/duhton/test/test_gc.py b/duhton/test/test_gc.py
--- a/duhton/test/test_gc.py
+++ b/duhton/test/test_gc.py
@@ -1,22 +1,25 @@
 from support import run
 
 
-def test_long1():
+def test_long1(limit=12):
     got = run("""
         (defun g (n)
-         (if (>= n 12)
+         (if (>= n %d)
              (print n)
            (g (+ n 1))
            (g (+ n 2))))
         (g 0)
-    """)
+    """ % limit)
     pieces = got.splitlines()
     expected = []
     def g(n):
-        if n >= 12:
+        if n >= limit:
             expected.append(str(n))
         else:
             g(n + 1)
             g(n + 2)
     g(0)
     assert pieces == expected
+
+def test_verylong1():
+    test_long1(limit=16)

From noreply at buildbot.pypy.org  Thu Jun 27 19:20:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 19:20:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Finish transaction.c
Message-ID: <20130627172041.546E01C3050@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r312:4d0b9fa7c7b8
Date: 2013-06-27 19:20 +0200
http://bitbucket.org/pypy/stmgc/changeset/4d0b9fa7c7b8/

Log:	Finish transaction.c

diff --git a/duhton/test/test_transaction.py b/duhton/test/test_transaction.py
--- a/duhton/test/test_transaction.py
+++ b/duhton/test/test_transaction.py
@@ -13,15 +13,27 @@
         (defun g (n)
          (if (>= n 12)
              (print n)
-           (g (+ n 1))
-           (g (+ n 2))))
-        (transaction g 0)
+           (transaction g (+ n 1))
+           (transaction g (+ n 2))))
+        (g 0)
     """)
     pieces = got.splitlines()
     assert len(pieces) == 377
     assert pieces.count('12') == 233
     assert pieces.count('13') == 144
 
+def test_multiple_starts_conflicts():
+    got = run("""
+        (setq c (list))
+        (defun g (n)
+         (if (>= n 12)
+             (append c n)
+           (transaction g (+ n 1))
+           (transaction g (+ n 2))))
+        (g 0)
+    """)
+    assert got == ''    # how to print the final c ??  at least check no crash
+
 def test_conflict_container():
     for i in range(20):
         res = run("""
diff --git a/duhton/transaction.c b/duhton/transaction.c
--- a/duhton/transaction.c
+++ b/duhton/transaction.c
@@ -2,7 +2,9 @@
 #include <pthread.h>
 #include <unistd.h>
 
+#ifndef NUM_THREADS
 #define NUM_THREADS  4
+#endif
 
 
 static DuConsObject du_pending_transactions = {
@@ -110,8 +112,21 @@
 
     if (next != Du_None) {
         /* we have more than one: add the others to the global list */
-        assert(!"XXX");
-        abort();
+        DuObject *tail = next;
+
+        while (1) {
+            _du_read1(tail);
+            DuObject *tailnext = ((DuConsObject *)tail)->cdr;
+            if (tailnext == Du_None)
+                break;
+            tail = tailnext;
+        }
+
+        DuConsObject * root = &du_pending_transactions;
+        _du_write1(tail);
+        _du_write1(root);
+        ((DuConsObject *)tail)->cdr = root->cdr;
+        root->cdr = next;
     }
 
     return result;

From noreply at buildbot.pypy.org  Thu Jun 27 19:45:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Jun 2013 19:45:02 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Optimize
Message-ID: <20130627174502.6DA441C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r313:bf59a2c0e4eb
Date: 2013-06-27 19:44 +0200
http://bitbucket.org/pypy/stmgc/changeset/bf59a2c0e4eb/

Log:	Optimize

diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -66,7 +66,7 @@
 static const revision_t GCFLAG_PUBLIC                 = STM_FIRST_GCFLAG << 2;
 static const revision_t GCFLAG_PREBUILT_ORIGINAL      = STM_FIRST_GCFLAG << 3;
 static const revision_t GCFLAG_PUBLIC_TO_PRIVATE      = STM_FIRST_GCFLAG << 4;
-static const revision_t GCFLAG_WRITE_BARRIER          = STM_FIRST_GCFLAG << 5;
+// in stmgc.h:          GCFLAG_WRITE_BARRIER          = STM_FIRST_GCFLAG << 5;
 static const revision_t GCFLAG_NURSERY_MOVED          = STM_FIRST_GCFLAG << 6;
 static const revision_t GCFLAG_BACKUP_COPY  /*debug*/ = STM_FIRST_GCFLAG << 7;
 static const revision_t GCFLAG_STUB         /*debug*/ = STM_FIRST_GCFLAG << 8;
diff --git a/c4/lists.h b/c4/lists.h
--- a/c4/lists.h
+++ b/c4/lists.h
@@ -185,7 +185,7 @@
    more.
 */
 
-#define FX_MASK      65535
+//#define FX_MASK      65535    in stmgc.h
 #define FX_ENTRIES   ((FX_MASK + 1) / sizeof(char *))
 #define FX_TOTAL     (FX_ENTRIES * 4 / 3)
 
@@ -206,8 +206,9 @@
     stm_read_barrier_cache = (char *)(fxcache->cache + fxcache->shift);
 }
 
-#define FXCACHE_AT(obj)  \
-    (*(gcptr *)(stm_read_barrier_cache + ((revision_t)(obj) & FX_MASK)))
+// moved to stmgc.h:
+//#define FXCACHE_AT(obj)
+//    (*(gcptr *)(stm_read_barrier_cache + ((revision_t)(obj) & FX_MASK)))
 
 static inline void fxcache_add(struct FXCache *fxcache, gcptr newobj)
 {
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -39,17 +39,20 @@
 _Bool stm_pointer_equal(gcptr, gcptr);
 
 /* to push/pop objects into the local shadowstack */
-/* (could be turned into macros or something later) */
+#if 0     // (optimized version below)
 void stm_push_root(gcptr);
 gcptr stm_pop_root(void);
+#endif
 
 /* initialize/deinitialize the stm framework in the current thread */
 void stm_initialize(void);
 void stm_finalize(void);
 
 /* read/write barriers (the most general versions only for now) */
+#if 0     // (optimized version below)
 gcptr stm_read_barrier(gcptr);
 gcptr stm_write_barrier(gcptr);
+#endif
 
 /* start a new transaction, calls callback(), and when it returns
    finish that transaction.  callback() is called with the 'arg'
@@ -81,4 +84,44 @@
    It is set to NULL by stm_initialize(). */
 extern __thread gcptr stm_thread_local_obj;
 
+
+
+/* macro-like functionality */
+
+extern __thread gcptr *stm_shadowstack;
+
+static inline void stm_push_root(gcptr obj) {
+    *stm_shadowstack++ = obj;
+}
+static inline gcptr stm_pop_root(void) {
+    return *--stm_shadowstack;
+}
+
+extern __thread revision_t stm_private_rev_num;
+gcptr stm_DirectReadBarrier(gcptr);
+gcptr stm_WriteBarrier(gcptr);
+static const revision_t GCFLAG_WRITE_BARRIER = STM_FIRST_GCFLAG << 5;
+extern __thread char *stm_read_barrier_cache;
+#define FX_MASK 65535
+#define FXCACHE_AT(obj)  \
+    (*(gcptr *)(stm_read_barrier_cache + ((revision_t)(obj) & FX_MASK)))
+
+#define UNLIKELY(test)  __builtin_expect(test, 0)
+static inline gcptr stm_read_barrier(gcptr obj) {
+    /* XXX optimize to get the smallest code */
+    if (UNLIKELY((obj->h_revision != stm_private_rev_num) &&
+                 (FXCACHE_AT(obj) != obj)))
+        obj = stm_DirectReadBarrier(obj);
+    return obj;
+}
+
+static inline gcptr stm_write_barrier(gcptr obj) {
+    if (UNLIKELY((obj->h_revision != stm_private_rev_num) |
+                 ((obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)))
+        obj = stm_WriteBarrier(obj);
+    return obj;
+}
+#undef UNLIKELY
+
+
 #endif
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -4,7 +4,7 @@
 #define LENGTH_SHADOW_STACK   163840
 
 
-static __thread gcptr *stm_shadowstack;
+__thread gcptr *stm_shadowstack;
 static unsigned long stm_regular_length_limit = 10000;
 
 void stm_set_transaction_length(long length_max)
@@ -16,16 +16,6 @@
     stm_regular_length_limit = length_max;
 }
 
-void stm_push_root(gcptr obj)
-{
-    *stm_shadowstack++ = obj;
-}
-
-gcptr stm_pop_root(void)
-{
-    return *--stm_shadowstack;
-}
-
 static void init_shadowstack(void)
 {
     struct tx_descriptor *d = thread_descriptor;
@@ -75,27 +65,6 @@
     DescriptorDone();
 }
 
-gcptr stm_read_barrier(gcptr obj)
-{
-    //if (FXCACHE_AT(obj) == obj)
-    //    dprintf(("read_barrier: in cache: %p\n", obj));
-
-    /* XXX inline in the caller, optimize to get the smallest code */
-    if (UNLIKELY((obj->h_revision != stm_private_rev_num) &&
-                 (FXCACHE_AT(obj) != obj)))
-        obj = stm_DirectReadBarrier(obj);
-    return obj;
-}
-
-gcptr stm_write_barrier(gcptr obj)
-{
-    /* XXX inline in the caller */
-    if (UNLIKELY((obj->h_revision != stm_private_rev_num) |
-                 ((obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)))
-        obj = stm_WriteBarrier(obj);
-    return obj;
-}
-
 /************************************************************/
 
 static revision_t sync_required = 0;

From noreply at buildbot.pypy.org  Thu Jun 27 20:40:21 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 27 Jun 2013 20:40:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, implement more of np.array(),
 allow changing dtype, correctly create views when copy=False
Message-ID: <20130627184021.B8B081C01E5@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65042:8597c6c08cd3
Date: 2013-06-27 21:39 +0300
http://bitbucket.org/pypy/pypy/changeset/8597c6c08cd3/

Log:	test, implement more of np.array(), allow changing dtype, correctly
	create views when copy=False

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1028,15 +1028,19 @@
                                   order)
 
     dtype = interp_dtype.decode_w_dtype(space, w_dtype)
-    if isinstance(w_object, W_NDimArray):
-        if (not space.is_none(w_dtype) and
-            w_object.get_dtype() is not dtype):
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                                  "copying over different dtypes unsupported"))
+    if isinstance(w_object, W_NDimArray) and \
+        (space.is_none(w_dtype) or w_object.get_dtype() is dtype):
+        shape = w_object.get_shape()
         if copy:
-            return w_object.descr_copy(space)
-        return w_object
-
+            w_ret = w_object.descr_copy(space)
+        else:
+            new_impl = w_object.implementation.set_shape(space, w_object, shape)
+            w_ret = W_NDimArray(new_impl)
+        if ndmin > len(shape):
+            shape = [1] * (ndmin - len(shape)) + shape
+            w_ret.implementation = w_ret.implementation.set_shape(space,
+                                            w_ret, shape)
+        return w_ret
     shape, elems_w = find_shape_and_elems(space, w_object, dtype)
     if dtype is None or (
                  dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -276,6 +276,24 @@
         arr = array([1], ndmin=3)
         assert arr.shape == (1, 1, 1)
 
+    def test_array_copy(self):
+        from numpypy import array
+        a = array(range(12)).reshape(3,4)
+        b = array(a, ndmin=4)
+        assert b.shape == (1, 1, 3, 4)
+        b = array(a, copy=False)
+        b[0, 0] = 100
+        assert a[0, 0] == 100
+        b = array(a, copy=True, ndmin=2)
+        b[0, 0] = 0
+        assert a[0, 0] == 100
+        b = array(a, dtype=float)
+        assert (b[0] == [100, 1, 2, 3]).all()
+        assert b.dtype.kind == 'f'
+        b = array(a, copy=False, ndmin=4)
+        b[0,0,0,0] = 0
+        assert a[0, 0] == 0
+
     def test_type(self):
         from numpypy import array
         ar = array(range(5))

From noreply at buildbot.pypy.org  Thu Jun 27 22:14:56 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 27 Jun 2013 22:14:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for noop
Message-ID: <20130627201456.9190F1C1067@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65043:8bceb82940f0
Date: 2013-06-27 23:14 +0300
http://bitbucket.org/pypy/pypy/changeset/8bceb82940f0/

Log:	fix for noop

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1034,6 +1034,8 @@
         if copy:
             w_ret = w_object.descr_copy(space)
         else:
+            if ndmin<= len(shape):
+                return w_object
             new_impl = w_object.implementation.set_shape(space, w_object, shape)
             w_ret = W_NDimArray(new_impl)
         if ndmin > len(shape):

From noreply at buildbot.pypy.org  Thu Jun 27 22:43:09 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Jun 2013 22:43:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: the JIT now sees less of parsing,
 because it does not look into rbigint. This
Message-ID: <20130627204309.41B2F1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r65044:345c2ca63bc3
Date: 2013-06-27 22:39 +0200
http://bitbucket.org/pypy/pypy/changeset/345c2ca63bc3/

Log:	the JIT now sees less of parsing, because it does not look into
	rbigint. This does not seem like a loss though.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -80,39 +80,11 @@
             i23 = strgetitem(p10, i19)
             p25 = newstr(1)
             strsetitem(p25, 0, i23)
-            p28 = call(ConstClass(strip_spaces), p25, descr=<Callr . r EF=4>)
+            p93 = call(ConstClass(fromstr), p25, 16, ConstPtr(ptr70), descr=<Callr . rir EF=3>)
             guard_no_exception(descr=...)
-            i29 = strlen(p28)
-            i30 = int_is_true(i29)
-            guard_true(i30, descr=...)
-            i32 = int_sub(i29, 1)
-            i33 = strgetitem(p28, i32)
-            i35 = int_eq(i33, 108)
-            guard_false(i35, descr=...)
-            i37 = int_eq(i33, 76)
-            guard_false(i37, descr=...)
-            i39 = strgetitem(p28, 0)
-            i41 = int_eq(i39, 45)
-            guard_false(i41, descr=...)
-            i43 = int_eq(i39, 43)
-            guard_false(i43, descr=...)
-            i43 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr42), descr=<Calli 1 rr EF=0>)
-            guard_false(i43, descr=...)
-            i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<Calli 1 rr EF=0>)
-            guard_false(i46, descr=...)
-            p51 = new_with_vtable(...)
-            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            p55 = call(ConstClass(parse_digit_string), p51, descr=<Callr . r EF=4>)
+            i94 = call(ConstClass(rbigint.toint), p93, descr=<Calli . r EF=3>)
             guard_no_exception(descr=...)
-            i57 = call(ConstClass(rbigint.toint), p55, descr=<Calli . r EF=3>)
-            guard_no_exception(descr=...)
-            i58 = int_add_ovf(i6, i57)
+            i95 = int_add_ovf(i6, i94)
             guard_no_overflow(descr=...)
             --TICK--
             jump(..., descr=...)

From noreply at buildbot.pypy.org  Fri Jun 28 07:14:03 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 07:14:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix zjit compilation
Message-ID: <20130628051403.9FF351C01E5@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65045:0c553eed18ac
Date: 2013-06-28 08:13 +0300
http://bitbucket.org/pypy/pypy/changeset/0c553eed18ac/

Log:	fix zjit compilation

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -51,6 +51,7 @@
     w_IndexError = W_TypeObject("IndexError")
     w_OverflowError = W_TypeObject("OverflowError")
     w_NotImplementedError = W_TypeObject("NotImplementedError")
+    w_AttributeError = W_TypeObject("AttributeError")
     w_None = None
 
     w_bool = W_TypeObject("bool")

From noreply at buildbot.pypy.org  Fri Jun 28 09:27:39 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Jun 2013 09:27:39 +0200 (CEST)
Subject: [pypy-commit] pypy identity-set: (andrewchambers) identity set
	strategy
Message-ID: <20130628072739.D9E0F1C15A4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: identity-set
Changeset: r65046:bab634a518bf
Date: 2013-06-28 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/bab634a518bf/

Log:	(andrewchambers) identity set strategy

diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -26,7 +26,7 @@
     def __repr__(self):
         """representation for debugging purposes"""
         reprlist = [repr(w_item) for w_item in self.getkeys()]
-        return "<%s(%s)>" % (self.__class__.__name__, ', '.join(reprlist))
+        return "<%s(%s)(%s)>" % (self.__class__.__name__, self.strategy, ', '.join(reprlist))
 
     def from_storage_and_strategy(self, storage, strategy):
         obj = self._newobj(self.space, None)
@@ -780,6 +780,8 @@
             strategy = self.space.fromcache(StringSetStrategy)
         elif type(w_key) is W_UnicodeObject:
             strategy = self.space.fromcache(UnicodeSetStrategy)
+        elif self.space.type(w_key).compares_by_identity():
+            strategy = self.space.fromcache(IdentitySetStrategy)
         else:
             strategy = self.space.fromcache(ObjectSetStrategy)
         w_set.strategy = strategy
@@ -1336,6 +1338,41 @@
                 break
             d_obj[w_item] = None
 
+class IdentitySetStrategy(AbstractUnwrappedSetStrategy, SetStrategy):
+    erase, unerase = rerased.new_erasing_pair("identityset")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+    
+    def get_empty_storage(self):
+        return self.erase({})
+
+    def get_empty_dict(self):
+        return {}
+
+    def is_correct_type(self, w_key):
+        w_type = self.space.type(w_key)
+        return w_type.compares_by_identity()
+
+    def may_contain_equal_elements(self, strategy):
+        #empty first, probably more likely
+        if strategy is self.space.fromcache(EmptySetStrategy):
+            return False
+        if strategy is self.space.fromcache(IntegerSetStrategy):
+            return False
+        if strategy is self.space.fromcache(StringSetStrategy):
+            return False
+        if strategy is self.space.fromcache(UnicodeSetStrategy):
+            return False
+        return True
+
+    def unwrap(self, w_item):
+        return w_item
+
+    def wrap(self, item):
+        return item    
+    
+    def iter(self, w_set):
+        return IdentityIteratorImplementation(self.space, self, w_set)        
 
 class IteratorImplementation(object):
     def __init__(self, space, strategy, implementation):
@@ -1427,6 +1464,17 @@
         else:
             return None
 
+class IdentityIteratorImplementation(IteratorImplementation):
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
+        self.iterator = d.iterkeys()
+
+    def next_entry(self):
+        for key in self.iterator:
+            return self.space.wrap(key)
+        else:
+            return None
 
 class RDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, w_set):
@@ -1545,6 +1593,15 @@
         w_set.strategy = space.fromcache(UnicodeSetStrategy)
         w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
         return
+    
+    # check for compares by identity
+    for w_item in iterable_w:
+        if not space.type(w_item).compares_by_identity():
+            break
+    else:
+        w_set.strategy = space.fromcache(IdentitySetStrategy)
+        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        return    
 
     w_set.strategy = space.fromcache(ObjectSetStrategy)
     w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
diff --git a/pypy/objspace/std/test/test_identityset.py b/pypy/objspace/std/test/test_identityset.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_identityset.py
@@ -0,0 +1,98 @@
+import py
+
+
+
+class AppTestIdentitySet(object):
+    
+    #needed for compares_by_identity
+    spaceconfig = {"objspace.std.withidentitydict": True}
+    
+    def setup_class(cls):
+        from pypy.objspace.std import identitydict
+        if cls.runappdirect:
+            py.test.skip("interp2app doesn't work on appdirect")
+    
+    def w_uses_strategy(self, s , obj):
+        import __pypy__
+        return s in __pypy__.internal_repr(obj)
+    
+    def test_use_identity_strategy(self):
+        
+        class Plain(object):
+            pass
+
+        class CustomEq(object):
+            def __eq__(self, other):
+                return True
+
+        class CustomCmp (object):
+            def __cmp__(self, other):
+                return 0
+
+        class CustomHash(object):
+            def __hash__(self):
+                return 0
+            
+        s = set()
+        
+        assert not self.uses_strategy('IdentitySetStrategy',s)
+        
+        s.add(Plain())
+        
+        assert self.uses_strategy('IdentitySetStrategy',s)
+        
+        for cls in [CustomEq,CustomCmp,CustomHash]:
+            s = set()
+            s.add(cls())
+            assert not self.uses_strategy('IdentitySetStrategy',s)
+        
+        
+    def test_use_identity_strategy_list(self):
+        
+        class X(object):
+            pass
+        
+        assert self.uses_strategy('IdentitySetStrategy',set([X(),X()]))
+        assert not self.uses_strategy('IdentitySetStrategy',set([X(),""]))
+        assert not self.uses_strategy('IdentitySetStrategy',set([X(),u""]))
+        assert not self.uses_strategy('IdentitySetStrategy',set([X(),1]))
+        
+    def test_identity_strategy_add(self):
+        
+        class X(object):
+            pass
+        
+        class NotIdent(object):
+            def __eq__(self,other):
+                pass
+        
+        s = set([X(),X()])
+        s.add('foo')
+        assert not self.uses_strategy('IdentitySetStrategy',s)
+        s = set([X(),X()])
+        s.add(NotIdent())
+        assert not self.uses_strategy('IdentitySetStrategy',s)
+    
+    def test_identity_strategy_sanity(self):
+        
+        class X(object):
+            pass
+        
+        class Y(object):
+            pass
+        
+        a,b,c,d,e,f = X(),Y(),X(),Y(),X(),Y()
+        
+        s = set([a,b]).union(set([c]))
+        assert self.uses_strategy('IdentitySetStrategy',s) 
+        assert set([a,b,c]) == s
+        s = set([a,b,c,d,e,f]) - set([d,e,f])
+        assert self.uses_strategy('IdentitySetStrategy',s)
+        assert set([a,b,c]) == s
+        
+        
+        s = set([a])
+        s.update([b,c])
+        
+        assert s == set([a,b,c])
+

From noreply at buildbot.pypy.org  Fri Jun 28 09:58:43 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 09:58:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, fix fill for complex types
Message-ID: <20130628075843.55D211C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65047:89ca223757d9
Date: 2013-06-28 10:58 +0300
http://bitbucket.org/pypy/pypy/changeset/89ca223757d9/

Log:	test, fix fill for complex types

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -343,13 +343,16 @@
         assert a[1] == 1.0
 
     def test_ones(self):
-        from numpypy import ones
+        from numpypy import ones, dtype
         a = ones(3)
         assert len(a) == 3
         assert a[0] == 1
         raises(IndexError, "a[3]")
         a[2] = 4
         assert a[2] == 4
+        b = ones(3, complex)
+        assert b[0] == 1+0j
+        assert b.dtype is dtype(complex)
 
     def test_copy(self):
         from numpypy import arange, array
@@ -2194,6 +2197,10 @@
         d.fill(100)
         assert d == 100
 
+        e = array(10, dtype=complex)
+        e.fill(1.5-3j)
+        assert e == 1.5-3j
+
     def test_array_indexing_bool(self):
         from numpypy import arange
         a = arange(10)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1100,6 +1100,13 @@
         op = '+' if imag >= 0 else ''
         return ''.join(['(', real_str, op, imag_str, ')'])
 
+    def fill(self, storage, width, box, start, stop, offset):
+        real, imag = self.unbox(box)
+        for i in xrange(start, stop, width):
+            raw_storage_setitem(storage, i+offset, real)
+            raw_storage_setitem(storage,
+                    i+offset+rffi.sizeof(self.T), imag)
+
     @staticmethod
     def for_computation(v):
         return float(v[0]), float(v[1])

From noreply at buildbot.pypy.org  Fri Jun 28 10:46:41 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 10:46:41 +0200 (CEST)
Subject: [pypy-commit] buildbot numpy-tests: start to write a target for
 testing numpy as a site-package, using git and nose
Message-ID: <20130628084641.831141C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: numpy-tests
Changeset: r828:f18b4fa91292
Date: 2013-06-28 11:46 +0300
http://bitbucket.org/pypy/buildbot/changeset/f18b4fa91292/

Log:	start to write a target for testing numpy as a site-package, using
	git and nose

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -202,6 +202,30 @@
         ShellCmd.start(self)
 
 
+class UpdateGitCheckout(ShellCmd):
+    description = 'git checkout'
+    command = 'UNKNOWN'
+
+    def __init__(self, workdir=None, haltOnFailure=True, force_branch=None,
+                 **kwargs):
+        ShellCmd.__init__(self, workdir=workdir, haltOnFailure=haltOnFailure,
+                          **kwargs)
+        self.force_branch = force_branch
+        self.addFactoryArguments(force_branch=force_branch)
+
+    def start(self):
+        if self.force_branch is not None:
+            branch = self.force_branch
+            # Note: We could add a warning to the output if we
+            # ignore the branch set by the user.
+        else:
+            properties = self.build.getProperties()
+            branch = properties['branch'] or 'default'
+        command = ["git", "checkout", "-f", branch]
+        self.setCommand(command)
+        ShellCmd.start(self)
+
+
 class CheckGotRevision(ShellCmd):
     description = 'got_revision'
     command = ['hg', 'parents', '--template', 'got_revision:{rev}:{node}']
@@ -265,6 +289,47 @@
                 command=WithProperties("hg update --clean %(revision)s"),
                 workdir=workdir))
 
+def update_git(platform, factory, repourl, workdir, use_branch,
+              force_branch=None):
+    if platform == 'win32':
+        command = "if not exist .git rmdir /q /s ."
+    else:
+        command = "if [ ! -d .git ]; then rm -fr * .[a-z]*; fi"
+    factory.addStep(ShellCmd(description="rmdir?",
+                             command=command,
+                             workdir=workdir,
+                             haltOnFailure=False))
+    #
+    if platform == "win32":
+        command = "if not exist .git %s"
+    else:
+        command = "if [ ! -d .git ]; then %s; fi"
+    command = command % ("git clone " + repourl + " .")
+    factory.addStep(ShellCmd(description="git clone",
+                             command=command,
+                             workdir=workdir,
+                             timeout=3600,
+                             haltOnFailure=True))
+    #
+    factory.addStep(
+        ShellCmd(description="git clean",
+                 command="git clean",
+                 workdir=workdir,
+                 haltOnFailure=True))
+    #
+    factory.addStep(ShellCmd(description="git pull",
+                             command="git pull",
+                             workdir=workdir))
+    #
+    if use_branch or force_branch:
+        factory.addStep(UpdateGitCheckout(workdir=workdir,
+                                       haltOnFailure=True,
+                                       force_branch=force_branch))
+    else:
+        factory.addStep(ShellCmd(description="git checkout",
+                command=WithProperties("git checkout -f %(revision)s"),
+                workdir=workdir))
+
 
 def setup_steps(platform, factory, workdir=None,
                 repourl='https://bitbucket.org/pypy/pypy/',
@@ -670,3 +735,63 @@
         self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json",
                                          masterdest=WithProperties(resultfile),
                                          workdir="."))
+
+class NativeNumpyTests(factory.BuildFactory):
+    '''
+    Download a pypy nightly, install nose and numpy, and run the numpy test suite
+    '''
+    def __init__(self, platform='linux',
+                 app_tests=False,
+                 lib_python=False,
+                 pypyjit=False,
+                 prefix=None,
+                 translationArgs=[]
+                 ):
+        factory.BuildFactory.__init__(self)
+
+        # XXX extend to checkout the specific revision of the build
+        setup_steps(platform, self)
+
+        # download corresponding nightly build
+        self.addStep(ShellCmd(
+            description="Clear pypy-c",
+            command=['rm', '-rf', 'pypy-c'],
+            workdir='.'))
+        extension = get_extension(platform)
+        name = build_name(platform, pypyjit, translationArgs, placeholder='%(revision)s') + extension
+        self.addStep(PyPyDownload(
+            basename=name,
+            mastersrc='~/nightly',
+            slavedest='pypy_build' + extension,
+            workdir='pypy-c'))
+
+        # extract downloaded file
+        if platform.startswith('win'):
+            raise NotImplementedError
+        else:
+            self.addStep(ShellCmd(
+                description="decompress pypy-c",
+                command=['tar', '--extract', '--file=pypy_build'+ extension, '--strip-components=1', '--directory=.'],
+                workdir='pypy-c'))
+
+        # copy pypy-c to the expected location within the pypy source checkout
+        self.addStep(ShellCmd(
+            description="move pypy-c",
+            command=['cp', '-v', 'pypy-c/bin/pypy', 'build/pypy/goal/pypy-c'],
+            workdir='.'))
+        # copy generated and copied header files to build/include
+        self.addStep(ShellCmd(
+            description="move header files",
+            command=['cp', '-vr', 'pypy-c/include', 'build'],
+            workdir='.'))
+        # copy ctypes_resource_cache generated during translation
+        self.addStep(ShellCmd(
+            description="move ctypes resource cache",
+            command=['cp', '-rv', 'pypy-c/lib_pypy/ctypes_config_cache', 'build/lib_pypy'],
+            workdir='.'))
+
+        # obtain a pypy-compatible branch of numpy
+        numpy_url = 'https://github.com/mattip/numpy'
+        numpy_pypy_branch = 'pypy'
+        update_git(platform, factory, numpy_url, 'numpy_src', use_branch=True,
+              force_branch=numpy_pypy_branch)

From noreply at buildbot.pypy.org  Fri Jun 28 11:21:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 11:21:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Test and fix
Message-ID: <20130628092125.CAFCA1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r314:c43233de7f0e
Date: 2013-06-28 11:17 +0200
http://bitbucket.org/pypy/stmgc/changeset/c43233de7f0e/

Log:	Test and fix

diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -646,7 +646,8 @@
     }
 
     /* Are we at the end of the nursery? */
-    if (d->nursery_nextlimit == d->nursery_end) {
+    if (d->nursery_nextlimit == d->nursery_end ||
+        d->nursery_current == d->nursery_end) {   // stmgc_minor_collect_soon()
         /* Yes */
         if (tid == -1)
             return NULL;    /* cannot collect */
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -73,6 +73,7 @@
     void stm_add_prebuilt_root(gcptr);
     void stm_clear_between_tests(void);
     void stmgc_minor_collect(void);
+    void stmgc_minor_collect_soon(void);
     gcptr _stm_nonrecord_barrier(gcptr);
     int _stm_is_private(gcptr);
     void stm_start_sharedlock(void);
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -301,3 +301,7 @@
     for words in range(80):
         p1 = nalloc(HDR + words * WORD)
     # assert did not crash
+
+def test_collect_soon():
+    lib.stmgc_minor_collect_soon()
+    nalloc(HDR)

From noreply at buildbot.pypy.org  Fri Jun 28 11:21:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 11:21:26 +0200 (CEST)
Subject: [pypy-commit] stmgc default: The original test showing the problem
 fixed in c43233de7f0e.
Message-ID: <20130628092126.DEAEA1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r315:860cf40b0858
Date: 2013-06-28 11:21 +0200
http://bitbucket.org/pypy/stmgc/changeset/860cf40b0858/

Log:	The original test showing the problem fixed in c43233de7f0e.

diff --git a/duhton/test/test_gc.py b/duhton/test/test_gc.py
--- a/duhton/test/test_gc.py
+++ b/duhton/test/test_gc.py
@@ -23,3 +23,14 @@
 
 def test_verylong1():
     test_long1(limit=16)
+
+
+def test_long_lists():
+    run("""
+        (defun g (lst n)
+          (while (> n 30000)
+            (append lst n)
+            (setq n (- n 1))))
+        (g (list) 34000)
+    """)
+    assert 1  # did not crash

From noreply at buildbot.pypy.org  Fri Jun 28 11:23:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 11:23:54 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Passing tests.
Message-ID: <20130628092354.61B7F1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r316:45380d4cb89c
Date: 2013-06-28 11:23 +0200
http://bitbucket.org/pypy/stmgc/changeset/45380d4cb89c/

Log:	Passing tests.

diff --git a/duhton/test/test_transaction.py b/duhton/test/test_transaction.py
--- a/duhton/test/test_transaction.py
+++ b/duhton/test/test_transaction.py
@@ -131,3 +131,28 @@
         (transaction f)
         """)
     assert res == "[ 20 30 ]\n"
+
+def test_long_lists_no_conflict():
+    res = run("""
+        (defun g (lst n)
+          (while (> n 30000)
+            (append lst n)
+            (setq n (- n 1)))
+          (print (len lst)))
+        (transaction g (list) 34000)
+        (transaction g (list) 34000)
+    """)
+    assert res == "4000\n4000\n"
+
+def test_long_lists_with_conflict():
+    res = run("""
+        (defun g (lst n)
+          (while (> n 30000)
+            (append lst n)
+            (setq n (- n 1)))
+          (print (len lst)))
+        (setq L (list))
+        (transaction g L 34000)
+        (transaction g L 34000)
+    """)
+    assert res == "4000\n8000\n"

From noreply at buildbot.pypy.org  Fri Jun 28 12:08:05 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 12:08:05 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: the only number which is allowed to
 start with 0 is 0 itself
Message-ID: <20130628100805.325BA1C08AF@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65048:6e71ab1760e1
Date: 2013-06-26 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/6e71ab1760e1/

Log:	the only number which is allowed to start with 0 is 0 itself

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -154,6 +154,9 @@
             i += 1
         elif self.ll_chars[i] == '+':
             i += 1
+        elif self.ll_chars[i] == '0':
+            i += 1
+            return i, 0
         i, intval, _ = self.parse_digits(i)
         return i, sign * intval
 
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -113,6 +113,10 @@
         check('42E-1', 4.2)
         check('42E+1', 420.0)
         check('42.123E3', 42123.0)
+        check('0', 0)
+        check('-0', 0)
+        check('0.123', 0.123)
+        check('0E3', 0.0)
 
     def test_decode_numeric_invalid(self):
         import _fastjson
@@ -125,6 +129,7 @@
         error('12.-3')
         error('12E')
         error('12E-')
+        error('0123') # numbers can't start with 0
 
     def test_decode_object(self):
         import _fastjson

From noreply at buildbot.pypy.org  Fri Jun 28 12:08:06 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 12:08:06 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: leading 0s are allowed in the exponent
	part
Message-ID: <20130628100806.6FF421C08AF@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65049:05ec68a8c965
Date: 2013-06-26 15:18 +0200
http://bitbucket.org/pypy/pypy/changeset/05ec68a8c965/

Log:	leading 0s are allowed in the exponent part

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -117,7 +117,7 @@
         self._raise("Error when decoding false at char %d", i)
 
     def decode_numeric(self, i):
-        i, intval = self.parse_integer(i)
+        i, intval = self.parse_integer(i, allow_leading_0=False)
         #
         is_float = False
         exp = 0
@@ -134,7 +134,7 @@
         # check for the optional exponent part
         if ch == 'E' or ch == 'e':
             is_float = True
-            i, exp = self.parse_integer(i+1)
+            i, exp = self.parse_integer(i+1, allow_leading_0=True)
         #
         self.pos = i
         if is_float:
@@ -146,7 +146,7 @@
         else:
             return self.space.wrap(intval)
 
-    def parse_integer(self, i):
+    def parse_integer(self, i, allow_leading_0=False):
         "Parse a decimal number with an optional minus sign"
         sign = 1
         if self.ll_chars[i] == '-':
@@ -154,7 +154,7 @@
             i += 1
         elif self.ll_chars[i] == '+':
             i += 1
-        elif self.ll_chars[i] == '0':
+        elif not allow_leading_0 and self.ll_chars[i] == '0':
             i += 1
             return i, 0
         i, intval, _ = self.parse_digits(i)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -117,6 +117,7 @@
         check('-0', 0)
         check('0.123', 0.123)
         check('0E3', 0.0)
+        check('5E0001', 50.0)
 
     def test_decode_numeric_invalid(self):
         import _fastjson
@@ -125,6 +126,7 @@
         #
         error('  42   abc')
         error('.123')
+        error('+123')
         error('12.')
         error('12.-3')
         error('12E')

From noreply at buildbot.pypy.org  Fri Jun 28 12:08:07 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 12:08:07 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: this gives a ~30% speedup: it's the
 same speedup which was observed in 24a2398ade5b but got lost when the
 function became too big and stopped to become automatically inlined
Message-ID: <20130628100807.8C1BB1C08AF@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65050:da468fc481b3
Date: 2013-06-26 18:49 +0200
http://bitbucket.org/pypy/pypy/changeset/da468fc481b3/

Log:	this gives a ~30% speedup: it's the same speedup which was observed
	in 24a2398ade5b but got lost when the function became too big and
	stopped to become automatically inlined

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -159,6 +159,7 @@
             return i, 0
         i, intval, _ = self.parse_digits(i)
         return i, sign * intval
+    parse_integer._always_inline_ = True
 
     def parse_digits(self, i):
         "Parse a sequence of digits as a decimal number. No sign allowed"
@@ -175,7 +176,7 @@
         if count == 0:
             self._raise("Expected digit at char %d", i)
         return i, intval, count
-        
+
     def decode_array(self, i):
         w_list = self.space.newlist([])
         start = i

From noreply at buildbot.pypy.org  Fri Jun 28 12:08:09 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 12:08:09 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add support for big numbers which are
	turned into longs
Message-ID: <20130628100809.171931C08AF@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65051:01e0cc2cf8c8
Date: 2013-06-28 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/01e0cc2cf8c8/

Log:	add support for big numbers which are turned into longs

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -1,3 +1,4 @@
+import sys
 import math
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
@@ -6,6 +7,8 @@
 from pypy.interpreter import unicodehelper
 from rpython.rtyper.annlowlevel import llstr, hlunicode
 
+OVF_DIGITS = len(str(sys.maxint))
+
 def is_whitespace(ch):
     return ch == ' ' or ch == '\t' or ch == '\r' or ch == '\n'
 
@@ -53,8 +56,8 @@
         self.last_type = TYPE_UNKNOWN
 
     def getslice(self, start, end):
-        assert start > 0
-        assert end > 0
+        assert start >= 0
+        assert end >= 0
         return self.s[start:end]
 
     def skip_whitespace(self, i):
@@ -117,7 +120,20 @@
         self._raise("Error when decoding false at char %d", i)
 
     def decode_numeric(self, i):
-        i, intval = self.parse_integer(i, allow_leading_0=False)
+        w_res = self.decode_numeric_fast(i)
+        if w_res is self.space.w_None:
+            # possible overflow, reparse it
+            return self.decode_numeric_slow(i)
+        return w_res
+
+    def decode_numeric_fast(self, i):
+        i, ovf_maybe, intval = self.parse_integer(i, allow_leading_0=False)
+        if ovf_maybe:
+            # apparently we get a ~30% slowdown on my microbenchmark if we
+            # return None instead of w_None, probably because the annotation
+            # of the results geta can_be_None=True. We need to check if this
+            # is still true also for the full pypy
+            return self.space.w_None
         #
         is_float = False
         exp = 0
@@ -134,7 +150,7 @@
         # check for the optional exponent part
         if ch == 'E' or ch == 'e':
             is_float = True
-            i, exp = self.parse_integer(i+1, allow_leading_0=True)
+            i, ovf_maybe, exp = self.parse_integer(i+1, allow_leading_0=True)
         #
         self.pos = i
         if is_float:
@@ -146,8 +162,21 @@
         else:
             return self.space.wrap(intval)
 
+    def decode_numeric_slow(self, i):
+        start = i
+        if self.ll_chars[i] == '-':
+            i += 1
+        while self.ll_chars[i].isdigit():
+            i += 1
+        s = self.getslice(start, i)
+        self.pos = i
+        w_res = self.space.call_function(self.space.w_int, self.space.wrap(s))
+        #assert w_res is not None # XXX check if this brings any speedup in pypy-c
+        return w_res
+
     def parse_integer(self, i, allow_leading_0=False):
         "Parse a decimal number with an optional minus sign"
+        start = i
         sign = 1
         if self.ll_chars[i] == '-':
             sign = -1
@@ -156,9 +185,12 @@
             i += 1
         elif not allow_leading_0 and self.ll_chars[i] == '0':
             i += 1
-            return i, 0
+            return i, False, 0
         i, intval, _ = self.parse_digits(i)
-        return i, sign * intval
+        # if the number has more digits than OVF_DIGITS, it might have
+        # overflowed
+        ovf_maybe = (i-start >= OVF_DIGITS)
+        return i, ovf_maybe, sign * intval
     parse_integer._always_inline_ = True
 
     def parse_digits(self, i):
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -118,6 +118,8 @@
         check('0.123', 0.123)
         check('0E3', 0.0)
         check('5E0001', 50.0)
+        check(str(1 << 32), 1 << 32)
+        check(str(1 << 64), 1 << 64)
 
     def test_decode_numeric_invalid(self):
         import _fastjson

From noreply at buildbot.pypy.org  Fri Jun 28 12:08:10 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 12:08:10 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: do the correct thing when we have a
 float number with a large number of digits
Message-ID: <20130628100810.452B61C08AF@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65052:09970bd41cfe
Date: 2013-06-28 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/09970bd41cfe/

Log:	do the correct thing when we have a float number with a large number
	of digits

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -163,14 +163,26 @@
             return self.space.wrap(intval)
 
     def decode_numeric_slow(self, i):
+        is_float = False
         start = i
         if self.ll_chars[i] == '-':
             i += 1
+        # skip the integral part
         while self.ll_chars[i].isdigit():
             i += 1
+        if self.ll_chars[i] == '.':
+            is_float = True
+            i += 1
+            # skip the fractional part
+            while self.ll_chars[i].isdigit():
+                i += 1
         s = self.getslice(start, i)
         self.pos = i
-        w_res = self.space.call_function(self.space.w_int, self.space.wrap(s))
+        if is_float:
+            w_func = self.space.w_float
+        else:
+            w_func = self.space.w_int
+        w_res = self.space.call_function(w_func, self.space.wrap(s))
         #assert w_res is not None # XXX check if this brings any speedup in pypy-c
         return w_res
 
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -99,6 +99,7 @@
 
 
     def test_decode_numeric(self):
+        import sys
         import _fastjson
         def check(s, val):
             res = _fastjson.loads(s)
@@ -120,6 +121,8 @@
         check('5E0001', 50.0)
         check(str(1 << 32), 1 << 32)
         check(str(1 << 64), 1 << 64)
+        x = str(sys.maxint+1) + '.123'
+        check(x, float(x))
 
     def test_decode_numeric_invalid(self):
         import _fastjson

From noreply at buildbot.pypy.org  Fri Jun 28 12:08:11 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 12:08:11 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: handle also the exponent
Message-ID: <20130628100811.6C7251C08AF@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65053:bd082881bde9
Date: 2013-06-28 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/bd082881bde9/

Log:	handle also the exponent

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -170,14 +170,23 @@
         # skip the integral part
         while self.ll_chars[i].isdigit():
             i += 1
+        #
+        # skip the fractional part, if any
         if self.ll_chars[i] == '.':
             is_float = True
             i += 1
-            # skip the fractional part
             while self.ll_chars[i].isdigit():
                 i += 1
+        #
+        # skip the exponent part, if any
+        if self.ll_chars[i] == 'e' or self.ll_chars[i] == 'E':
+            is_float = True
+            i += 1
+            while self.ll_chars[i].isdigit():
+                i += 1
+        #
+        self.pos = i
         s = self.getslice(start, i)
-        self.pos = i
         if is_float:
             w_func = self.space.w_float
         else:
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -121,8 +121,11 @@
         check('5E0001', 50.0)
         check(str(1 << 32), 1 << 32)
         check(str(1 << 64), 1 << 64)
+        #
         x = str(sys.maxint+1) + '.123'
         check(x, float(x))
+        x = str(sys.maxint+1) + 'E1'
+        check(x, float(x))
 
     def test_decode_numeric_invalid(self):
         import _fastjson

From noreply at buildbot.pypy.org  Fri Jun 28 12:34:43 2013
From: noreply at buildbot.pypy.org (andrewchambers)
Date: Fri, 28 Jun 2013 12:34:43 +0200 (CEST)
Subject: [pypy-commit] pypy identity-set: Added tests for some cases where
 classes are changed at runtime, and added
Message-ID: <20130628103443.A7A6E1C010B@cobra.cs.uni-duesseldorf.de>

Author: Andrew Chambers <andrewchamberss at gmail.com>
Branch: identity-set
Changeset: r65054:858c2dc597dc
Date: 2013-06-28 22:24 +1200
http://bitbucket.org/pypy/pypy/changeset/858c2dc597dc/

Log:	Added tests for some cases where classes are changed at runtime, and
	added a simple iteration test case for the identityset class.

diff --git a/pypy/objspace/std/test/test_identityset.py b/pypy/objspace/std/test/test_identityset.py
--- a/pypy/objspace/std/test/test_identityset.py
+++ b/pypy/objspace/std/test/test_identityset.py
@@ -95,4 +95,115 @@
         s.update([b,c])
         
         assert s == set([a,b,c])
-
+    
+    
+    def test_identity_strategy_iterators(self):
+        
+        class X(object):
+            pass
+        
+        s = set([X() for i in range(10)])
+        counter = 0
+        for item in s:
+            counter += 1
+            assert item in s
+        
+        assert counter == 10
+    
+    
+    def test_identity_strategy_other_cmp(self):
+        
+        #test tries to hit positive and negative in 
+        # may_contain_equal_elements
+        
+        class X(object):
+            pass
+            
+        s = set([X() for i in range(10)]) 
+        
+        assert s.intersection(set([1,2,3])) == set()
+        assert s.intersection(set(['a','b','c'])) == set()
+        assert s.intersection(set(['a','b','c'])) == set()
+        assert s.intersection(set([X(),X()])) == set()
+        
+        other = set(['a','b','c',s.__iter__().next()])
+        intersect = s.intersection(other)
+        assert len(intersect) == 1
+        assert intersect.__iter__().next() in s
+        assert intersect.__iter__().next() in other
+        
+    def test_class_monkey_patch(self):
+        
+        class X(object):
+            pass
+        
+        s = set()
+        
+        s.add(X())
+        assert self.uses_strategy('IdentitySetStrategy',s)
+        X.__eq__ = lambda self,other : None
+        s.add(X())
+        assert not self.uses_strategy('IdentitySetStrategy',s)
+        assert not self.uses_strategy('IdentitySetStrategy',set([X(),X()]))
+        assert not self.uses_strategy('IdentitySetStrategy',set([X(),""]))
+        assert not self.uses_strategy('IdentitySetStrategy',set([X(),u""]))
+        assert not self.uses_strategy('IdentitySetStrategy',set([X(),1]))
+        
+        # An interesting case, add an instance, mutate the class, 
+        # then add the same instance.
+        
+        class X(object):
+            pass
+        
+        s = set()
+        inst = X()
+        s.add(inst)
+        X.__eq__ = lambda x,y : x is y
+        s.add(inst)
+        
+        assert len(s) == 1
+        assert s.__iter__().next() is inst
+        assert not self.uses_strategy('IdentitySetStrategy',s)
+        
+        
+        #Add instance, mutate class, check membership of that instance.
+        
+        class X(object):
+            pass
+        
+        
+        inst = X()
+        s = set()
+        s.add(inst)
+        X.__eq__ = lambda x,y : x is y
+        assert inst in s
+        
+        # Test Wrong strategy
+        # If the strategy is changed by mutation, but the instance
+        # does not change, then this tests the methods that call
+        # may_contain_equal_elements still function.
+        # i.e. same instance in two sets, one with object strategy, one with
+        # identity strategy.
+        
+        class X(object):
+            pass
+        
+        
+        inst = X()
+        s1 = set()
+        s1.add(inst)
+        assert self.uses_strategy('IdentitySetStrategy',s1)
+        X.__eq__ = lambda x,y : x is y
+        s2 = set()
+        s2.add(inst)
+        assert not self.uses_strategy('IdentitySetStrategy',s2)
+        
+        assert s1.intersection(s2) == set([inst])
+        assert (s1 - s2) == set()
+        assert (s2 - s1) == set()
+        
+        s1.difference_update(s2)
+        assert s1 == set()
+        
+        
+        

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:38 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:38 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: test view
Message-ID: <20130628124038.B20311C0351@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65056:1cdbf8183c73
Date: 2013-06-28 12:22 +0300
http://bitbucket.org/pypy/pypy/changeset/1cdbf8183c73/

Log:	test view

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1411,6 +1411,16 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
+    def test_ndarray_view(self):
+        from numpypy import array, int8, int16, dtype
+        x = array([(1, 2)], dtype=[('a', int8), ('b', int8)])
+        y = x.view(dtype=int16)
+        assert y[0] == 513
+        assert y.dtype == dtype('int16')
+        y[0] = 670
+        assert x['a'] == -98
+        assert x['b'] == 2
+
     def test_tolist_scalar(self):
         from numpypy import int32, bool_
         x = int32(23)

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:37 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:37 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: add view to ndarray
Message-ID: <20130628124037.7E2801C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65055:fc0ee8f6246b
Date: 2013-06-28 12:04 +0300
http://bitbucket.org/pypy/pypy/changeset/fc0ee8f6246b/

Log:	add view to ndarray


From noreply at buildbot.pypy.org  Fri Jun 28 14:40:39 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:39 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: test scalars and ndarray,
	start to implement
Message-ID: <20130628124039.E83111C0883@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65057:78f0fb200c25
Date: 2013-06-28 13:05 +0300
http://bitbucket.org/pypy/pypy/changeset/78f0fb200c25/

Log:	test scalars and ndarray, start to implement

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -618,8 +618,17 @@
             "trace not implemented yet"))
 
     def descr_view(self, space, w_dtype=None, w_type=None) :
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "view not implemented yet"))
+        if w_type is not None:
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "view(... type=<class>) not implemented yet"))
+        if w_dtype:
+            dtype = space.interp_w(interp_dtype.W_Dtype,
+                space.call_function(space.gettypefor(interp_dtype.W_Dtype),
+                                                                   w_dtype))
+        else:
+            dtype = self.get_dtype()
+        return W_NDimArray(self.implementation.get_view(self, dtype))
+
 
     # --------------------- operations ----------------------------
 
@@ -996,6 +1005,7 @@
     round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
+    view = interp2app(W_NDimArray.descr_view),
 
     ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented
     __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1412,7 +1412,7 @@
         assert a[2].imag == -5
 
     def test_ndarray_view(self):
-        from numpypy import array, int8, int16, dtype
+        from numpypy import array, int8, int16, dtype, int64
         x = array([(1, 2)], dtype=[('a', int8), ('b', int8)])
         y = x.view(dtype=int16)
         assert y[0] == 513
@@ -1420,6 +1420,14 @@
         y[0] = 670
         assert x['a'] == -98
         assert x['b'] == 2
+        f = array([1000, -1234], dtype='i')
+        nnp = self.non_native_prefix
+        d = f.view(dtype=nnp + 'i')
+        assert (d == [-402456576,  788267007]).all()
+        s = int64(12)
+        exc = raises(TypeError, s.view, dtype='int8')
+        assert exc.value[0] == "view() takes no keyword arguments"
+        assert s.view('double') < 7e-323
 
     def test_tolist_scalar(self):
         from numpypy import int32, bool_

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:41 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:41 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: test view shape changes
Message-ID: <20130628124041.2A6B21C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65058:050f965366b8
Date: 2013-06-28 13:48 +0300
http://bitbucket.org/pypy/pypy/changeset/050f965366b8/

Log:	test view shape changes

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -75,6 +75,10 @@
         else:
             return None
 
+    def get_view(self, dtype, new_shape):
+        pass
+        #return SliceArray(self.start,
+
     def get_real(self, orig_array):
         strides = self.get_strides()
         backstrides = self.get_backstrides()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1428,6 +1428,12 @@
         exc = raises(TypeError, s.view, dtype='int8')
         assert exc.value[0] == "view() takes no keyword arguments"
         assert s.view('double') < 7e-323
+        x = array(range(15), dtype='int16').reshape(3,5)
+        exc = raises(ValueError, x.view, dtype='int32')
+        assert exc.value[0] == "new type not compatible with array."
+        assert x.view('int8').shape == (3, 10)
+        x = array(range(15), dtype='int16').reshape(3,5).T
+        assert x.view('int8').shape == (10, 3)
 
     def test_tolist_scalar(self):
         from numpypy import int32, bool_

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:42 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:42 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: implementation wip
Message-ID: <20130628124042.6440C1C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65059:c38b51152a62
Date: 2013-06-28 14:09 +0300
http://bitbucket.org/pypy/pypy/changeset/c38b51152a62/

Log:	implementation wip

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -75,9 +75,11 @@
         else:
             return None
 
-    def get_view(self, dtype, new_shape):
-        pass
-        #return SliceArray(self.start,
+    def get_view(self, orig_array, dtype, new_shape):
+        strides, backstrides = support.calc_strides(new_shape, dtype,
+                                                    self.order)
+        return SliceArray(self.start, strides, backstrides, new_shape,
+                          self, orig_array, dtype=dtype)
 
     def get_real(self, orig_array):
         strides = self.get_strides()
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -627,7 +627,24 @@
                                                                    w_dtype))
         else:
             dtype = self.get_dtype()
-        return W_NDimArray(self.implementation.get_view(self, dtype))
+        factor = float(dtype.get_size()) / self.get_dtype().get_size()
+        print 'new dtype',dtype,'self dtype',self.get_dtype(),'factor',factor
+        if self.get_size() % factor != 0:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "new type not compatible with array."))
+        new_shape = self.get_shape()
+        impl = self.implementation
+        if impl.get_strides()[0] < impl.get_strides()[-1]:
+            new_shape[0] = int(new_shape[0] * factor)
+            if new_shape[0] == 0:
+                raise OperationError(space.w_ValueError, space.wrap(
+                    "new type not compatible with array shape"))
+        else:
+            new_shape[-1] = int(new_shape[-1] * factor)
+            if new_shape[-1] == 0:
+                raise OperationError(space.w_ValueError, space.wrap(
+                    "new type not compatible with array shape"))
+        return W_NDimArray(self.implementation.get_view(self, dtype, new_shape))
 
 
     # --------------------- operations ----------------------------
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1415,6 +1415,7 @@
         from numpypy import array, int8, int16, dtype, int64
         x = array([(1, 2)], dtype=[('a', int8), ('b', int8)])
         y = x.view(dtype=int16)
+        print y,y.shape
         assert y[0] == 513
         assert y.dtype == dtype('int16')
         y[0] = 670

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:43 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:43 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: ndarray passes
Message-ID: <20130628124043.88CCD1C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65060:c4938bea5968
Date: 2013-06-28 14:26 +0300
http://bitbucket.org/pypy/pypy/changeset/c4938bea5968/

Log:	ndarray passes

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -635,12 +635,12 @@
         new_shape = self.get_shape()
         impl = self.implementation
         if impl.get_strides()[0] < impl.get_strides()[-1]:
-            new_shape[0] = int(new_shape[0] * factor)
+            new_shape[0] = int(new_shape[0] / factor)
             if new_shape[0] == 0:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array shape"))
         else:
-            new_shape[-1] = int(new_shape[-1] * factor)
+            new_shape[-1] = int(new_shape[-1] / factor)
             if new_shape[-1] == 0:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array shape"))
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1412,7 +1412,7 @@
         assert a[2].imag == -5
 
     def test_ndarray_view(self):
-        from numpypy import array, int8, int16, dtype, int64
+        from numpypy import array, int8, int16, dtype
         x = array([(1, 2)], dtype=[('a', int8), ('b', int8)])
         y = x.view(dtype=int16)
         print y,y.shape
@@ -1421,20 +1421,23 @@
         y[0] = 670
         assert x['a'] == -98
         assert x['b'] == 2
-        f = array([1000, -1234], dtype='i')
+        f = array([1000, -1234], dtype='i4')
         nnp = self.non_native_prefix
-        d = f.view(dtype=nnp + 'i')
+        d = f.view(dtype=nnp + 'i4')
         assert (d == [-402456576,  788267007]).all()
+        x = array(range(15), dtype='i2').reshape(3,5)
+        exc = raises(ValueError, x.view, dtype='i4')
+        assert exc.value[0] == "new type not compatible with array."
+        assert x.view('int8').shape == (3, 10)
+        x = array(range(15), dtype='int16').reshape(3,5).T
+        assert x.view('int8').shape == (10, 3)
+
+    def test_scalar_view(self):
+        from numpypy import int64
         s = int64(12)
         exc = raises(TypeError, s.view, dtype='int8')
         assert exc.value[0] == "view() takes no keyword arguments"
         assert s.view('double') < 7e-323
-        x = array(range(15), dtype='int16').reshape(3,5)
-        exc = raises(ValueError, x.view, dtype='int32')
-        assert exc.value[0] == "new type not compatible with array."
-        assert x.view('int8').shape == (3, 10)
-        x = array(range(15), dtype='int16').reshape(3,5).T
-        assert x.view('int8').shape == (10, 3)
 
     def test_tolist_scalar(self):
         from numpypy import int32, bool_

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:44 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:44 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: implement for zeroD arrays,
	skip for dtype scalars
Message-ID: <20130628124044.AB7EC1C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65061:e6bc534ed503
Date: 2013-06-28 15:34 +0300
http://bitbucket.org/pypy/pypy/changeset/e6bc534ed503/

Log:	implement for zeroD arrays, skip for dtype scalars

diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -63,6 +63,11 @@
     def transpose(self, _):
         return self
 
+    def get_view(self, orig_array, dtype, new_shape):
+        scalar = Scalar(dtype)
+        scalar.value = self.value.convert_to(dtype)
+        return scalar
+
     def get_real(self, orig_array):
         if self.dtype.is_complex_type():
             scalar = Scalar(self.dtype.float_type)
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -240,6 +240,16 @@
         v = self.convert_to(self.get_dtype(space))
         return self.get_dtype(space).itemtype.round(v, decimals)
 
+    def descr_view(self, space, w_dtype):
+        from pypy.module.micronumpy.interp_dtype import W_Dtype
+        dtype = space.interp_w(W_Dtype,
+            space.call_function(space.gettypefor(W_Dtype), w_dtype))
+        if dtype.get_size() != self.get_dtype(space).get_size():
+            raise OperationError(space.w_ValueError, space.wrap(
+                "new type not compatible with array."))
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "view not implelemnted yet"))
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -507,6 +517,7 @@
     all = interp2app(W_GenericBox.descr_all),
     ravel = interp2app(W_GenericBox.descr_ravel),
     round = interp2app(W_GenericBox.descr_round),
+    view = interp2app(W_GenericBox.descr_view),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -628,23 +628,27 @@
         else:
             dtype = self.get_dtype()
         factor = float(dtype.get_size()) / self.get_dtype().get_size()
-        print 'new dtype',dtype,'self dtype',self.get_dtype(),'factor',factor
         if self.get_size() % factor != 0:
             raise OperationError(space.w_ValueError, space.wrap(
                 "new type not compatible with array."))
+        impl = self.implementation
         new_shape = self.get_shape()
-        impl = self.implementation
-        if impl.get_strides()[0] < impl.get_strides()[-1]:
-            new_shape[0] = int(new_shape[0] / factor)
-            if new_shape[0] == 0:
+        if len(new_shape) > 0:
+            if impl.get_strides()[0] < impl.get_strides()[-1]:
+                new_shape[0] = int(new_shape[0] / factor)
+                if new_shape[0] == 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array shape"))
+            else:
+                new_shape[-1] = int(new_shape[-1] / factor)
+                if new_shape[-1] == 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array shape"))
+        else:
+            if factor != 1:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array shape"))
-        else:
-            new_shape[-1] = int(new_shape[-1] / factor)
-            if new_shape[-1] == 0:
-                raise OperationError(space.w_ValueError, space.wrap(
-                    "new type not compatible with array shape"))
-        return W_NDimArray(self.implementation.get_view(self, dtype, new_shape))
+        return W_NDimArray(impl.get_view(self, dtype, new_shape))
 
 
     # --------------------- operations ----------------------------
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1433,10 +1433,15 @@
         assert x.view('int8').shape == (10, 3)
 
     def test_scalar_view(self):
-        from numpypy import int64
+        from numpypy import int64, array
+        a = array(0, dtype='int32')
+        b = a.view(dtype='float32')
+        assert b.shape == ()
+        assert b == 0
         s = int64(12)
-        exc = raises(TypeError, s.view, dtype='int8')
-        assert exc.value[0] == "view() takes no keyword arguments"
+        exc = raises(ValueError, s.view, 'int8')
+        assert exc.value[0] == "new type not compatible with array."
+        skip('not implemented yet')
         assert s.view('double') < 7e-323
 
     def test_tolist_scalar(self):

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:45 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:45 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: document branch
Message-ID: <20130628124045.C758A1C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65062:2a679231487a
Date: 2013-06-28 15:36 +0300
http://bitbucket.org/pypy/pypy/changeset/2a679231487a/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -61,3 +61,7 @@
 
 .. branch: faster-str-of-bigint
 Improve performance of str(long).
+
+.. branch: ndarray-view
+Add view to ndarray and zeroD arrays, not on dtype scalars yet
+

From noreply at buildbot.pypy.org  Fri Jun 28 14:40:46 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:46 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-view: close branch about to be merged
Message-ID: <20130628124046.DD29A1C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-view
Changeset: r65063:963ba00d3e44
Date: 2013-06-28 15:37 +0300
http://bitbucket.org/pypy/pypy/changeset/963ba00d3e44/

Log:	close branch about to be merged


From noreply at buildbot.pypy.org  Fri Jun 28 14:40:48 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Jun 2013 14:40:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge ndarray-view which implements
 view() for ndarrays and zeroD arrays, but not for scalars
Message-ID: <20130628124048.173A31C010B@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65064:0abc7abacba2
Date: 2013-06-28 15:39 +0300
http://bitbucket.org/pypy/pypy/changeset/0abc7abacba2/

Log:	merge ndarray-view which implements view() for ndarrays and zeroD
	arrays, but not for scalars

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -61,3 +61,7 @@
 
 .. branch: faster-str-of-bigint
 Improve performance of str(long).
+
+.. branch: ndarray-view
+Add view to ndarray and zeroD arrays, not on dtype scalars yet
+
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -75,6 +75,12 @@
         else:
             return None
 
+    def get_view(self, orig_array, dtype, new_shape):
+        strides, backstrides = support.calc_strides(new_shape, dtype,
+                                                    self.order)
+        return SliceArray(self.start, strides, backstrides, new_shape,
+                          self, orig_array, dtype=dtype)
+
     def get_real(self, orig_array):
         strides = self.get_strides()
         backstrides = self.get_backstrides()
diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -63,6 +63,11 @@
     def transpose(self, _):
         return self
 
+    def get_view(self, orig_array, dtype, new_shape):
+        scalar = Scalar(dtype)
+        scalar.value = self.value.convert_to(dtype)
+        return scalar
+
     def get_real(self, orig_array):
         if self.dtype.is_complex_type():
             scalar = Scalar(self.dtype.float_type)
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -240,6 +240,16 @@
         v = self.convert_to(self.get_dtype(space))
         return self.get_dtype(space).itemtype.round(v, decimals)
 
+    def descr_view(self, space, w_dtype):
+        from pypy.module.micronumpy.interp_dtype import W_Dtype
+        dtype = space.interp_w(W_Dtype,
+            space.call_function(space.gettypefor(W_Dtype), w_dtype))
+        if dtype.get_size() != self.get_dtype(space).get_size():
+            raise OperationError(space.w_ValueError, space.wrap(
+                "new type not compatible with array."))
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "view not implelemnted yet"))
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -507,6 +517,7 @@
     all = interp2app(W_GenericBox.descr_all),
     ravel = interp2app(W_GenericBox.descr_ravel),
     round = interp2app(W_GenericBox.descr_round),
+    view = interp2app(W_GenericBox.descr_view),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -618,8 +618,38 @@
             "trace not implemented yet"))
 
     def descr_view(self, space, w_dtype=None, w_type=None) :
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "view not implemented yet"))
+        if w_type is not None:
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "view(... type=<class>) not implemented yet"))
+        if w_dtype:
+            dtype = space.interp_w(interp_dtype.W_Dtype,
+                space.call_function(space.gettypefor(interp_dtype.W_Dtype),
+                                                                   w_dtype))
+        else:
+            dtype = self.get_dtype()
+        factor = float(dtype.get_size()) / self.get_dtype().get_size()
+        if self.get_size() % factor != 0:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "new type not compatible with array."))
+        impl = self.implementation
+        new_shape = self.get_shape()
+        if len(new_shape) > 0:
+            if impl.get_strides()[0] < impl.get_strides()[-1]:
+                new_shape[0] = int(new_shape[0] / factor)
+                if new_shape[0] == 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array shape"))
+            else:
+                new_shape[-1] = int(new_shape[-1] / factor)
+                if new_shape[-1] == 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array shape"))
+        else:
+            if factor != 1:
+                raise OperationError(space.w_ValueError, space.wrap(
+                    "new type not compatible with array shape"))
+        return W_NDimArray(impl.get_view(self, dtype, new_shape))
+
 
     # --------------------- operations ----------------------------
 
@@ -996,6 +1026,7 @@
     round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
+    view = interp2app(W_NDimArray.descr_view),
 
     ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented
     __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1411,6 +1411,39 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
+    def test_ndarray_view(self):
+        from numpypy import array, int8, int16, dtype
+        x = array([(1, 2)], dtype=[('a', int8), ('b', int8)])
+        y = x.view(dtype=int16)
+        print y,y.shape
+        assert y[0] == 513
+        assert y.dtype == dtype('int16')
+        y[0] = 670
+        assert x['a'] == -98
+        assert x['b'] == 2
+        f = array([1000, -1234], dtype='i4')
+        nnp = self.non_native_prefix
+        d = f.view(dtype=nnp + 'i4')
+        assert (d == [-402456576,  788267007]).all()
+        x = array(range(15), dtype='i2').reshape(3,5)
+        exc = raises(ValueError, x.view, dtype='i4')
+        assert exc.value[0] == "new type not compatible with array."
+        assert x.view('int8').shape == (3, 10)
+        x = array(range(15), dtype='int16').reshape(3,5).T
+        assert x.view('int8').shape == (10, 3)
+
+    def test_scalar_view(self):
+        from numpypy import int64, array
+        a = array(0, dtype='int32')
+        b = a.view(dtype='float32')
+        assert b.shape == ()
+        assert b == 0
+        s = int64(12)
+        exc = raises(ValueError, s.view, 'int8')
+        assert exc.value[0] == "new type not compatible with array."
+        skip('not implemented yet')
+        assert s.view('double') < 7e-323
+
     def test_tolist_scalar(self):
         from numpypy import int32, bool_
         x = int32(23)

From noreply at buildbot.pypy.org  Fri Jun 28 14:49:56 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 28 Jun 2013 14:49:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: gah,
 the rbigint.fromstr factory in the previous form was not at all
 elidable, 
Message-ID: <20130628124956.D2F261C010B@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r65065:5122bb09fd0e
Date: 2013-06-28 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/5122bb09fd0e/

Log:	gah, the rbigint.fromstr factory in the previous form was not at all
	elidable, due to the extra parser argument. Fix it, by splitting
	things up differently.

diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py
--- a/pypy/objspace/std/inttype.py
+++ b/pypy/objspace/std/inttype.py
@@ -10,6 +10,7 @@
 from rpython.rlib.objectmodel import instantiate
 from rpython.rlib.rbigint import rbigint
 from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
+from rpython.rlib import jit
 
 # ____________________________________________________________
 
@@ -61,6 +62,7 @@
 
 # ____________________________________________________________
 
+ at jit.elidable
 def string_to_int_or_long(space, string, base=10):
     w_longval = None
     value = 0
@@ -73,10 +75,10 @@
         w_longval = retry_to_w_long(space, e.parser)
     return value, w_longval
 
-def retry_to_w_long(space, parser, base=0):
+def retry_to_w_long(space, parser):
     parser.rewind()
     try:
-        bigint = rbigint.fromstr(None, base=base, parser=parser)
+        bigint = rbigint._from_numberstring_parser(parser)
     except ParseStringError, e:
         raise OperationError(space.w_ValueError,
                              space.wrap(e.msg))
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -248,23 +248,26 @@
     @staticmethod
     @jit.elidable
     def fromdecimalstr(s):
-        # This function is marked as pure, so you must not call it and
+        # This function is marked as elidable, so you must not call it and
         # then modify the result.
         return _decimalstr_to_bigint(s)
 
     @staticmethod
     @jit.elidable
-    def fromstr(s, base=0, parser=None):
+    def fromstr(s, base=0):
         """As string_to_int(), but ignores an optional 'l' or 'L' suffix
         and returns an rbigint."""
         from rpython.rlib.rstring import NumberStringParser, \
             strip_spaces
-        if parser is None:
-            s = literal = strip_spaces(s)
-            if (s.endswith('l') or s.endswith('L')) and base < 22:
-                # in base 22 and above, 'L' is a valid digit!  try: long('L',22)
-                s = s[:-1]
-            parser = NumberStringParser(s, literal, base, 'long')
+        s = literal = strip_spaces(s)
+        if (s.endswith('l') or s.endswith('L')) and base < 22:
+            # in base 22 and above, 'L' is a valid digit!  try: long('L',22)
+            s = s[:-1]
+        parser = NumberStringParser(s, literal, base, 'long')
+        return rbigint._from_numberstring_parser(parser)
+
+    @staticmethod
+    def _from_numberstring_parser(parser):
         return parse_digit_string(parser)
 
     @staticmethod
diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -222,6 +222,11 @@
         assert rbigint.fromstr('123L', 21).tolong() == 441 + 42 + 3
         assert rbigint.fromstr('1891234174197319').tolong() == 1891234174197319
 
+    def test_from_numberstring_parser(self):
+        from rpython.rlib.rstring import NumberStringParser
+        parser = NumberStringParser("1231231241", "1231231241", 10, "long")
+        assert rbigint._from_numberstring_parser(parser).tolong() == 1231231241
+
     def test_add(self):
         x = 123456789123456789000000L
         y = 123858582373821923936744221L

From noreply at buildbot.pypy.org  Fri Jun 28 16:28:08 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 16:28:08 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: don't crash in this case
Message-ID: <20130628142808.AFBFC1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65066:6f79e542d94b
Date: 2013-06-28 15:34 +0200
http://bitbucket.org/pypy/pypy/changeset/6f79e542d94b/

Log:	don't crash in this case

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -2,6 +2,7 @@
 import math
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import rfloat
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import unicodehelper
@@ -157,7 +158,10 @@
             # build the float
             floatval = intval + frcval
             if exp != 0:
-                floatval = floatval * math.pow(10, exp)
+                try:
+                    floatval = floatval * math.pow(10, exp)
+                except OverflowError:
+                    floatval = rfloat.INFINITY
             return self.space.wrap(floatval)
         else:
             return self.space.wrap(intval)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -126,6 +126,8 @@
         check(x, float(x))
         x = str(sys.maxint+1) + 'E1'
         check(x, float(x))
+        #
+        check('1E400', float('inf'))
 
     def test_decode_numeric_invalid(self):
         import _fastjson

From noreply at buildbot.pypy.org  Fri Jun 28 16:28:09 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 16:28:09 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: bah,
 this line was moved by mistake by 89c55c007418. Also,
 rename the test to have a better name
Message-ID: <20130628142809.E8C411C01E5@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65067:4f40b6591af0
Date: 2013-06-28 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/4f40b6591af0/

Log:	bah, this line was moved by mistake by 89c55c007418. Also, rename
	the test to have a better name

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -316,8 +316,8 @@
             val = int(hexdigits, 16)
             if val & 0xfc00 == 0xd800:
                 # surrogate pair
+                val = self.decode_surrogate_pair(i, val)
                 i += 6
-                val = self.decode_surrogate_pair(i, val)
         except ValueError:
             self._raise("Invalid \uXXXX escape (char %d)", i-1)
             return # help the annotator to know that we'll never go beyond
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -149,7 +149,7 @@
         raises(ValueError, "_fastjson.loads('[1: 2]')")
         raises(ValueError, "_fastjson.loads('[1, 2')")
 
-    def test_big_unicode_decode(self):
+    def test_unicode_surrogate_pair(self):
         import _fastjson
         expected = u'z\U0001d120x'
         res = _fastjson.loads('"z\\ud834\\udd20x"')

From noreply at buildbot.pypy.org  Fri Jun 28 16:28:11 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 16:28:11 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: merge heads
Message-ID: <20130628142811.1FEBB1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65068:54438eb6c42c
Date: 2013-06-28 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/54438eb6c42c/

Log:	merge heads

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -1,11 +1,15 @@
+import sys
 import math
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import rfloat
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import unicodehelper
 from rpython.rtyper.annlowlevel import llstr, hlunicode
 
+OVF_DIGITS = len(str(sys.maxint))
+
 def is_whitespace(ch):
     return ch == ' ' or ch == '\t' or ch == '\r' or ch == '\n'
 
@@ -53,8 +57,8 @@
         self.last_type = TYPE_UNKNOWN
 
     def getslice(self, start, end):
-        assert start > 0
-        assert end > 0
+        assert start >= 0
+        assert end >= 0
         return self.s[start:end]
 
     def skip_whitespace(self, i):
@@ -117,7 +121,20 @@
         self._raise("Error when decoding false at char %d", i)
 
     def decode_numeric(self, i):
-        i, intval = self.parse_integer(i)
+        w_res = self.decode_numeric_fast(i)
+        if w_res is self.space.w_None:
+            # possible overflow, reparse it
+            return self.decode_numeric_slow(i)
+        return w_res
+
+    def decode_numeric_fast(self, i):
+        i, ovf_maybe, intval = self.parse_integer(i, allow_leading_0=False)
+        if ovf_maybe:
+            # apparently we get a ~30% slowdown on my microbenchmark if we
+            # return None instead of w_None, probably because the annotation
+            # of the results geta can_be_None=True. We need to check if this
+            # is still true also for the full pypy
+            return self.space.w_None
         #
         is_float = False
         exp = 0
@@ -134,28 +151,72 @@
         # check for the optional exponent part
         if ch == 'E' or ch == 'e':
             is_float = True
-            i, exp = self.parse_integer(i+1)
+            i, ovf_maybe, exp = self.parse_integer(i+1, allow_leading_0=True)
         #
         self.pos = i
         if is_float:
             # build the float
             floatval = intval + frcval
             if exp != 0:
-                floatval = floatval * math.pow(10, exp)
+                try:
+                    floatval = floatval * math.pow(10, exp)
+                except OverflowError:
+                    floatval = rfloat.INFINITY
             return self.space.wrap(floatval)
         else:
             return self.space.wrap(intval)
 
-    def parse_integer(self, i):
+    def decode_numeric_slow(self, i):
+        is_float = False
+        start = i
+        if self.ll_chars[i] == '-':
+            i += 1
+        # skip the integral part
+        while self.ll_chars[i].isdigit():
+            i += 1
+        #
+        # skip the fractional part, if any
+        if self.ll_chars[i] == '.':
+            is_float = True
+            i += 1
+            while self.ll_chars[i].isdigit():
+                i += 1
+        #
+        # skip the exponent part, if any
+        if self.ll_chars[i] == 'e' or self.ll_chars[i] == 'E':
+            is_float = True
+            i += 1
+            while self.ll_chars[i].isdigit():
+                i += 1
+        #
+        self.pos = i
+        s = self.getslice(start, i)
+        if is_float:
+            w_func = self.space.w_float
+        else:
+            w_func = self.space.w_int
+        w_res = self.space.call_function(w_func, self.space.wrap(s))
+        #assert w_res is not None # XXX check if this brings any speedup in pypy-c
+        return w_res
+
+    def parse_integer(self, i, allow_leading_0=False):
         "Parse a decimal number with an optional minus sign"
+        start = i
         sign = 1
         if self.ll_chars[i] == '-':
             sign = -1
             i += 1
         elif self.ll_chars[i] == '+':
             i += 1
+        elif not allow_leading_0 and self.ll_chars[i] == '0':
+            i += 1
+            return i, False, 0
         i, intval, _ = self.parse_digits(i)
-        return i, sign * intval
+        # if the number has more digits than OVF_DIGITS, it might have
+        # overflowed
+        ovf_maybe = (i-start >= OVF_DIGITS)
+        return i, ovf_maybe, sign * intval
+    parse_integer._always_inline_ = True
 
     def parse_digits(self, i):
         "Parse a sequence of digits as a decimal number. No sign allowed"
@@ -172,7 +233,7 @@
         if count == 0:
             self._raise("Expected digit at char %d", i)
         return i, intval, count
-        
+
     def decode_array(self, i):
         w_list = self.space.newlist([])
         start = i
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -99,6 +99,7 @@
 
 
     def test_decode_numeric(self):
+        import sys
         import _fastjson
         def check(s, val):
             res = _fastjson.loads(s)
@@ -113,6 +114,20 @@
         check('42E-1', 4.2)
         check('42E+1', 420.0)
         check('42.123E3', 42123.0)
+        check('0', 0)
+        check('-0', 0)
+        check('0.123', 0.123)
+        check('0E3', 0.0)
+        check('5E0001', 50.0)
+        check(str(1 << 32), 1 << 32)
+        check(str(1 << 64), 1 << 64)
+        #
+        x = str(sys.maxint+1) + '.123'
+        check(x, float(x))
+        x = str(sys.maxint+1) + 'E1'
+        check(x, float(x))
+        #
+        check('1E400', float('inf'))
 
     def test_decode_numeric_invalid(self):
         import _fastjson
@@ -121,10 +136,12 @@
         #
         error('  42   abc')
         error('.123')
+        error('+123')
         error('12.')
         error('12.-3')
         error('12E')
         error('12E-')
+        error('0123') # numbers can't start with 0
 
     def test_decode_object(self):
         import _fastjson

From noreply at buildbot.pypy.org  Fri Jun 28 16:28:12 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 16:28:12 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: handle another not-so-corner case: an
 object with spaces inside
Message-ID: <20130628142812.34BBA1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65069:31fc5dae889c
Date: 2013-06-28 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/31fc5dae889c/

Log:	handle another not-so-corner case: an object with spaces inside

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -265,12 +265,14 @@
     def decode_object(self, i):
         start = i
         w_dict = self.space.newdict()
+        #
+        i = self.skip_whitespace(i)
+        ch = self.ll_chars[i]
+        if ch == '}':
+            self.pos = i+1
+            return w_dict
+
         while True:
-            ch = self.ll_chars[i]
-            if ch == '}':
-                self.pos = i+1
-                return w_dict
-            #
             # parse a key: value
             self.last_type = TYPE_UNKNOWN
             w_name = self.decode_any(i)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -146,6 +146,7 @@
     def test_decode_object(self):
         import _fastjson
         assert _fastjson.loads('{}') == {}
+        assert _fastjson.loads('{  }') == {}
         #
         s = '{"hello": "world", "aaa": "bbb"}'
         assert _fastjson.loads(s) == {'hello': 'world',
@@ -172,3 +173,4 @@
         res = _fastjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
 
+

From noreply at buildbot.pypy.org  Fri Jun 28 16:28:13 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 28 Jun 2013 16:28:13 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: test&fix for a trailing comma when
	parsing an array
Message-ID: <20130628142813.593E01C01E5@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65070:3659a717d8e8
Date: 2013-06-28 16:11 +0200
http://bitbucket.org/pypy/pypy/changeset/3659a717d8e8/

Log:	test&fix for a trailing comma when parsing an array

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -239,11 +239,11 @@
         start = i
         count = 0
         i = self.skip_whitespace(start)
+        if self.ll_chars[i] == ']':
+            self.pos = i+1
+            return w_list
+        #
         while True:
-            ch = self.ll_chars[i]
-            if ch == ']':
-                self.pos = i+1
-                return w_list
             w_item = self.decode_any(i)
             i = self.pos
             self.space.call_method(w_list, 'append', w_item)
@@ -267,11 +267,10 @@
         w_dict = self.space.newdict()
         #
         i = self.skip_whitespace(i)
-        ch = self.ll_chars[i]
-        if ch == '}':
+        if self.ll_chars[i] == '}':
             self.pos = i+1
             return w_dict
-
+        #
         while True:
             # parse a key: value
             self.last_type = TYPE_UNKNOWN
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -166,7 +166,8 @@
         assert _fastjson.loads('[1, 2]') == [1, 2]
         raises(ValueError, "_fastjson.loads('[1: 2]')")
         raises(ValueError, "_fastjson.loads('[1, 2')")
-
+        raises(ValueError, """_fastjson.loads('["extra comma",]')""")
+        
     def test_unicode_surrogate_pair(self):
         import _fastjson
         expected = u'z\U0001d120x'

From noreply at buildbot.pypy.org  Fri Jun 28 16:47:43 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 28 Jun 2013 16:47:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix translation,
	int%float is not RPython.
Message-ID: <20130628144743.F243A1C01E5@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r65071:d77e4dd8ef04
Date: 2013-06-28 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/d77e4dd8ef04/

Log:	Fix translation, int%float is not RPython. Don't use floats when
	integer arithmetics is possible, I'm sure there were possible
	rounding errors as well.

	This also fixes a bug for empty arrays.

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -627,27 +627,29 @@
                                                                    w_dtype))
         else:
             dtype = self.get_dtype()
-        factor = float(dtype.get_size()) / self.get_dtype().get_size()
-        if self.get_size() % factor != 0:
-            raise OperationError(space.w_ValueError, space.wrap(
-                "new type not compatible with array."))
+        old_itemsize = self.get_dtype().get_size()
+        new_itemsize = dtype.get_size()
         impl = self.implementation
         new_shape = self.get_shape()
-        if len(new_shape) > 0:
-            if impl.get_strides()[0] < impl.get_strides()[-1]:
-                new_shape[0] = int(new_shape[0] / factor)
-                if new_shape[0] == 0:
-                    raise OperationError(space.w_ValueError, space.wrap(
-                        "new type not compatible with array shape"))
-            else:
-                new_shape[-1] = int(new_shape[-1] / factor)
-                if new_shape[-1] == 0:
-                    raise OperationError(space.w_ValueError, space.wrap(
-                        "new type not compatible with array shape"))
-        else:
-            if factor != 1:
+        dims = len(new_shape)
+        if dims == 0:
+            # Cannot resize scalars
+            if old_itemsize != new_itemsize:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array shape"))
+        else:
+            if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]:
+                # Column-major, resize first dimension
+                if new_shape[0] * old_itemsize % new_itemsize != 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                new_shape[0] = new_shape[0] * old_itemsize / new_itemsize
+            else:
+                # Row-major, resize last dimension
+                if new_shape[-1] * old_itemsize % new_itemsize != 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                new_shape[-1] = new_shape[-1] * old_itemsize / new_itemsize
         return W_NDimArray(impl.get_view(self, dtype, new_shape))
 
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1432,6 +1432,11 @@
         x = array(range(15), dtype='int16').reshape(3,5).T
         assert x.view('int8').shape == (10, 3)
 
+    def test_ndarray_view_empty(self):
+        from numpypy import array, int8, int16, dtype
+        x = array([], dtype=[('a', int8), ('b', int8)])
+        y = x.view(dtype=int16)
+
     def test_scalar_view(self):
         from numpypy import int64, array
         a = array(0, dtype='int32')

From noreply at buildbot.pypy.org  Fri Jun 28 19:59:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 19:59:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update this --- support for switch()
 with keyword arguments --- to the
Message-ID: <20130628175949.EE58D1C0883@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65072:307d5501ecf6
Date: 2013-06-28 19:45 +0200
http://bitbucket.org/pypy/pypy/changeset/307d5501ecf6/

Log:	Update this --- support for switch() with keyword arguments --- to
	the current version of greenlet-for-CPython.

diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -46,16 +46,16 @@
         if parent is not None:
             self.parent = parent
 
-    def switch(self, *args):
+    def switch(self, *args, **kwds):
         "Switch execution to this greenlet, optionally passing the values "
         "given as argument(s).  Returns the value passed when switching back."
-        return self.__switch('switch', args)
+        return self.__switch('switch', (args, kwds))
 
     def throw(self, typ=GreenletExit, val=None, tb=None):
         "raise exception in greenlet, return value passed when switching back"
         return self.__switch('throw', typ, val, tb)
 
-    def __switch(target, methodname, *args):
+    def __switch(target, methodname, *baseargs):
         current = getcurrent()
         #
         while not (target.__main or _continulet.is_pending(target)):
@@ -65,9 +65,9 @@
                     greenlet_func = _greenlet_start
                 else:
                     greenlet_func = _greenlet_throw
-                _continulet.__init__(target, greenlet_func, *args)
+                _continulet.__init__(target, greenlet_func, *baseargs)
                 methodname = 'switch'
-                args = ()
+                baseargs = ()
                 target.__started = True
                 break
             # already done, go to the parent instead
@@ -78,11 +78,15 @@
         #
         try:
             unbound_method = getattr(_continulet, methodname)
-            args = unbound_method(current, *args, to=target)
+            args, kwds = unbound_method(current, *baseargs, to=target)
         finally:
             _tls.current = current
         #
-        if len(args) == 1:
+        if kwds:
+            if args:
+                return args, kwds
+            return kwds
+        elif len(args) == 1:
             return args[0]
         else:
             return args
@@ -128,15 +132,15 @@
     _tls.main = gmain
     _tls.current = gmain
 
-def _greenlet_start(greenlet, args):
+def _greenlet_start(greenlet, (args, kwds)):
     _tls.current = greenlet
     try:
-        res = greenlet.run(*args)
+        res = greenlet.run(*args, **kwds)
     except GreenletExit, e:
         res = e
     finally:
         _continuation.permute(greenlet, greenlet.parent)
-    return (res,)
+    return ((res,), None)
 
 def _greenlet_throw(greenlet, exc, value, tb):
     _tls.current = greenlet
diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/test_lib_pypy/test_greenlet.py
--- a/pypy/module/test_lib_pypy/test_greenlet.py
+++ b/pypy/module/test_lib_pypy/test_greenlet.py
@@ -319,3 +319,25 @@
         g = G(lambda: 42)
         x = g.switch()
         assert x == 42
+
+    def test_kwargs_to_f(self):
+        import greenlet
+        seen = []
+        def f(*args, **kwds):
+            seen.append([args, kwds])
+        g = greenlet.greenlet(f)
+        g.switch(1, 2, x=3, y=4)
+        assert seen == [[(1, 2), {'x': 3, 'y': 4}]]
+
+    def test_kwargs_to_switch(self):
+        import greenlet
+        main = greenlet.getcurrent()
+        assert main.switch() == ()
+        assert main.switch(5) == 5
+        assert main.switch(5, 6) == (5, 6)
+        #
+        assert main.switch(x=5) == {'x': 5}
+        assert main.switch(x=5, y=6) == {'x': 5, 'y': 6}
+        assert main.switch(3, x=5) == ((3,), {'x': 5})
+        assert main.switch(3, x=5, y=6) == ((3,), {'x': 5, 'y': 6})
+        assert main.switch(2, 3, x=6) == ((2, 3), {'x': 6})

From noreply at buildbot.pypy.org  Fri Jun 28 19:59:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 19:59:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: If we are running on top of pypy,
 we import only _minimal_curses.
Message-ID: <20130628175951.2DEA11C08AF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65073:b07a14bc6fb7
Date: 2013-06-28 19:59 +0200
http://bitbucket.org/pypy/pypy/changeset/b07a14bc6fb7/

Log:	If we are running on top of pypy, we import only _minimal_curses.
	Don't try to fall back to _curses, because that's going to use cffi
	and fall again more loudly.

diff --git a/lib_pypy/pyrepl/curses.py b/lib_pypy/pyrepl/curses.py
--- a/lib_pypy/pyrepl/curses.py
+++ b/lib_pypy/pyrepl/curses.py
@@ -19,11 +19,15 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-# avoid importing the whole curses, if possible
-try:
+# If we are running on top of pypy, we import only _minimal_curses.
+# Don't try to fall back to _curses, because that's going to use cffi
+# and fall again more loudly.
+import sys
+if '__pypy__' in sys.builtin_module_names:
     # pypy case
     import _minimal_curses as _curses
-except ImportError:
+else:
+    # cpython case
     try:
         import _curses
     except ImportError:

From noreply at buildbot.pypy.org  Fri Jun 28 20:07:29 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:07:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Attempt a large change,
 for now just by commenting out one line:
Message-ID: <20130628180729.9DD191C0883@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65074:a7e480ec1a78
Date: 2013-06-28 20:06 +0200
http://bitbucket.org/pypy/pypy/changeset/a7e480ec1a78/

Log:	Attempt a large change, for now just by commenting out one line:
	disable the "validator" logic that deselects modules "silently" if
	they cannot be imported. Now if a module is enabled by default, but
	importing it raises (typically because the .h files are not around),
	then the exception is propagated to the user rather than just giving
	an easily-missed message.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -144,7 +144,7 @@
                    requires=module_dependencies.get(modname, []),
                    suggests=module_suggests.get(modname, []),
                    negation=modname not in essential_modules,
-                   validator=get_module_validator(modname))
+                   ) #validator=get_module_validator(modname))
         for modname in all_modules]),
 
     BoolOption("allworkingmodules", "use as many working modules as possible",

From noreply at buildbot.pypy.org  Fri Jun 28 20:55:18 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:18 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc-2: Old checkin
Message-ID: <20130628185518.305901C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc-2
Changeset: r65075:516680f7722a
Date: 2013-06-28 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/516680f7722a/

Log:	Old checkin

diff --git a/rpython/memory/gc/stmshared.py b/rpython/memory/gc/stmshared.py
--- a/rpython/memory/gc/stmshared.py
+++ b/rpython/memory/gc/stmshared.py
@@ -295,7 +295,7 @@
         some other data structure.  Note that it is not zero-filled."""
         nsize = llmemory.raw_malloc_usage(totalsize)
         if nsize <= self.sharedarea.small_request_threshold:
-            size_class = (nsize + WORD_POWER_2 - 1) >> WORD_POWER_2
+            size_class = (nsize + WORD - 1) >> WORD_POWER_2
             return self._malloc_size_class(size_class)
         else:
             count = llmemory.raw_malloc_usage(totalsize)

From noreply at buildbot.pypy.org  Fri Jun 28 20:55:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:19 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Start a different branch in which to
 use the GC provided as C code
Message-ID: <20130628185519.786A11C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65076:f08bde0e50b4
Date: 2013-06-28 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/f08bde0e50b4/

Log:	Start a different branch in which to use the GC provided as C code
	by stmgc/c4/.


From noreply at buildbot.pypy.org  Fri Jun 28 20:55:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:20 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Start by removing this.
Message-ID: <20130628185520.BB0EB1C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65077:b74586ea71e5
Date: 2013-06-28 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/b74586ea71e5/

Log:	Start by removing this.

diff too long, truncating to 2000 out of 2843 lines

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
deleted file mode 100644
--- a/rpython/memory/gc/stmgc.py
+++ /dev/null
@@ -1,558 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup
-from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage, raw_memcopy
-from rpython.memory.gc.base import GCBase, MovingGCBase
-from rpython.memory.gc.env import addressable_size
-from rpython.memory.support import mangle_hash
-from rpython.rtyper.annlowlevel import llhelper
-from rpython.rlib.rarithmetic import LONG_BIT, r_uint
-from rpython.rlib.debug import ll_assert, fatalerror
-from rpython.rlib import rthread
-from rpython.memory.gc import stmshared
-
-
-WORD = LONG_BIT // 8
-NULL = llmemory.NULL
-
-first_gcflag = 1 << (LONG_BIT//2)
-
-# Documentation:
-# https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/stm2012/stmimpl.rst
-#
-# Terminology used here:
-#
-#   - Objects can be LOCAL or GLOBAL.  This is what STM is based on.
-#     Local objects are not visible to any other transaction, whereas
-#     global objects are, and are read-only.
-#
-#   - Each object lives either in the shared area, or in a thread-local
-#     nursery.  The shared area contains:
-#       - the prebuilt objects
-#       - the small objects allocated via minimarkpage.py (XXX so far,
-#         just with malloc)
-#       - the non-small raw-malloced objects
-#
-#   - The GLOBAL objects are all located in the shared area.
-#     All GLOBAL objects are non-movable.
-#
-#   - The LOCAL objects might be YOUNG or OLD depending on whether they
-#     already survived a collection.  YOUNG LOCAL objects are either in
-#     the nursery or, if they are big, raw-malloced.  OLD LOCAL objects
-#     are in the shared area.  Getting the write barrier right for both
-#     this and the general STM mechanisms is tricky, so for now this GC
-#     is not actually generational (slow when running long transactions
-#     or before running transactions at all).  (XXX fix me)
-#
-#   - So far, the GC is always running in "transactional" mode.  Later,
-#     it would be possible to speed it up in case there is only one
-#     (non-blocked) thread.
-#
-# GC Flags on objects:
-#
-#   - GCFLAG_GLOBAL: identifies GLOBAL objects.  All prebuilt objects
-#     start as GLOBAL; conversely, all freshly allocated objects start
-#     as LOCAL, and become GLOBAL if they survive an end-of-transaction.
-#     All objects that are GLOBAL are immortal and leaking for now
-#     (global_collect() will be done later).
-#
-#   - GCFLAG_POSSIBLY_OUTDATED: see stmimpl.rst.  Used by C.
-#   - GCFLAG_NOT_WRITTEN: see stmimpl.rst.  Used by C.
-#   - GCFLAG_LOCAL_COPY: see stmimpl.rst.  Used by C.
-#     Note that GCFLAG_LOCAL_COPY and GCFLAG_GLOBAL are exclusive.
-#
-#   - GCFLAG_VISITED: used temporarily to mark local objects found to be
-#     surviving during a collection.  Between collections, it is set on
-#     the LOCAL COPY objects, but only on them.
-#
-#   - GCFLAG_HASH_FIELD: the object contains an extra field added at the
-#     end.  If GCFLAG_WITH_HASH (usual case), then the field contains
-#     both the hash and id value given to this object.  Otherwise,
-#     it's a prebuilt object; if GCFLAG_PREBUILT_ORIGINAL then the
-#     field contains the hash result but the id is the address of the
-#     object; otherwise the field contains the address of the original
-#     prebuilt object, where the hash result can be indirectly found.
-#
-#   - GCFLAG_WITH_HASH: the hash/id has been taken.  On a nursery object,
-#     means that it has an entry in 'nursery_objects_shadows'.  Otherwise,
-#     if GCFLAG_HASH_FIELD is set, that field stores the hash/id value.
-#     Otherwise, means that the hash/id is equal to this exact object's
-#     address.
-#
-# Invariant: between two transactions, all objects visible from the current
-# thread are always GLOBAL.  In particular:
-#
-#   - The LOCAL objects of a thread are not visible at all from other threads.
-#     This means that there is *no* pointer from a GLOBAL object directly to
-#     a LOCAL object.  At most, there can be pointers from a GLOBAL object to
-#     another GLOBAL object that itself has a LOCAL COPY --- or, of course,
-#     pointers from a LOCAL object to anything.
-#
-# Collection: for now we have only local_collection(), which ignores all
-# GLOBAL objects.
-#
-#   - To find the roots, we take the list (maintained by the C code)
-#     of the LOCAL COPY objects of the current transaction, together with
-#     the stack.  GLOBAL objects can be ignored because they have no
-#     pointer to any LOCAL object at all.
-#
-#   - A special case is the end-of-transaction collection, done by the same
-#     local_collection() with a twist: all surviving objects (after being
-#     copied out of the nursery) receive the flags GLOBAL and NOT_WRITTEN.
-#     At the end, we are back to the situation where this thread sees only
-#     GLOBAL objects.  This is PerformLocalCollect() in stmimpl.rst.
-#     It is done just before CommitTransaction(), implemented in C.
-#
-# All objects have an address-sized 'revision' field in their header.
-# It is generally used by the C code (marked with (*) below), but it
-# is also (ab)used by the GC itself (marked with (!) below).
-#
-#   - for local objects with GCFLAG_LOCAL_COPY, it points to the GLOBAL
-#     original (*).
-#
-#   - it contains the 'next' object of the 'sharedarea_tls.chained_list'
-#     list, which describes all LOCAL objects malloced outside the
-#     nursery (!).
-#
-#   - during collection, the nursery objects that are copied outside
-#     the nursery grow GCFLAG_VISITED and their 'revision' points
-#     to the new copy (!).
-#
-#   - on any GLOBAL object, 'revision' is managed by C code (*).
-#     It must be initialized to 1 when the GC code turns a LOCAL object
-#     into a GLOBAL one.
-#
-
-GCFLAG_GLOBAL            = first_gcflag << 0     # keep in sync with et.h
-GCFLAG_POSSIBLY_OUTDATED = first_gcflag << 1     # keep in sync with et.h
-GCFLAG_NOT_WRITTEN       = first_gcflag << 2     # keep in sync with et.h
-GCFLAG_LOCAL_COPY        = first_gcflag << 3     # keep in sync with et.h
-GCFLAG_VISITED           = first_gcflag << 4     # keep in sync with et.h
-GCFLAG_HASH_FIELD        = first_gcflag << 5
-GCFLAG_WITH_HASH         = first_gcflag << 6
-GCFLAG_PREBUILT_ORIGINAL = first_gcflag << 7
-
-GCFLAG_PREBUILT_FLAGS    = (GCFLAG_GLOBAL |
-                            GCFLAG_NOT_WRITTEN |
-                            GCFLAG_PREBUILT_ORIGINAL)
-REV_INITIAL              = r_uint(1)
-
-
-def always_inline(fn):
-    fn._always_inline_ = True
-    return fn
-def dont_inline(fn):
-    fn._dont_inline_ = True
-    return fn
-
-
-class StmGC(MovingGCBase):
-    _alloc_flavor_ = "raw"
-    inline_simple_malloc = True
-    inline_simple_malloc_varsize = True
-    prebuilt_gc_objects_are_static_roots = False
-    malloc_zero_filled = True
-
-    HDR = lltype.Struct('header', ('tid', lltype.Signed),
-                                  ('revision', lltype.Unsigned))
-    typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'tid', GCFLAG_HASH_FIELD
-
-    TRANSLATION_PARAMS = {
-        'stm_operations': 'use_real_one',
-        'nursery_size': 32*1024*1024,           # 32 MB
-
-        "page_size": stmshared.TRANSLATED_PAGE_SIZE,
-        "small_request_threshold":stmshared.TRANSLATED_SMALL_REQUEST_THRESHOLD,
-    }
-
-    def __init__(self, config,
-                 stm_operations='use_emulator',
-                 nursery_size=1024,
-                 page_size=14*WORD,
-                 small_request_threshold=5*WORD,
-                 **kwds):
-        MovingGCBase.__init__(self, config, multithread=True, **kwds)
-        #
-        if isinstance(stm_operations, str):
-            assert stm_operations == 'use_real_one', (
-                "XXX not provided so far: stm_operations == %r" % (
-                stm_operations,))
-            from rpython.translator.stm.stmgcintf import StmOperations
-            stm_operations = StmOperations()
-        #
-        self.stm_operations = stm_operations
-        self.nursery_size = nursery_size
-        self.major_collection_threshold = 1.82     # xxx
-        self.min_heap_size = 8.0 * self.nursery_size
-        self.limit_for_major_gc = r_uint(self.min_heap_size)
-        #self.maximum_extra_threshold = 0
-        self.sharedarea = stmshared.StmGCSharedArea(self, page_size,
-                                                    small_request_threshold)
-        #
-        def _stm_duplicate(obj):     # indirection to hide 'self'
-            return self.stm_duplicate(obj)
-        self._stm_duplicate = _stm_duplicate
-
-    def setup(self):
-        """Called at run-time to initialize the GC."""
-        #
-        # Hack: MovingGCBase.setup() sets up stuff related to id(), which
-        # we implement differently anyway.  So directly call GCBase.setup().
-        GCBase.setup(self)
-        #
-        # The following line causes the _stm_getsize() function to be
-        # generated in the C source with a specific signature, where it
-        # can be called by the C code.
-        llop.nop(lltype.Void, llhelper(self.stm_operations.DUPLICATE,
-                                       self._stm_duplicate))
-        #
-        self.ll_global_lock = rthread.null_ll_lock
-        self.sharedarea.setup()
-        #
-        self.linked_list_stmtls = None
-        #
-        self.stm_operations.descriptor_init()
-        self.stm_operations.begin_inevitable_transaction()
-        self.setup_thread()
-        #
-        # Now the gc is running, we can allocate this lock object
-        self.ll_global_lock = rthread.allocate_ll_lock()
-
-    def setup_thread(self):
-        """Build the StmGCTLS object and start a transaction at the level
-        of the GC.  The C-level transaction should already be started."""
-        ll_assert(self.stm_operations.in_transaction(),
-                  "setup_thread: not in a transaction")
-        from rpython.memory.gc.stmtls import StmGCTLS
-        stmtls = StmGCTLS(self)
-        self.add_in_linked_list(stmtls)
-        stmtls.start_transaction()
-
-    def teardown_thread(self):
-        """Stop the current transaction, commit it at the level of
-        C code, and tear down the StmGCTLS object.  For symmetry, this
-        ensures that the level of C has another (empty) transaction
-        started."""
-        ll_assert(bool(self.stm_operations.in_transaction()),
-                  "teardown_thread: not in a transaction")
-        stmtls = self.get_tls()
-        stmtls.stop_transaction()
-        self.stm_operations.commit_transaction()
-        self.stm_operations.begin_inevitable_transaction()
-        self.remove_from_linked_list(stmtls)
-        stmtls.delete()
-    teardown_thread._dont_inline_ = True
-
-    def acquire_global_lock(self):
-        if self.ll_global_lock:
-            rthread.acquire_NOAUTO(self.ll_global_lock, True)
-
-    def release_global_lock(self):
-        if self.ll_global_lock:
-            rthread.release_NOAUTO(self.ll_global_lock)
-
-    def add_in_linked_list(self, stmtls):
-        self.acquire_global_lock()
-        stmnext = self.linked_list_stmtls
-        stmtls.linked_list_prev = None
-        stmtls.linked_list_next = stmnext
-        if stmnext is not None:
-            ll_assert(stmnext.linked_list_prev is None,
-                      "add_in_linked_list: bad linked list")
-            stmnext.linked_list_prev = stmtls
-        self.linked_list_stmtls = stmtls
-        self.release_global_lock()
-
-    def remove_from_linked_list(self, stmtls):
-        self.acquire_global_lock()
-        c = 0
-        stmprev = stmtls.linked_list_prev
-        stmnext = stmtls.linked_list_next
-        if stmnext is not None:
-            stmnext.linked_list_prev = stmprev
-        if stmprev is not None:
-            stmprev.linked_list_next = stmnext
-            c += 1
-        if stmtls is self.linked_list_stmtls:
-            self.linked_list_stmtls = stmnext
-            c += 1
-        ll_assert(c == 1, "remove_from_linked_list: bad linked list")
-        self.release_global_lock()
-
-    @always_inline
-    def get_tls(self):
-        from rpython.memory.gc.stmtls import StmGCTLS
-        tls = self.stm_operations.get_tls()
-        return StmGCTLS.cast_address_to_tls_object(tls)
-
-    @staticmethod
-    def JIT_max_size_of_young_obj():
-        return None
-
-    @staticmethod
-    def JIT_minimal_size_in_nursery():
-        return 0
-
-    # ----------
-
-    def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False,
-                               is_finalizer_light=False,
-                               contains_weakptr=False):
-        #assert not needs_finalizer, "XXX" --- finalizer is just ignored
-        #
-        # Get the memory from the thread-local nursery.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        totalsize = size_gc_header + size
-        tls = self.get_tls()
-        result = tls.allocate_bump_pointer(totalsize)
-        #
-        # Build the object.
-        llarena.arena_reserve(result, totalsize)
-        obj = result + size_gc_header
-        self.init_gc_object(result, typeid, flags=0)
-        if contains_weakptr:   # check constant-folded
-            tls.fresh_new_weakref(obj)
-        #
-        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-
-
-    def malloc_varsize_clear(self, typeid, length, size, itemsize,
-                             offset_to_length):
-        # XXX blindly copied from malloc_fixedsize_clear() for now.
-        # XXX Be more subtle, e.g. detecting overflows, at least
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        nonvarsize = size_gc_header + size
-        totalsize = nonvarsize + itemsize * length
-        totalsize = llarena.round_up_for_allocation(totalsize)
-        result = self.get_tls().allocate_bump_pointer(totalsize)
-        llarena.arena_reserve(result, totalsize)
-        obj = result + size_gc_header
-        self.init_gc_object(result, typeid, flags=0)
-        (obj + offset_to_length).signed[0] = length
-        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-
-
-    def collect(self, gen=1):
-        self.get_tls().local_collection()
-        if gen > 0:
-            self.major_collection()
-
-    def start_transaction(self):
-        self.get_tls().start_transaction()
-
-    def stop_transaction(self):
-        self.get_tls().stop_transaction()
-
-
-    @always_inline
-    def get_type_id(self, obj):
-        tid = self.header(obj).tid
-        return llop.extract_ushort(llgroup.HALFWORD, tid)
-
-    def get_size_incl_hash(self, obj):
-        size = self.get_size(obj)
-        hdr = self.header(obj)
-        if hdr.tid & GCFLAG_HASH_FIELD:
-            size += llmemory.sizeof(lltype.Signed)
-        return size
-
-    @always_inline
-    def combine(self, typeid16, flags):
-        return llop.combine_ushort(lltype.Signed, typeid16, flags)
-
-    @always_inline
-    def init_gc_object(self, addr, typeid16, flags=0):
-        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.tid = self.combine(typeid16, flags)
-
-    def init_gc_object_immortal(self, addr, typeid16, flags=0):
-        flags |= GCFLAG_PREBUILT_FLAGS
-        self.init_gc_object(addr, typeid16, flags)
-        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.revision = REV_INITIAL
-
-    def obj_revision(self, obj):
-        return hdr_revision(self.header(obj))
-
-    def set_obj_revision(self, obj, nrevision):
-        set_hdr_revision(self.header(obj), nrevision)
-
-    def stm_duplicate(self, obj):
-        tls = self.get_tls()
-        try:
-            localobj = tls.duplicate_obj(obj, self.get_size(obj))
-        except MemoryError:
-            # should not really let the exception propagate.
-            # XXX do something slightly better, like abort the transaction
-            # and raise a MemoryError when retrying
-            fatalerror("FIXME: MemoryError in stm_duplicate")
-            return llmemory.NULL
-        #
-        hdr = self.header(localobj)
-        hdr.tid &= ~(GCFLAG_GLOBAL | GCFLAG_POSSIBLY_OUTDATED |
-                     GCFLAG_PREBUILT_ORIGINAL)
-        hdr.tid |= (GCFLAG_VISITED | GCFLAG_LOCAL_COPY)
-        return localobj
-
-    # ----------
-    # set_extra_threshold support
-
-##    def set_extra_threshold(self, reserved_size):
-##        if reserved_size > self.maximum_extra_threshold:
-##            self.maximum_extra_threshold = reserved_size
-##        stmtls = self.get_tls()
-##        stmtls.set_extra_threshold(reserved_size)
-
-    # ----------
-    # major collections
-
-    def maybe_major_collection(self):
-        """Check the memory usage, and maybe do a major GC collection."""
-        if (self.sharedarea.fetch_count_total_bytes() >=
-                self.limit_for_major_gc):
-            self.major_collection()
-
-    def major_collection(self):
-        """Do a major collection.  This uses a stop-the-world system."""
-        #
-        # Setting this limit to 0 now will invite other threads to enter
-        # major_collection() soon too.  Note that this doesn't create a
-        # race, because all threads are first going to run until they are
-        # all at the start_single_thread() below, or in C code at a
-        # reached_safe_point() or  outside a transaction.
-        self.limit_for_major_gc = r_uint(0)
-        #
-        # Now wait until we can acquire the RW lock in exclusive mode.
-        self.stm_operations.start_single_thread()
-        #
-        # If several threads were blocked on the previous line, the first
-        # one to proceed sees 0 in this variable.  It's the thread that
-        # will do the major collection.  Afterwards the other threads will
-        # also acquire the RW lock in exclusive mode, but won't do anything.
-        if self.limit_for_major_gc == r_uint(0):
-            #
-            # do the major collection
-            self.sharedarea.do_major_collection()
-            #
-            # reset 'limit_for_major_gc' to the correct value at which the
-            # following major collection should take place.
-            in_use = self.sharedarea.fetch_count_total_bytes()
-            threshold = in_use * self.major_collection_threshold
-            if threshold < self.min_heap_size:
-                threshold = self.min_heap_size
-            if threshold > addressable_size * 0.99:
-                threshold = addressable_size * 0.99
-            self.limit_for_major_gc = r_uint(threshold)
-        #
-        self.stm_operations.stop_single_thread()
-    major_collection._dont_inline_ = True
-
-    # ----------
-    # id() and identityhash() support
-
-    def id_or_identityhash(self, gcobj, is_hash):
-        """Implement the common logic of id() and identityhash()
-        of an object, given as a GCREF.
-        """
-        # First go to the most up-to-date version of gcobj.  It can
-        # be the latest global version, or the local version if it was
-        # already modified during this transaction.
-        gcobj = llop.stm_read_barrier(lltype.typeOf(gcobj), gcobj)
-        obj = llmemory.cast_ptr_to_adr(gcobj)
-        #
-        flags = self.header(obj).tid & (GCFLAG_HASH_FIELD | GCFLAG_WITH_HASH)
-        #
-        if flags == GCFLAG_HASH_FIELD | GCFLAG_WITH_HASH:
-            # 'obj' has already an explicit hash/id field, and is not a
-            # prebuilt object at all.  Return the content of that field.
-            return self._get_hash_field(obj)
-        #
-        elif flags == GCFLAG_HASH_FIELD | 0:
-            # 'obj' is a prebuilt object with a hash field, or a runtime
-            # copy of such an object.
-            if not (self.header(obj).tid & GCFLAG_PREBUILT_ORIGINAL):
-                # 'obj' is a runtime copy of an original prebuilt object.
-                # Fetch from the "hash field" the address of this original.
-                obj = self._get_hash_field(obj)
-                obj = llmemory.cast_int_to_adr(obj)
-                ll_assert((self.header(obj).tid & GCFLAG_PREBUILT_ORIGINAL)
-                          != 0, "id/hash: expected a prebuilt_original")
-            #
-            if is_hash:
-                return self._get_hash_field(obj)
-        #
-        elif flags == 0 | GCFLAG_WITH_HASH:
-            # 'obj' doesn't have a hash/id field, but we already took its
-            # hash/id.  If it is a nursery object, go to its shadow.
-            tls = self.get_tls()
-            if tls.is_in_nursery(obj):
-                obj = tls.nursery_objects_shadows.get(obj)
-                ll_assert(obj != NULL,
-                         "GCFLAG_WITH_HASH on nursery obj but no shadow found")
-        #
-        else:  # flags == 0, 'obj' has no hash/id at all so far.
-            # We are going to force one; this is a write operation.
-            # Note that we cannot get here twice for the same gcobj
-            # in the same transaction: after stm_write_barrier, the
-            # stm_read_barrier() above will return the local object
-            # with GCFLAG_WITH_HASH set.
-            localgcobj = llop.stm_write_barrier(lltype.typeOf(gcobj),gcobj)
-            obj = llmemory.cast_ptr_to_adr(localgcobj)
-            realobj = obj
-            #
-            # If 'obj' is a nursery object, we need to make a shadow
-            tls = self.get_tls()
-            if tls.is_in_nursery(obj):
-                size_gc_header = self.gcheaderbuilder.size_gc_header
-                size = self.get_size(obj)
-                shadowhdr = tls.sharedarea_tls.malloc_object_addr(
-                    size_gc_header + size)
-                # XXX must initialize the shadow enough to be considered
-                # a valid gc object by the next major collection
-                obj = shadowhdr + size_gc_header
-                tls.nursery_objects_shadows.setitem(realobj, obj)
-            #
-            self.header(realobj).tid |= GCFLAG_WITH_HASH
-        #
-        # Cases that fall through are cases where the answer is the
-        # mangled address of 'obj'.
-        return self._get_mangled_address(obj)
-
-    def id(self, gcobj):
-        return self.id_or_identityhash(gcobj, False)
-
-    def identityhash(self, gcobj):
-        return self.id_or_identityhash(gcobj, True)
-
-    def _get_hash_field(self, obj):
-        objsize = self.get_size(obj)
-        obj = llarena.getfakearenaaddress(obj)
-        return (obj + objsize).signed[0]
-
-    def _get_mangled_address(self, obj):
-        i = llmemory.cast_adr_to_int(obj)
-        return mangle_hash(i)
-
-    def can_move(self, addr):
-        raise NotImplementedError("stmgc.can_move")
-        tls = self.get_tls()
-        if tls.is_in_nursery(addr):
-            return True
-        else:
-            # XXX for now a pointer to a non-nursery object is
-            # always valid, as long as we don't have global collections
-            # Later, we will need to remember that we answered False for
-            # this 'addr', and then no longer fix any pointer to it
-            # during global collections, so that the exact 'addr'
-            # remains valid as long as the object is reachable.
-            return False
-
-# ____________________________________________________________
-# helpers
-
-def hdr_revision(hdr):
-    return llmemory.cast_int_to_adr(hdr.revision)
-
-def set_hdr_revision(hdr, nrevision):
-    hdr.revision = llmemory.cast_adr_to_uint_symbolic(nrevision)
diff --git a/rpython/memory/gc/stmshared.py b/rpython/memory/gc/stmshared.py
deleted file mode 100644
--- a/rpython/memory/gc/stmshared.py
+++ /dev/null
@@ -1,365 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi
-from rpython.rlib.rarithmetic import LONG_BIT, r_uint
-from rpython.rlib.objectmodel import free_non_gc_object, we_are_translated
-from rpython.rlib.debug import ll_assert, fatalerror
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
-from rpython.rlib import rthread, atomic_ops
-
-WORD = LONG_BIT // 8
-NULL = llmemory.NULL
-WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
-assert 1 << WORD_POWER_2 == WORD
-
-# Linux's glibc is good at 'malloc(1023*WORD)': the blocks ("pages") it
-# returns are exactly 1024 words apart, reserving only one extra word
-# for its internal data.  Here we assume that even on other systems it
-# will not use more than two words.
-TRANSLATED_PAGE_SIZE = 1022 * WORD
-
-# This is the largest size that StmGCSharedArea will map to its internal
-# "pages" structures.
-TRANSLATED_SMALL_REQUEST_THRESHOLD = 35 * WORD
-
-# ------------------------------------------------------------
-# The basic idea here is that each page will contain objects that are
-# each of the same size.  Moreover each page belongs to one thread only:
-# only this thread can use it to satisfy more allocations --- with however
-# one exception: pages with low usage after a major collection are moved
-# to a global list where any thread can pick them up.
-
-PAGE_PTR = lltype.Ptr(lltype.ForwardReference())
-PAGE_HEADER = lltype.Struct('PageHeader',
-    # -- The following pointer makes a chained list of pages.
-    ('nextpage', PAGE_PTR),
-    # -- The following is only used when the page belongs to StmGCSharedArea.
-    #    It makes another free list, used for various purposes.
-    ('secondary_free_list', llmemory.Address),
-    # -- The structure above is 2 words, which is a good value:
-    #    '(1022-2) % N' is zero or very small for various small N's,
-    #    i.e. there is not much wasted space.
-    )
-PAGE_PTR.TO.become(PAGE_HEADER)
-PAGE_NULL = lltype.nullptr(PAGE_HEADER)
-
-# ------------------------------------------------------------
-
-
-class StmGCSharedArea(object):
-    _alloc_flavor_ = 'raw'
-
-    def __init__(self, gc, page_size, small_request_threshold):
-        "NOT_RPYTHON"
-        self.gc = gc
-        self.page_size = page_size
-        self.small_request_threshold = small_request_threshold
-        #
-        # This array contains 'length' chained lists of pages.
-        # For each size N between WORD and 'small_request_threshold'
-        # (included), the corresponding chained list contains pages
-        # which store objects of size N.  This is only used for pages
-        # with low usage after a major collection; as soon as a page
-        # is used, or if its usage says high after a major collection,
-        # it belongs to the lists of StmGCThreadLocalAllocator.
-        length = small_request_threshold / WORD + 1
-        self.low_usage_pages = lltype.malloc(rffi.CArray(PAGE_PTR), length,
-                                             flavor='raw', zero=True,
-                                             immortal=True)
-        # ^^^ XXX not used so far
-        self.full_pages = lltype.malloc(rffi.CArray(PAGE_PTR), length,
-                                        flavor='raw', zero=True,
-                                        immortal=True)
-        self.nblocks_for_size = lltype.malloc(rffi.CArray(lltype.Signed),
-                                              length, flavor='raw', zero=True,
-                                              immortal=True)
-        self.hdrsize = llmemory.raw_malloc_usage(llmemory.sizeof(PAGE_HEADER))
-        for i in range(1, length):
-            self.nblocks_for_size[i] = (page_size - self.hdrsize) // (WORD * i)
-        assert self.nblocks_for_size[length-1] >= 1
-        self.length = length
-        #
-        # Counters for statistics
-        self.count_global_pages = 0
-        self.num_major_collects = 0
-        self.v_count_total_bytes = lltype.malloc(rffi.CArray(lltype.Unsigned),
-                                                 1, flavor='raw',
-                                                 immortal=True, zero=True)
-
-    def setup(self):
-        pass
-
-    def fetch_count_total_bytes(self):
-        return self.v_count_total_bytes[0]
-
-    def fetch_count_total_bytes_and_add(self, increment):
-        adr = rffi.cast(llmemory.Address, self.v_count_total_bytes)
-        return r_uint(atomic_ops.fetch_and_add(adr, increment))
-
-    def do_major_collection(self):
-        """Perform a major collection."""
-        # At this point all other threads should be blocked or running
-        # external C code or careful non-GC-using code, with all GC roots
-        # in their shadow stack.  Even if some nurseries are not empty
-        # we can still trace through them: a major collection does not
-        # move any object.  The point is only that after the "sweep" phase,
-        # we have identified all locations that are now free, and added
-        # them to the chained lists of StmGCSharedArea for reuse.
-        debug_start("gc-collect")
-        debug_print()
-        debug_print(".----------- Full collection ------------------")
-        debug_print("| used before collection:   ",
-                    self.fetch_count_total_bytes(), "bytes")
-        #
-        # Note that a major collection is non-moving.  The goal is only to
-        # find and free some of the objects allocated by the ArenaCollection.
-        # We first visit all objects and set the flag GCFLAG_VISITED on them.
-        self.objects_to_trace = self.AddressStack()
-        #
-        # The stacks...
-        self.collect_roots_from_stacks()
-        #
-        # The raw structures...
-        self.collect_from_raw_structures()
-        #
-        # The tldicts...
-        self.collect_roots_from_tldicts()
-        #
-        self.visit_all_objects()
-        #
-        self.num_major_collects += 1
-        debug_print("| used after collection:    ",
-                    self.fetch_count_total_bytes(), "bytes")
-        debug_print("| number of major collects: ",
-                    self.num_major_collects)
-        debug_print("`----------------------------------------------")
-        debug_stop("gc-collect")
-
-    def collect_roots_from_stacks(self):
-        self.gc.root_walker.walk_all_stack_roots(StmGCSharedArea._collect_stack_root,
-                                                 self)
-
-    def collect_from_raw_structures(self):
-        self.gc.root_walker.walk_current_nongc_roots(
-            StmGCSharedArea._collect_stack_root, self)
-
-    def _collect_stack_root(self, root):
-        self.objects_to_trace.append(root.address[0])
-
-    def collect_roots_from_tldicts(self):
-        CALLBACK = self.gc.stm_operations.CALLBACK_ENUM
-        llop.nop(lltype.Void, llhelper(CALLBACK,
-                                       StmGCSharedArea._stm_enum_external_callback))
-        # The previous line causes the _stm_enum_external_callback() function to be
-        # generated in the C source with a specific signature, where it
-        # can be called by the C code.
-        stmtls = self.gc.linked_list_stmtls
-        while stmtls is not None:
-            self.visit_all_objects()   # empty the list first
-            # for every stmtls:
-            self.gc.stm_operations.tldict_enum_external(stmtls.thread_descriptor)
-            stmtls = stmtls.linked_list_next
-
-    @staticmethod
-    def _stm_enum_external_callback(globalobj, localobj):
-        localhdr = self.gc.header(localobj)
-        ll_assert(localhdr.tid & GCFLAG_VISITED != 0,
-                  "[shared] in a root: missing GCFLAG_VISITED")
-        localhdr.tid &= ~GCFLAG_VISITED
-        self.objects_to_trace.append(localobj)
-        self.objects_to_trace.append(globalobj)
-
-    def visit_all_objects(self):
-        pending = self.objects_to_trace
-        while pending.non_empty():
-            obj = pending.pop()
-            self.visit(obj)
-
-    def visit(self, obj):
-        # 'obj' is a live object.  Check GCFLAG_VISITED to know if we
-        # have already seen it before.
-        hdr = self.gc.header(obj)
-        if hdr.tid & GCFLAG_VISITED:
-            return
-        #
-        # It's the first time.  We set the flag.
-        hdr.tid |= GCFLAG_VISITED
-        #
-        # Trace the content of the object and put all objects it references
-        # into the 'objects_to_trace' list.
-        self.gc.trace(obj, self._collect_ref_rec, None)
-
-    def _collect_ref_rec(self, root, ignored):
-        self.objects_to_trace.append(root.address[0])
-
-
-# ------------------------------------------------------------
-
-
-class StmGCThreadLocalAllocator(object):
-    """A thread-local allocator for the shared area.
-    This is an optimization only: it lets us use thread-local variables
-    to keep track of what we allocated.
-    """
-    _alloc_flavor_ = 'raw'
-
-    def __init__(self, sharedarea):
-        self.gc = sharedarea.gc
-        self.sharedarea = sharedarea
-        #
-        # The array 'pages_for_size' contains 'length' chained lists
-        # of pages currently managed by this thread.
-        # For each size N between WORD and 'small_request_threshold'
-        # (included), the corresponding chained list contains pages
-        # which store objects of size N.
-        length = sharedarea.length
-        self.pages_for_size = lltype.malloc(
-            rffi.CArray(PAGE_PTR), length, flavor='raw', zero=True,
-            track_allocation=False)
-        self.count_pages = 0    # for statistics
-        #
-        # This array contains 'length' chained lists of free locations.
-        self.free_loc_for_size = lltype.malloc(
-            rffi.CArray(llmemory.Address), length, flavor='raw', zero=True,
-            track_allocation=False)
-        #
-        if not we_are_translated():
-            self._seen_pages = set()
-
-    def _malloc_size_class(self, size_class):
-        """Malloc one object of the given size_class (== number of WORDs)."""
-        ll_assert(size_class > 0, "malloc_size_class: null or neg size_class")
-        ll_assert(size_class <= self.sharedarea.small_request_threshold,
-                  "malloc_size_class: too big")
-        #
-        # The result is simply 'free_loc_for_size[size_class]'
-        result = self.free_loc_for_size[size_class]
-        if not result:
-            result = self._allocate_new_page(size_class)
-        self.free_loc_for_size[size_class] = result.address[0]
-        llarena.arena_reset(result, llmemory.sizeof(llmemory.Address), 0)
-        return result
-    _malloc_size_class._always_inline_ = True
-
-    def _free_size_class(self, adr, size_class):
-        """Free a single location 'adr', which is of the given size_class."""
-        # just link 'adr' to the start of 'free_loc_for_size[size_class]'
-        adr = llarena.getfakearenaaddress(adr)
-        llarena.arena_reset(adr, size_class << WORD_POWER_2, 0)
-        llarena.arena_reserve(adr, llmemory.sizeof(llmemory.Address))
-        adr.address[0] = self.free_loc_for_size[size_class]
-        self.free_loc_for_size[size_class] = adr
-    _free_size_class._always_inline_ = True
-
-    def _allocate_new_page(self, size_class):
-        """Allocate and return a new page for the given size_class."""
-        #
-        sharedarea = self.sharedarea
-        result = llarena.arena_malloc(sharedarea.page_size, 0)
-        if not result:
-            fatalerror("FIXME: Out of memory! (should raise MemoryError)")
-            return NULL
-        if not we_are_translated():
-            self._seen_pages.add(result)
-        self.count_pages += 1
-        sharedarea.fetch_count_total_bytes_and_add(sharedarea.page_size)
-        #
-        # Initialize the fields of the resulting page
-        llarena.arena_reserve(result, llmemory.sizeof(PAGE_HEADER))
-        page = llmemory.cast_adr_to_ptr(result, PAGE_PTR)
-        page.nextpage = self.pages_for_size[size_class]
-        self.pages_for_size[size_class] = page
-        #
-        # Initialize the chained list in the page
-        head = result + llmemory.sizeof(PAGE_HEADER)
-        ll_assert(not self.free_loc_for_size[size_class],
-                  "free_loc_for_size is supposed to contain NULL here")
-        self.free_loc_for_size[size_class] = head
-        #
-        i = sharedarea.nblocks_for_size[size_class]
-        nsize = size_class << WORD_POWER_2
-        current = head
-        while True:
-            llarena.arena_reserve(current, llmemory.sizeof(llmemory.Address))
-            i -= 1
-            if i == 0:
-                break
-            next = current + nsize
-            current.address[0] = next
-            current = next
-        current.address[0] = llmemory.NULL
-        #
-        return head
-
-
-    def malloc_object_addr(self, totalsize):
-        """Malloc.  You should also call add_regular() later, or keep it in
-        some other data structure.  Note that it is not zero-filled."""
-        nsize = llmemory.raw_malloc_usage(totalsize)
-        if nsize <= self.sharedarea.small_request_threshold:
-            size_class = (nsize + WORD - 1) >> WORD_POWER_2
-            return self._malloc_size_class(size_class)
-        else:
-            count = llmemory.raw_malloc_usage(totalsize)
-            result = llarena.arena_malloc(count, 0)
-            # increment the counter *after* arena_malloc() returned
-            # successfully, otherwise we might increment it of a huge
-            # bogus number
-            self.sharedarea.fetch_count_total_bytes_and_add(count)
-            return result
-
-    def malloc_object(self, objsize):
-        totalsize = self.gc.gcheaderbuilder.size_gc_header + objsize
-        addr = self.malloc_object_addr(totalsize)
-        llarena.arena_reserve(addr, _dummy_size(totalsize))
-        return addr + self.gc.gcheaderbuilder.size_gc_header
-
-    def free_object(self, obj):
-        adr1 = obj - self.gc.gcheaderbuilder.size_gc_header
-        totalsize = (self.gc.gcheaderbuilder.size_gc_header +
-                     self.gc.get_size_incl_hash(obj))
-        if totalsize <= self.sharedarea.small_request_threshold:
-            size_class = (totalsize + WORD_POWER_2 - 1) >> WORD_POWER_2
-            self._free_size_class(adr1, size_class)
-        else:
-            # decrement the counter *before* we free the memory,
-            # otherwise there could in theory be a race condition that
-            # ends up overflowing the counter
-            self.sharedarea.fetch_count_total_bytes_and_add(-totalsize)
-            llarena.arena_free(llarena.getfakearenaaddress(adr1))
-
-    def gift_all_pages_to_shared_area(self):
-        """Send to the shared area all my pages.  For now we don't extract
-        the information about which locations are free or not; we just stick
-        them into 'full_pages' and leave it to the next global GC to figure
-        them out.
-        """
-        stmshared = self.sharedarea
-        stmshared.gc.acquire_global_lock()
-        i = stmshared.length - 1
-        while i >= 1:
-            lpage = self.pages_for_size[i]
-            if lpage:
-                gpage = stmshared.full_pages[i]
-                gpage_addr = llmemory.cast_ptr_to_adr(gpage)
-                lpage.secondary_free_list = gpage_addr
-                stmshared.full_pages[i] = lpage
-            i -= 1
-        stmshared.count_global_pages += self.count_pages
-        stmshared.gc.release_global_lock()
-
-    def delete(self):
-        self.gift_all_pages_to_shared_area()
-        lltype.free(self.free_loc_for_size, flavor='raw',
-                    track_allocation=False)
-        lltype.free(self.pages_for_size, flavor='raw',
-                    track_allocation=False)
-        free_non_gc_object(self)
-
-
-# ____________________________________________________________
-
-def _dummy_size(size):
-    if we_are_translated():
-        return size
-    if isinstance(size, int):
-        size = llmemory.sizeof(lltype.Char) * size
-    return size
diff --git a/rpython/memory/gc/stmtls.py b/rpython/memory/gc/stmtls.py
deleted file mode 100644
--- a/rpython/memory/gc/stmtls.py
+++ /dev/null
@@ -1,725 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi
-from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr, llhelper
-from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype
-from rpython.rlib.objectmodel import we_are_translated, free_non_gc_object
-from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib.debug import ll_assert, fatalerror
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
-
-from rpython.memory.gc.stmgc import WORD, NULL
-from rpython.memory.gc.stmgc import always_inline, dont_inline
-from rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED
-from rpython.memory.gc.stmgc import GCFLAG_LOCAL_COPY
-from rpython.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED
-from rpython.memory.gc.stmgc import GCFLAG_NOT_WRITTEN
-from rpython.memory.gc.stmgc import GCFLAG_HASH_FIELD, GCFLAG_WITH_HASH
-from rpython.memory.gc.stmgc import GCFLAG_PREBUILT_ORIGINAL
-from rpython.memory.gc.stmgc import hdr_revision, set_hdr_revision
-
-SIZE_OF_SIGNED = llmemory.sizeof(lltype.Signed)
-
-
-class StmGCTLS(object):
-    """The thread-local structure: we have one instance of these per thread,
-    including one for the main thread."""
-
-    _alloc_flavor_ = 'raw'
-
-    nontranslated_dict = {}
-
-    def __init__(self, gc, StmGCThreadLocalAllocator=None):
-        debug_start("gc-init")
-        self.gc = gc
-        self.stm_operations = self.gc.stm_operations
-        self.null_address_dict = self.gc.null_address_dict
-        self.AddressStack = self.gc.AddressStack
-        self.AddressDict = self.gc.AddressDict
-        if we_are_translated():
-            self.adr_of_stack_base = llop.gc_adr_of_root_stack_base(
-                llmemory.Address)
-            self.adr_of_stack_top  = llop.gc_adr_of_root_stack_top(
-                llmemory.Address)
-        #
-        # --- current position, or NULL when mallocs are forbidden
-        self.nursery_free = NULL
-        self.nursery_pending_clear = 0
-        # --- the start and size of the nursery belonging to this thread.
-        #     never changes.
-        self.nursery_size  = self.gc.nursery_size                    # fixed
-        self.nursery_start = self._alloc_nursery(self.nursery_size)  # fixed
-        self.nursery_stop  = self.nursery_start + self.nursery_size  # fixed
-        #self.extra_threshold = 0
-        #self.set_extra_threshold(self.gc.maximum_extra_threshold)
-        #
-        # --- current nursery top: usually equal to nursery_stop, except
-        #     when doing a major collection, when the thread that initiates
-        #     it changes all nursery_top of other threads
-        self.nursery_top  = self.nursery_stop
-        #
-        # --- a thread-local allocator for the shared area
-        if StmGCThreadLocalAllocator is None:
-            from rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
-        self.sharedarea_tls = StmGCThreadLocalAllocator(
-            self.gc.sharedarea)
-        # --- a chained list of regular local objects, linked via 'revision'.
-        self.regular_chained_list = NULL
-        # --- the LOCAL objects which are weakrefs.  They are also listed
-        #     in the appropriate place, like sharedarea_tls, if needed.
-        self.local_weakrefs = self.AddressStack()
-        #
-        # Support for id and identityhash: map nursery objects with
-        # GCFLAG_HAS_SHADOW to their future location after the next
-        # local collection.
-        self.nursery_objects_shadows = self.AddressDict()
-        #
-        self._register_with_C_code()
-        debug_stop("gc-init")
-
-    def delete(self):
-        self._cleanup_state()
-        self._unregister_with_C_code()
-        self.local_weakrefs.delete()
-        self.sharedarea_tls.delete()
-        self._free_nursery(self.nursery_start)
-        free_non_gc_object(self)
-
-    def _alloc_nursery(self, nursery_size):
-        nursery = llarena.arena_malloc(nursery_size, 2)
-        debug_print("nursery is at", nursery, "size", nursery_size)
-        if not nursery:
-            raise MemoryError("cannot allocate nursery")
-        return nursery
-
-    def _free_nursery(self, nursery):
-        llarena.arena_free(nursery)
-
-    def _register_with_C_code(self):
-        if we_are_translated():
-            tls = cast_instance_to_base_ptr(self)
-            tlsaddr = llmemory.cast_ptr_to_adr(tls)
-        else:
-            n = 10000 + len(StmGCTLS.nontranslated_dict)
-            tlsaddr = rffi.cast(llmemory.Address, n)
-            StmGCTLS.nontranslated_dict[n] = self
-        self.thread_descriptor = self.stm_operations.set_tls(tlsaddr)
-
-    def _unregister_with_C_code(self):
-        ll_assert(self.gc.get_tls() is self,
-                  "unregister_with_C_code: wrong thread")
-        self.stm_operations.del_tls()
-
-    @staticmethod
-    @always_inline
-    def cast_address_to_tls_object(tlsaddr):
-        if we_are_translated():
-            tls = llmemory.cast_adr_to_ptr(tlsaddr, base_ptr_lltype())
-            return cast_base_ptr_to_instance(StmGCTLS, tls)
-        else:
-            n = rffi.cast(lltype.Signed, tlsaddr)
-            return StmGCTLS.nontranslated_dict[n]
-
-    def _disable_mallocs(self):
-        ll_assert(bool(self.nursery_free), "disable_mallocs: already disabled")
-        self.nursery_pending_clear = self.nursery_free - self.nursery_start
-        self.nursery_free = NULL
-
-    # ----------
-    # set_extra_threshold support
-
-##    def set_extra_threshold(self, reserved_size):
-##        diff = reserved_size - self.extra_threshold
-##        if self.nursery_top != NULL:
-##            if diff > 0 and self.nursery_free + diff > self.nursery_top:
-##                self.local_collection()
-##            self.nursery_top -= diff
-##        self.nursery_size -= diff
-##        self.extra_threshold += diff
-
-    # ------------------------------------------------------------
-
-    def start_transaction(self):
-        """Start a transaction: performs any pending cleanups, and set
-        up a fresh state for allocating.  Called at the start of
-        each transaction, including at the start of a thread."""
-        # Note that the calls to start_transaction() and
-        # stop_transaction() are not balanced: if a
-        # transaction is aborted, the latter might never be called.
-        # Be ready here to clean up any state.
-        self._cleanup_state()
-        if self.nursery_free:
-            clear_size = self.nursery_free - self.nursery_start
-        else:
-            clear_size = self.nursery_pending_clear
-        self.nursery_pending_clear = 0
-        if clear_size > 0:
-            llarena.arena_reset(self.nursery_start, clear_size, 2)
-        self.nursery_free = self.nursery_start
-        # At this point, all visible objects are GLOBAL, but newly
-        # malloced objects will be LOCAL.
-        if self.gc.DEBUG:
-            self.check_all_global_objects(False)
-        #self.relocalize_from_stack()
-
-    def stop_transaction(self):
-        """Stop a transaction: do a local collection to empty the
-        nursery and track which objects are still alive now, and
-        then mark all these objects as global."""
-        self.local_collection(end_of_transaction=True)
-        self.gc.maybe_major_collection()
-        if not self.local_nursery_is_empty():
-            self.local_collection(end_of_transaction=True,
-                                  run_finalizers=False)
-        self._promote_locals_to_globals()
-        self._disable_mallocs()
-        if self.gc.DEBUG:
-            self.check_all_global_objects(True)
-
-    def local_nursery_is_empty(self):
-        ll_assert(bool(self.nursery_free),
-                  "local_nursery_is_empty: gc not running")
-        return self.nursery_free == self.nursery_start
-
-    # ------------------------------------------------------------
-
-    def local_collection(self, end_of_transaction=False, run_finalizers=True):
-        """Do a local collection.  This should be equivalent to a minor
-        collection only, but the GC is not generational so far, so it is
-        for now the same as a full collection --- but only on LOCAL
-        objects, not touching the GLOBAL objects.  More precisely, this
-        finds all LOCAL objects alive, moving them if necessary out of the
-        nursery.  This starts from the roots from the stack and the LOCAL
-        COPY objects.
-        """
-        #
-        debug_start("gc-local")
-        #
-        if end_of_transaction:
-            self.detect_flag_combination = GCFLAG_LOCAL_COPY | GCFLAG_VISITED
-        else:
-            self.detect_flag_combination = -1
-        #
-        # Move away the previous chained_list and start a new one.
-        previous_chained_list = self.regular_chained_list
-        self.regular_chained_list = NULL
-        #
-        # List of LOCAL objects pending a visit.  Note that no GLOBAL
-        # object can at any point contain a reference to a LOCAL object.
-        self.pending = self.AddressStack()
-        #
-        # First, find the roots that point to LOCAL objects.  All YOUNG
-        # (i.e. nursery) objects found are copied out of the nursery.
-        # All OLD objects found are flagged with GCFLAG_VISITED.
-        # At this point, the content of the objects is not modified;
-        # they are simply added to 'pending'.
-        self.collect_roots_from_stack(end_of_transaction)
-        #
-        # Find the roots that are living in raw structures.
-        self.collect_from_raw_structures()
-        #
-        # Find the roots in the THREADLOCALREF structure, and
-        # the other extra roots hold by C code
-        self.collect_from_threadlocalref_and_misc()
-        #
-        # Also find the roots that are the local copy of global objects.
-        self.collect_roots_from_tldict()
-        #
-        # Now repeatedly follow objects until 'pending' is empty.
-        self.collect_flush_pending()
-        #
-        # Walk the list of LOCAL weakrefs, and update it if necessary.
-        if self.local_weakrefs.non_empty():
-            self.update_local_weakrefs()
-        #
-        # Clear this mapping.
-        if self.nursery_objects_shadows.length() > 0:
-            self.nursery_objects_shadows.clear()
-        #
-        # Visit all previous OLD objects.  Free the ones that have not been
-        # visited above, and reset GCFLAG_VISITED on the others.
-        self.mass_free_old_local(previous_chained_list)
-        #
-        # Note that the last step guarantees the invariant that between
-        # collections, all the objects linked within 'self.sharedarea_tls'
-        # don't have GCFLAG_VISITED.  As the newly allocated nursery
-        # objects don't have it either, at the start of the next
-        # collection, the only LOCAL objects that have it are the ones
-        # in the C tldict, together with GCFLAG_LOCAL_COPY.
-        #
-        # All live nursery objects are out, and the rest dies.  Fill
-        # the whole nursery with zero and reset the current nursery pointer.
-        ll_assert(bool(self.nursery_free), "nursery_free is NULL")
-        size_used = self.nursery_free - self.nursery_start
-        llarena.arena_reset(self.nursery_start, size_used, 2)
-        self.nursery_free = self.nursery_start
-        #
-        debug_stop("gc-local")
-
-    # ------------------------------------------------------------
-
-    @always_inline
-    def allocate_bump_pointer(self, size):
-        free = self.nursery_free
-        top  = self.nursery_top
-        if (top - free) < llmemory.raw_malloc_usage(size):
-            free = self.allocate_object_of_size(size)
-        self.nursery_free = free + size
-        return free
-
-    @dont_inline
-    def allocate_object_of_size(self, size):
-        if not self.nursery_free:
-            fatalerror("malloc in a non-main thread but outside a transaction")
-        if llmemory.raw_malloc_usage(size) > self.nursery_size // 8 * 7:
-            fatalerror("XXX object too large to ever fit in the nursery")
-        #
-        self.local_collection(run_finalizers=True)
-        self.gc.maybe_major_collection()
-        #
-        # if we have now enough space, return it
-        free = self.nursery_free
-        top  = self.nursery_top
-        if (top - free) >= llmemory.raw_malloc_usage(size):
-            return free
-        #
-        # no, it might be because we ran finalizers or did a global GC.
-        # Try again without running the finalizers.
-        self.local_collection(run_finalizers=False)
-        #
-        # now we must have enough space
-        free = self.nursery_free
-        top  = self.nursery_top
-        if (top - free) >= llmemory.raw_malloc_usage(size):
-            return free
-        #
-        ll_assert(False,
-                  "local_collection(run_finalizers=0) didn't free enough mem")
-        return NULL
-
-    def is_in_nursery(self, addr):
-        ll_assert(llmemory.cast_adr_to_int(addr) & 1 == 0,
-                  "odd-valued (i.e. tagged) pointer unexpected here")
-        return self.nursery_start <= addr < self.nursery_stop
-
-    def fresh_new_weakref(self, obj):
-        self.local_weakrefs.append(obj)
-
-    # ------------------------------------------------------------
-
-    def _promote_locals_to_globals(self):
-        ll_assert(self.local_nursery_is_empty(), "nursery must be empty [1]")
-        #
-        # Promote all objects in 'regular_chained_list' to global.
-        # This is the "real" equivalent of _FakeReach() in et.c.
-        obj = self.regular_chained_list
-        self.regular_chained_list = NULL
-        #
-        while obj:
-            hdr = self.gc.header(obj)
-            obj = hdr_revision(hdr)
-            ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "already GLOBAL [1]")
-            ll_assert(hdr.tid & GCFLAG_VISITED == 0, "unexpected VISITED [1]")
-            ll_assert(hdr.tid & GCFLAG_LOCAL_COPY == 0,"already LOCAL_COPY [1]")
-            hdr.tid |= GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN
-            self._clear_revision_for_global_object(hdr)
-
-    def _clear_revision_for_global_object(self, hdr):
-        # Reset the 'revision' to initialize a newly global object.
-        # When translated with C code, we set it to 1.
-        # When non-translated, we reset it instead to '_uninitialized'
-        # to simulate the fact that the C code might change it.
-        if we_are_translated():
-            hdr.revision = r_uint(1)
-        else:
-            del hdr.revision
-
-    def _cleanup_state(self):
-        #if self.rawmalloced_objects:
-        #    xxx     # free the rawmalloced_objects still around
-
-        # free the old unused local objects still in the chained list
-        obj = self.regular_chained_list
-        self.regular_chained_list = NULL
-        while obj:
-            next = self.gc.obj_revision(obj)
-            self.sharedarea_tls.free_object(obj)
-            obj = next
-        # forget the local weakrefs.
-        self.local_weakrefs.clear()
-
-
-    def collect_roots_from_stack(self, end_of_transaction):
-##        if end_of_transaction:
-##            # in this mode, we flag the reference to local objects in order
-##            # to re-localize them when we later start the next transaction
-##            # using this section of the shadowstack
-##            self.gc.root_walker.walk_current_stack_roots(
-##                StmGCTLS._trace_drag_out_and_flag_local, self)
-##        else:
-            self.gc.root_walker.walk_current_stack_roots(
-                StmGCTLS._trace_drag_out1, self)
-
-##    def _trace_drag_out_and_flag_local(self, root):
-##        x = llmemory.cast_adr_to_int(root.address[0])
-##        if x & 2:
-##            root.address[0] -= 2
-##        old_tid = self.gc.header(root.address[0]).tid
-##        #
-##        self._trace_drag_out1(root)
-##        #
-##        # if root.address[0] used to point to a local object, then flag
-##        # this by adding the bit of value 2.
-##        if old_tid & GCFLAG_GLOBAL == 0:
-##            obj = root.address[0]
-##            x = llmemory.cast_adr_to_int(obj)
-##            ll_assert(x & 3 == 0, "flag_local: misaligned obj")
-##            obj = llarena.getfakearenaaddress(obj)
-##            root.address[0] = obj + 2
-
-##    def relocalize_from_stack(self):
-##        self.gc.root_walker.walk_current_stack_roots(
-##            StmGCTLS._relocalize_from_stack, self)
-
-##    def _relocalize_from_stack(self, root):
-##        x = llmemory.cast_adr_to_int(root.address[0])
-##        if x & 2:
-##            obj = root.address[0] - 2
-##            localobj = self.gc.stm_writebarrier(obj)
-##            root.address[0] = localobj
-
-    def collect_from_raw_structures(self):
-        self.gc.root_walker.walk_current_nongc_roots(
-            StmGCTLS._trace_drag_out1, self)
-
-    def collect_from_threadlocalref_and_misc(self):
-        if not we_are_translated():
-            return
-        i = llop.stm_threadlocalref_llcount(lltype.Signed)
-        while i > 0:
-            i -= 1
-            root = llop.stm_threadlocalref_lladdr(llmemory.Address, i)
-            if self.gc.points_to_valid_gc_object(root):
-                self._trace_drag_out(root, None)
-        i = llop.stm_extraref_llcount(lltype.Signed)
-        while i > 0:
-            i -= 1
-            root = llop.stm_extraref_lladdr(llmemory.Address, i)
-            if self.gc.points_to_valid_gc_object(root):
-                self._trace_drag_out(root, None)
-
-    def trace_and_drag_out_of_nursery(self, obj):
-        # This is called to fix the references inside 'obj', to ensure that
-        # they are not pointing to the nursery any more.  Any non-global,
-        # not-visited-yet object found is added to 'self.pending'.
-        # This is called on the LOCAL copy of
-        # the roots, and on the freshly OLD copy of all other reached LOCAL
-        # objects.  This only looks inside 'obj': it does not depend on or
-        # touch the flags of 'obj'.
-        self.gc.trace(obj, self._trace_drag_out, None)
-
-    def _trace_drag_out1(self, root):
-        self._trace_drag_out(root, None)
-
-    @always_inline
-    def categorize_object(self, obj, can_be_in_nursery):
-        """Return the current surviving state of the object:
-            0: not marked as surviving, so far
-            1: survives and does not move
-            2: survives, but moves to 'hdr.revision'
-        """
-        hdr = self.gc.header(obj)
-        flag_combination = hdr.tid & (GCFLAG_GLOBAL |
-                                      GCFLAG_LOCAL_COPY |
-                                      GCFLAG_VISITED)
-        if flag_combination == 0:
-            return 0    # not marked as surviving, so far
-
-        if flag_combination == self.detect_flag_combination:
-            # At a normal time, self.detect_flag_combination is -1
-            # and this case is never seen.  At end of transactions,
-            # detect_flag_combination is GCFLAG_LOCAL_COPY|GCFLAG_VISITED.
-            # This case is to force pointers to the LOCAL copy to be
-            # replaced with pointers to the GLOBAL copy.
-            return 2
-
-        if can_be_in_nursery and self.is_in_nursery(obj):
-            return 2
-        else:
-            return 1
-
-    def _trace_drag_out(self, root, ignored):
-        """Trace callback: 'root' is the address of some pointer.  If that
-        pointer points to a YOUNG object, allocate an OLD copy of it and
-        fix the pointer.  Also, add the object to the 'pending' stack, if
-        it was not done so far.
-        """
-        obj = root.address[0]
-        size = self.gc.get_size(obj)
-        # ^^^ moved here in order to crash early if 'obj' is invalid
-        hdr = self.gc.header(obj)
-        #
-        # If 'obj' is not in the nursery, we set GCFLAG_VISITED
-        if not self.is_in_nursery(obj):
-            # we ignore both GLOBAL objects and objects which have already
-            # been VISITED
-            cat = self.categorize_object(obj, can_be_in_nursery=False)
-            if cat == 0:
-                hdr.tid |= GCFLAG_VISITED
-                self.pending.append(obj)
-            elif cat == 2:
-                root.address[0] = hdr_revision(hdr)
-            return
-        #
-        # If 'obj' was already forwarded, change it to its forwarding address.
-        if hdr.tid & GCFLAG_VISITED:
-            root.address[0] = hdr_revision(hdr)
-            return
-        #
-        # First visit to 'obj': we must move this YOUNG obj out of the
-        # nursery.  This is the common case.  Allocate a new location
-        # for it outside the nursery.
-        newobj = self.duplicate_obj(obj, size, from_nursery=True)
-        #
-        # Note that references from 'obj' to other objects in the
-        # nursery are kept unchanged in this step: they are copied
-        # verbatim from 'obj' into 'newobj'.
-        #
-        # Register the object here, not before the memcopy() that would
-        # overwrite its 'revision' field
-        self._register_regular_obj(newobj)
-        #
-        # Set the YOUNG copy's GCFLAG_VISITED and set its revision to
-        # point to the OLD copy.
-        hdr.tid |= GCFLAG_VISITED
-        set_hdr_revision(hdr, newobj)
-        #
-        # Change the original pointer to this object.
-        root.address[0] = newobj
-        #
-        # Add the newobj to the list 'pending', because it can contain
-        # further pointers to other young objects.  We will fix such
-        # references to point to the copy of the young objects when we
-        # walk 'pending_list'.
-        self.pending.append(newobj)
-
-    def duplicate_obj(self, obj, objsize, from_nursery=False):
-        size_gc_header = self.gc.gcheaderbuilder.size_gc_header
-        totalsize_without_hash = size_gc_header + objsize
-        hdr = self.gc.header(obj)
-        #
-        make_hash_field = False
-        has_shadow = False
-        if from_nursery:
-            # 'obj' is a nursery object: check if it has a shadow.
-            # Note that if it does, the shadow doesn't have an extra
-            # hash field either, but will simply have the same flag
-            # combination, i.e. (GCFLAG_WITH_HASH & ~GCFLAG_HASH_FIELD).
-            # So future reads of the hash/id on this new object will
-            # continue to return the mangled address of this new
-            # object (which was merely the shadow until now).
-            ll_assert((hdr.tid & GCFLAG_HASH_FIELD) == 0,
-                      "nursery object with GCFLAG_HASH_FIELD")
-            if hdr.tid & GCFLAG_WITH_HASH:
-                has_shadow = True
-        else:
-            # From a non-nursery object: we need a hash field if
-            # any of the following two flags is already set on 'obj'
-            if hdr.tid & (GCFLAG_HASH_FIELD|GCFLAG_WITH_HASH):
-                make_hash_field = True
-        #
-        if has_shadow:
-            newobj = self.nursery_objects_shadows.get(obj)
-            ll_assert(newobj != NULL,
-                "duplicate_obj: GCFLAG_WITH_HASH but no shadow found")
-            newaddr = newobj - size_gc_header
-        else:
-            if make_hash_field:
-                newtotalsize = totalsize_without_hash + (
-                    llmemory.sizeof(lltype.Signed))
-            else:
-                newtotalsize = totalsize_without_hash
-            newaddr = self.sharedarea_tls.malloc_object_addr(newtotalsize)
-        #
-        # Initialize the copy by doing a memcpy of the bytes.
-        # The object header of localobj will then be fixed by the C code.
-        llarena.arena_reserve(newaddr, totalsize_without_hash)
-        llmemory.raw_memcopy(obj - size_gc_header,
-                             newaddr,
-                             totalsize_without_hash)
-        newobj = newaddr + size_gc_header
-        #
-        if make_hash_field:
-            # we have to write a value inside the new hash field
-            #
-            if hdr.tid & GCFLAG_HASH_FIELD:
-                #
-                if hdr.tid & GCFLAG_WITH_HASH:
-                    # 'obj' has already an explicit hash/id field, and is not
-                    # a prebuilt object at all.  Just propagate the content
-                    # of that field.
-                    hash = self.gc._get_hash_field(obj)
-                    #
-                elif hdr.tid & GCFLAG_PREBUILT_ORIGINAL:
-                    # 'obj' is an original prebuilt object with a hash field.
-                    # In the new hash field, store the original's address
-                    hash = llmemory.cast_adr_to_int(obj)
-                else:
-                    # 'obj' is already a modified copy of a prebuilt object.
-                    # Propagate the content of the field.
-                    hash = self.gc._get_hash_field(obj)
-                #
-            else:
-                # No previous field; store in the new field the old mangled
-                # address, and fix the new tid flags.
-                newhdr = self.gc.header(newobj)
-                ll_assert((newhdr.tid & GCFLAG_WITH_HASH) != 0, "gc bug!")
-                newhdr.tid |= GCFLAG_HASH_FIELD
-                hash = self.gc._get_mangled_address(obj)
-            #
-            hashaddr = llarena.getfakearenaaddress(newobj) + objsize
-            llarena.arena_reserve(hashaddr, SIZE_OF_SIGNED)
-            hashaddr.signed[0] = hash
-        #
-        return newobj
-
-    def _register_regular_obj(self, obj):
-        self.gc.set_obj_revision(obj, self.regular_chained_list)
-        self.regular_chained_list = obj
-
-    def collect_roots_from_tldict(self):
-        if not we_are_translated():
-            if not hasattr(self.stm_operations, 'tldict_enum'):
-                return
-        CALLBACK = self.stm_operations.CALLBACK_ENUM
-        llop.nop(lltype.Void, llhelper(CALLBACK, StmGCTLS._stm_enum_callback))
-        # The previous line causes the _stm_enum_callback() function to be
-        # generated in the C source with a specific signature, where it
-        # can be called by the C code.
-        self.stm_operations.tldict_enum()
-
-    @staticmethod
-    def _stm_enum_callback(tlsaddr, localobj):
-        self = StmGCTLS.cast_address_to_tls_object(tlsaddr)
-        localhdr = self.gc.header(localobj)
-        globalobj = hdr_revision(localhdr)
-        ll_assert(localhdr.tid & GCFLAG_GLOBAL == 0,
-                  "in a root: unexpected GCFLAG_GLOBAL")
-        ll_assert(localhdr.tid & GCFLAG_LOCAL_COPY != 0,
-                  "in a root: missing GCFLAG_LOCAL_COPY")
-        ll_assert(localhdr.tid & GCFLAG_VISITED != 0,
-                  "in a root: missing GCFLAG_VISITED")
-        globalhdr = self.gc.header(globalobj)
-        ll_assert(globalhdr.tid & GCFLAG_GLOBAL != 0,
-                  "in a root: GLOBAL: missing GCFLAG_GLOBAL")
-        ll_assert(globalhdr.tid & GCFLAG_POSSIBLY_OUTDATED != 0,
-                  "in a root: GLOBAL: missing GCFLAG_POSSIBLY_OUTDATED")
-        ll_assert(globalhdr.tid & GCFLAG_VISITED == 0,
-                  "in a root: GLOBAL: unexpected GCFLAG_VISITED")
-        TL = lltype.cast_primitive(lltype.Signed,
-                                   self.gc.get_type_id(localobj))
-        TG = lltype.cast_primitive(lltype.Signed,
-                                   self.gc.get_type_id(globalobj))
-        ll_assert(TL == TG, "in a root: type(LOCAL) != type(GLOBAL)")
-        #
-        self.trace_and_drag_out_of_nursery(localobj)
-
-    def collect_flush_pending(self):
-        # Follow the objects in the 'pending' stack and move the
-        # young objects they point to out of the nursery.
-        while self.pending.non_empty():
-            obj = self.pending.pop()
-            self.trace_and_drag_out_of_nursery(obj)
-        self.pending.delete()
-
-    def update_local_weakrefs(self):
-        old = self.local_weakrefs
-        new = self.AddressStack()
-        while old.non_empty():
-            obj = old.pop()
-            hdr = self.gc.header(obj)
-            ll_assert(hdr.tid & (GCFLAG_GLOBAL|GCFLAG_LOCAL_COPY) == 0,
-                      "local weakref: bad flags")
-            if hdr.tid & GCFLAG_VISITED == 0:
-                continue # weakref itself dies
-            #
-            if self.is_in_nursery(obj):
-                obj = hdr_revision(hdr)
-                #hdr = self.gc.header(obj) --- not needed any more
-            offset = self.gc.weakpointer_offset(self.gc.get_type_id(obj))
-            pointing_to = (obj + offset).address[0]
-            ll_assert(bool(pointing_to), "weakref to NULL in local_weakrefs")
-            cat = self.categorize_object(pointing_to, can_be_in_nursery=True)
-            if cat == 0:
-                # the weakref points to a dying object; no need to remember it
-                (obj + offset).address[0] = llmemory.NULL
-            else:
-                # the weakref points to an object that stays alive
-                if cat == 2:      # update the pointer if needed
-                    pointing_hdr = self.gc.header(pointing_to)
-                    (obj + offset).address[0] = hdr_revision(pointing_hdr)
-                new.append(obj)   # re-register in the new local_weakrefs list
-        #
-        self.local_weakrefs = new
-        old.delete()
-
-    def mass_free_old_local(self, obj):
-        while obj != NULL:
-            hdr = self.gc.header(obj)
-            next = hdr_revision(hdr)
-            if hdr.tid & GCFLAG_VISITED:
-                # survives: relink in the new chained list
-                hdr.tid -= GCFLAG_VISITED
-                self._register_regular_obj(obj)
-            else:
-                # dies
-                self.sharedarea_tls.free_object(obj)
-            #
-            obj = next
-
-    # ------------------------------------------------------------
-
-    def _debug_check_all_global_from_stack_1(self, root):
-        obj = root.address[0]
-##        if llmemory.cast_adr_to_int(obj) & 2:
-##            obj -= 2     # will be fixed by relocalize_from_stack()
-        self._debug_check_all_global_obj(obj)
-
-    def _debug_check_all_global(self, root, ignored):
-        obj = root.address[0]
-        self._debug_check_all_global_obj(obj)
-
-    def _debug_check_all_global_obj(self, obj):
-        if self.debug_seen.contains(obj):
-            return
-        hdr = self.gc.header(obj)
-        is_global = hdr.tid & GCFLAG_GLOBAL != 0
-        is_local_copy = hdr.tid & GCFLAG_LOCAL_COPY != 0
-        if not self._allow_local_copy:
-            ll_assert(is_global, "debug_check: missing GLOBAL")
-        if is_global:
-            ll_assert(not is_local_copy, "debug_check: unexpected LOCAL_COPY")
-        else:
-            ll_assert(is_local_copy, "debug_check: missing LOCAL_COPY")
-        ll_assert(hdr.tid & GCFLAG_VISITED == 0,
-                  "debug_check: unexpected VISITED")
-        self.gc.get_size(obj)      # extra checks
-        self.pending.append(obj)
-        self.debug_seen.setitem(obj, obj)
-
-    def check_all_global_objects(self, allow_local_copy):
-        self.pending = self.AddressStack()
-        self.debug_seen = self.AddressDict()
-        self._allow_local_copy = allow_local_copy
-        self.gc.root_walker.walk_current_stack_roots(
-            StmGCTLS._debug_check_all_global_from_stack_1, self)
-        while self.pending.non_empty():
-            obj = self.pending.pop()
-            offset = self.gc.weakpointer_offset(self.gc.get_type_id(obj))
-            if offset < 0:    # common case: not a weakref
-                self.gc.trace(obj, self._debug_check_all_global, None)
-            else:
-                if (obj + offset).address[0]:
-                    self._debug_check_all_global(obj + offset, None)
-        self.pending.delete()
-        self.debug_seen.delete()
diff --git a/rpython/memory/gc/test/test_stmgc.py b/rpython/memory/gc/test/test_stmgc.py
deleted file mode 100644
--- a/rpython/memory/gc/test/test_stmgc.py
+++ /dev/null
@@ -1,818 +0,0 @@
-import py
-from rpython.rlib.rarithmetic import r_uint
-from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
-from rpython.memory.gc.stmgc import StmGC, WORD, REV_INITIAL
-from rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_NOT_WRITTEN
-from rpython.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED
-from rpython.memory.gc.stmgc import GCFLAG_LOCAL_COPY, GCFLAG_VISITED
-from rpython.memory.gc.stmgc import GCFLAG_HASH_FIELD
-from rpython.memory.gc.stmgc import hdr_revision, set_hdr_revision
-from rpython.memory.support import mangle_hash
-
-
-S = lltype.GcStruct('S', ('a', lltype.Signed), ('b', lltype.Signed),
-                         ('c', lltype.Signed))
-ofs_a = llmemory.offsetof(S, 'a')
-
-SR = lltype.GcForwardReference()
-SR.become(lltype.GcStruct('SR', ('s1', lltype.Ptr(S)),
-                                ('sr2', lltype.Ptr(SR)),
-                                ('sr3', lltype.Ptr(SR))))
-
-WR = lltype.GcStruct('WeakRef', ('wadr', llmemory.Address))
-SWR = lltype.GcStruct('SWR', ('wr', lltype.Ptr(WR)))
-
-
-class FakeStmOperations:
-    # The point of this class is to make sure about the distinction between
-    # RPython code in the GC versus C code in translator/stm/src_stm.  This
-    # class contains a fake implementation of what should be in C.  So almost
-    # any use of 'self._gc' is wrong here: it's stmgc.py that should call
-    # et.c, and not the other way around.
-
-    CALLBACK_ENUM = lltype.Ptr(lltype.FuncType([llmemory.Address] * 3,
-                                               lltype.Void))
-    DUPLICATE = lltype.Ptr(lltype.FuncType([llmemory.Address],
-                                           llmemory.Address))
-
-    threadnum = 0          # 0 = main thread; 1,2,3... = transactional threads
-
-    def descriptor_init(self):
-        self._in_transaction = False
-
-    def begin_inevitable_transaction(self):
-        assert self._in_transaction is False
-        self._in_transaction = True
-
-    def commit_transaction(self):
-        assert self._in_transaction is True
-        self._in_transaction = False
-
-    def in_transaction(self):
-        return self._in_transaction
-
-    def set_tls(self, tls):
-        assert lltype.typeOf(tls) == llmemory.Address
-        assert tls
-        if self.threadnum == 0:
-            assert not hasattr(self, '_tls_dict')
-            self._tls_dict = {0: tls}
-            self._tldicts = {0: []}
-            self._transactional_copies = []
-        else:
-            self._tls_dict[self.threadnum] = tls
-            self._tldicts[self.threadnum] = []
-
-    def get_tls(self):
-        return self._tls_dict[self.threadnum]
-
-    def del_tls(self):
-        del self._tls_dict[self.threadnum]
-        del self._tldicts[self.threadnum]
-
-    def get_tldict(self):
-        return self._tldicts[self.threadnum]
-
-    def tldict_lookup(self, obj):
-        assert lltype.typeOf(obj) == llmemory.Address
-        assert obj
-        for key, value in self.get_tldict():
-            if obj == key:
-                return value
-        else:
-            return llmemory.NULL
-
-    def tldict_add(self, obj, localobj):
-        assert lltype.typeOf(obj) == llmemory.Address
-        assert lltype.typeOf(localobj) == llmemory.Address
-        assert obj
-        assert localobj
-        tldict = self.get_tldict()
-        for key, _ in tldict:
-            assert obj != key
-        tldict.append((obj, localobj))
-
-    def tldict_enum(self):
-        from rpython.memory.gc.stmtls import StmGCTLS
-        callback = StmGCTLS._stm_enum_callback
-        tls = self.get_tls()
-        for key, value in self.get_tldict():
-            assert (llmemory.cast_int_to_adr(self._gc.header(value).revision)
-                    == key)
-            callback(tls, value)
-
-
-def fake_get_size(obj):
-    TYPE = obj.ptr._TYPE.TO
-    if isinstance(TYPE, lltype.GcStruct):
-        return llmemory.sizeof(TYPE)
-    else:
-        assert 0
-
-def fake_trace(obj, callback, arg):
-    TYPE = obj.ptr._TYPE.TO
-    if TYPE == S:
-        ofslist = []     # no pointers in S
-    elif TYPE == SR:
-        ofslist = [llmemory.offsetof(SR, 's1'),
-                   llmemory.offsetof(SR, 'sr2'),
-                   llmemory.offsetof(SR, 'sr3')]
-    elif TYPE == WR:
-        ofslist = []
-    elif TYPE == SWR:
-        ofslist = [llmemory.offsetof(SWR, 'wr')]
-    else:
-        assert 0
-    for ofs in ofslist:
-        addr = obj + ofs
-        if addr.address[0]:
-            callback(addr, arg)
-
-def fake_weakpointer_offset(tid):
-    if tid == 124:
-        return llmemory.offsetof(WR, 'wadr')
-    else:
-        return -1
-
-class FakeRootWalker:
-    def walk_current_stack_roots(self, *args):
-        pass     # no stack roots in this test file
-    def walk_current_nongc_roots(self, *args):
-        pass     # no nongc roots in this test file
-
-
-class StmGCTests:
-    GCClass = StmGC
-
-    def setup_method(self, meth):
-        from pypy.config.pypyoption import get_pypy_config
-        config = get_pypy_config(translating=True).translation
-        self.gc = self.GCClass(config, FakeStmOperations(),
-                               translated_to_c=False)
-        self.gc.stm_operations._gc = self.gc
-        self.gc.DEBUG = True
-        self.gc.get_size = fake_get_size
-        self.gc.trace = fake_trace
-        self.gc.weakpointer_offset = fake_weakpointer_offset
-        self.gc.root_walker = FakeRootWalker()
-        self.gc.setup()
-
-    def teardown_method(self, meth):
-        if not hasattr(self, 'gc'):
-            return
-        for key in self.gc.stm_operations._tls_dict.keys():
-            if key != 0:
-                self.gc.stm_operations.threadnum = key
-                self.gc.teardown_thread()
-        self.gc.stm_operations.threadnum = 0
-
-    # ----------
-    # test helpers
-    def malloc(self, STRUCT, weakref=False, globl=False):
-        size = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT))
-        tid = lltype.cast_primitive(llgroup.HALFWORD, 123 + weakref)
-        if globl:
-            totalsize = self.gc.gcheaderbuilder.size_gc_header + size
-            adr1 = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize),
-                                        1)
-            llarena.arena_reserve(adr1, totalsize)
-            addr = adr1 + self.gc.gcheaderbuilder.size_gc_header
-            self.gc.header(addr).tid = self.gc.combine(
-                tid, GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN)
-            self.gc.header(addr).revision = REV_INITIAL
-            realobj = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(STRUCT))
-        else:
-            gcref = self.gc.malloc_fixedsize_clear(tid, size,
-                                                   contains_weakptr=weakref)
-            realobj = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), gcref)
-        return realobj
-    def settldict(self, globl, locl):
-        self.gc.stm_operations.tldict_add(llmemory.cast_ptr_to_adr(globl),
-                                          llmemory.cast_ptr_to_adr(locl))
-    def select_thread(self, threadnum):
-        self.gc.stm_operations.threadnum = threadnum
-        if threadnum not in self.gc.stm_operations._tls_dict:
-            self.gc.setup_thread()
-            self.gc.start_transaction()
-    def gcsize(self, S):
-        return (llmemory.raw_malloc_usage(llmemory.sizeof(self.gc.HDR)) +
-                llmemory.raw_malloc_usage(llmemory.sizeof(S)))
-    def checkflags(self, obj, globl='?', localcopy='?', version='?'):
-        if lltype.typeOf(obj) != llmemory.Address:
-            obj = llmemory.cast_ptr_to_adr(obj)
-        hdr = self.gc.header(obj)
-        if globl != '?':
-            assert (hdr.tid & GCFLAG_GLOBAL != 0) == globl
-        if localcopy != '?':
-            assert (hdr.tid & GCFLAG_LOCAL_COPY != 0) == localcopy
-        if version != '?':
-            assert hdr.version == version
-
-    def header(self, P):
-        if lltype.typeOf(P) != llmemory.Address:
-            P = llmemory.cast_ptr_to_adr(P)
-        return self.gc.header(P)
-
-    def set_hdr_revision(self, hdr, P):
-        if lltype.typeOf(P) != llmemory.Address:
-            P = llmemory.cast_ptr_to_adr(P)
-        set_hdr_revision(hdr, P)
-
-    def stm_readbarrier(self, P):
-        P = llmemory.cast_ptr_to_adr(P)
-        hdr = self.header(P)
-        if hdr.tid & GCFLAG_GLOBAL == 0:
-            # already a local object
-            R = P
-        else:
-            R = self.stm_latest_global_rev(P)
-            L = self.gc.stm_operations.tldict_lookup(R)
-            if hdr.tid & GCFLAG_POSSIBLY_OUTDATED == 0:
-                assert not L
-            elif L:
-                return L.ptr
-        return R.ptr
-
-    def stm_latest_global_rev(self, G):
-        hdr = self.gc.header(G)
-        assert hdr.tid & GCFLAG_GLOBAL != 0
-        while hdr.revision != REV_INITIAL:
-            xxx
-        return G
-
-    def stm_writebarrier(self, P):
-        P = llmemory.cast_ptr_to_adr(P)
-        hdr = self.header(P)
-        if hdr.tid & GCFLAG_NOT_WRITTEN == 0:
-            # already a local, written-to object
-            assert hdr.tid & GCFLAG_GLOBAL == 0
-            assert hdr.tid & GCFLAG_POSSIBLY_OUTDATED == 0
-            W = P
-        else:
-            # slow case of the write barrier
-            if hdr.tid & GCFLAG_GLOBAL == 0:
-                W = P
-                R = hdr_revision(hdr)
-            else:
-                R = P
-                W = self.stm_localize(R)
-            self.gc.header(W).tid &= ~GCFLAG_NOT_WRITTEN
-            self.gc.header(R).tid |= GCFLAG_POSSIBLY_OUTDATED
-        return W.ptr
-
-    def stm_localize(self, R):
-        L = self.gc.stm_operations.tldict_lookup(R)
-        if L:
-            assert self.gc.header(R).tid & GCFLAG_POSSIBLY_OUTDATED
-        else:
-            L = self.gc._stm_duplicate(R)
-            hdr = self.gc.header(L)
-            assert hdr.tid & GCFLAG_GLOBAL == 0
-            assert hdr.tid & GCFLAG_POSSIBLY_OUTDATED == 0
-            assert hdr.tid & GCFLAG_LOCAL_COPY
-            assert hdr.tid & GCFLAG_NOT_WRITTEN
-            set_hdr_revision(hdr, R)     # back-reference to the original
-            self.gc.stm_operations.tldict_add(R, L)
-            self.gc.stm_operations._transactional_copies.append((R.ptr, L.ptr))
-        return L
-
-    def do_local_collection(self):
-        self.gc.stop_transaction()
-        tldict = self.gc.stm_operations.get_tldict()
-        for obj, localobj in tldict:
-            hdr = self.gc.header(obj)
-            localhdr = self.gc.header(localobj)
-            assert localhdr.tid & GCFLAG_GLOBAL == 0
-            assert localhdr.tid & GCFLAG_LOCAL_COPY
-            assert localhdr.tid & GCFLAG_POSSIBLY_OUTDATED == 0
-            localhdr.tid |= GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN
-            localhdr.tid &= ~GCFLAG_LOCAL_COPY
-            assert localhdr.revision.adr == obj
-            localhdr.revision = r_uint(43)
-            assert hdr.tid & GCFLAG_GLOBAL
-            assert hdr.tid & GCFLAG_NOT_WRITTEN
-            assert hdr.tid & GCFLAG_POSSIBLY_OUTDATED
-            hdr.revision = llmemory.cast_adr_to_uint_symbolic(localobj)
-        del tldict[:]
-        self.gc.start_transaction()
-
-
-class TestBasic(StmGCTests):
-
-    def test_gc_creation_works(self):
-        pass
-
-    def test_allocate_bump_pointer(self):
-        tls = self.gc.get_tls()
-        a3 = tls.allocate_bump_pointer(3)
-        a4 = tls.allocate_bump_pointer(4)
-        a5 = tls.allocate_bump_pointer(5)
-        a6 = tls.allocate_bump_pointer(6)
-        assert a4 - a3 == 3
-        assert a5 - a4 == 4
-        assert a6 - a5 == 5
-
-    def test_malloc_fixedsize_clear(self):
-        gcref = self.gc.malloc_fixedsize_clear(123, llmemory.sizeof(S))
-        s = lltype.cast_opaque_ptr(lltype.Ptr(S), gcref)
-        assert s.a == 0
-        assert s.b == 0
-        gcref2 = self.gc.malloc_fixedsize_clear(123, llmemory.sizeof(S))
-        assert gcref2 != gcref
-
-    def test_malloc_main_vs_thread(self):
-        gcref = self.gc.malloc_fixedsize_clear(123, llmemory.sizeof(S))
-        obj = llmemory.cast_ptr_to_adr(gcref)
-        assert self.gc.header(obj).tid & GCFLAG_GLOBAL == 0
-        #
-        self.select_thread(1)
-        gcref = self.gc.malloc_fixedsize_clear(123, llmemory.sizeof(S))
-        obj = llmemory.cast_ptr_to_adr(gcref)
-        assert self.gc.header(obj).tid & GCFLAG_GLOBAL == 0
-

From noreply at buildbot.pypy.org  Fri Jun 28 20:55:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:21 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Remove this directory too.
Message-ID: <20130628185521.EE6B81C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65078:c312e40420da
Date: 2013-06-28 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/c312e40420da/

Log:	Remove this directory too.

diff too long, truncating to 2000 out of 2021 lines

diff --git a/rpython/translator/stm/src_stm/atomic_ops.h b/rpython/translator/stm/src_stm/atomic_ops.h
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/atomic_ops.h
+++ /dev/null
@@ -1,110 +0,0 @@
-#ifndef _SRCSTM_ATOMIC_OPS_
-#define _SRCSTM_ATOMIC_OPS_
-
-#include <assert.h>
-
-
-/* "compiler fence" for preventing reordering of loads/stores to
-   non-volatiles */
-#define CFENCE          asm volatile ("":::"memory")
-
-#if defined(__amd64__) || defined(__i386__)
-#  define smp_wmb()       CFENCE
-#  define smp_spinloop()  asm volatile ("pause")
-#elif defined(__powerpc__)
-#  define smp_wmb()       asm volatile ("lwsync":::"memory")
-#  define smp_spinloop()  /* fill me? */
-#else
-#  error "Define smp_wmb() for your architecture"
-#endif
-
-
-#ifdef __llvm__
-#  define HAS_SYNC_BOOL_COMPARE_AND_SWAP
-#  define HAS_SYNC_FETCH_AND_ADD
-#endif
-
-#ifdef __GNUC__
-#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
-#    define HAS_SYNC_BOOL_COMPARE_AND_SWAP
-#    define HAS_SYNC_FETCH_AND_ADD
-#  endif
-#endif
-
-
-#ifdef HAS_SYNC_BOOL_COMPARE_AND_SWAP
-#  define bool_cas __sync_bool_compare_and_swap
-#else
-/* x86 (32 bits and 64 bits) */
-static inline _Bool
-bool_cas(volatile Unsigned *ptr, Unsigned old, Unsigned _new)
-{
-    Unsigned prev;
-#if defined(__amd64__)
-    assert(sizeof(Unsigned) == 8);
-#elif defined(__i386__)
-    assert(sizeof(Unsigned) == 4);
-#else
-#   error "the custom version of bool_cas() is only for x86 or x86-64"
-#endif
-    asm volatile("lock;"
-#if defined(__amd64__)
-                 "cmpxchgq %1, %2;"
-#else
-                 "cmpxchgl %1, %2;"
-#endif
-                 : "=a"(prev)
-                 : "q"(_new), "m"(*ptr), "a"(old)
-                 : "memory");
-    return prev == old;
-}
-/* end */
-#endif
-
-#ifdef HAS_SYNC_FETCH_AND_ADD
-#  define fetch_and_add __sync_fetch_and_add
-#else
-/* x86 (32 bits and 64 bits) */
-static inline Unsigned
-fetch_and_add(volatile Unsigned *ptr, Unsigned value)
-{
-    Unsigned prev;
-#if defined(__amd64__)
-    assert(sizeof(Unsigned) == 8);
-#elif defined(__i386__)
-    assert(sizeof(Unsigned) == 4);
-#else
-#   error "the custom version of fetch_and_add() is only for x86 or x86-64"
-#endif
-    asm volatile("lock;"
-#if defined(__amd64__)
-                 "xaddq %1, %2;"
-#else
-                 "xaddl %1, %2;"
-#endif
-                 : "=r"(prev)
-                 : "0"(value), "m"(*ptr)
-                 : "memory");
-    return prev;
-}
-/* end */
-#endif
-
-
-static inline void spinloop(void)
-{
-  smp_spinloop();
-  /* use "memory" here to make sure that gcc will reload the
-     relevant data from memory after the spinloop */
-  CFENCE;
-}
-
-
-#define stm_lock_acquire(lock)                                          \
-     do { while (!bool_cas(&(lock), 0, 1)) spinloop(); } while (0)
-
-#define stm_lock_release(lock)                  \
-     (lock) = 0;
-
-
-#endif  /* _SRCSTM_ATOMIC_OPS_ */
diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/et.c
+++ /dev/null
@@ -1,957 +0,0 @@
-/* -*- c-basic-offset: 2 -*- */
-
-/* XXX assumes that time never wraps around (in a 'long'), which may be
- * correct on 64-bit machines but not on 32-bit machines if the process
- * runs for long enough.
- */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE
-#endif
-#ifndef _XOPEN_SOURCE
-# define _XOPEN_SOURCE 500
-#endif
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#include <pthread.h>
-#include <time.h>
-
-#ifndef RPY_STM
-/* for tests, a few custom defines */
-#define RPY_STM_DEBUG_PRINT     1
-#define PYPY_DEBUG_START(s)     fprintf(stderr, "start: %s\n", s)
-#define PYPY_DEBUG_STOP(s)      fprintf(stderr, " stop: %s\n", s)
-#define PYPY_HAVE_DEBUG_PRINTS  1
-#define PYPY_DEBUG_FILE         stderr
-#else
-#define RPY_STM_DEBUG_PRINT     1    /* for now */
-#endif
-
-#include "et.h"
-
-/************************************************************/
-
-typedef Unsigned revision_t;
-#define LOCKED  ((revision_t)-0x10000)
-
-#include "atomic_ops.h"
-#include "lists.c"
-
-/************************************************************/
-
-#define ABORT_REASONS 5
-#define SPINLOOP_REASONS 4
-
-struct tx_descriptor {
-  jmp_buf *setjmp_buf;
-  revision_t start_time;
-  revision_t my_lock;
-  long atomic;   /* 0 = not atomic, > 0 atomic */
-  unsigned long count_reads;
-  unsigned long reads_size_limit;        /* see should_break_tr. */
-  unsigned long reads_size_limit_nonatomic;
-  int active;    /* 0 = inactive, 1 = regular, 2 = inevitable */
-  int readonly_updates;
-  struct timespec start_real_time;
-  unsigned int num_commits;
-  unsigned int num_aborts[ABORT_REASONS];
-  unsigned int num_spinloops[SPINLOOP_REASONS];
-  struct GcPtrList list_of_read_objects;
-  struct GcPtrList gcroots;
-  struct G2L global_to_local;
-  struct GcPtrList undolog;
-  struct GcPtrList abortinfo;
-  char *longest_abort_info;
-  long long longest_abort_info_time;
-  struct FXCache recent_reads_cache;
-};
-
-/* 'global_cur_time' is normally a multiple of 2, except when we turn
-   a transaction inevitable: we then add 1 to it. */
-static volatile revision_t global_cur_time = 2;
-static volatile revision_t next_locked_value = LOCKED + 3;   /* always odd */
-static __thread struct tx_descriptor *thread_descriptor = NULL;
-
-/* a multi-reader, single-writer lock: transactions normally take a reader
-   lock, so don't conflict with each other; when we need to do a global GC,
-   we take a writer lock to "stop the world".  Note the initializer here,
-   which should give the correct priority for reached_safe_point(). */
-static pthread_rwlock_t rwlock_in_transaction =
-         PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP;
-
-/************************************************************/
-
-static void ValidateNow(struct tx_descriptor *);
-static void CancelLocks(struct tx_descriptor *d);
-static void AbortTransaction(int num);
-static void SpinLoop(int num);
-static gcptr Localize(struct tx_descriptor *d, gcptr R);
-
-static _Bool is_inevitable(struct tx_descriptor *d)
-{
-  /* Assert that we are running a transaction.
-   *      Returns True if this transaction is inevitable. */
-  assert(d->active == 1 + !d->setjmp_buf);
-  return d->active == 2;
-}
-
-static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
-#define inev_mutex_acquire()  pthread_mutex_lock(&mutex_inevitable)
-#define inev_mutex_release()  pthread_mutex_unlock(&mutex_inevitable)
-
-/************************************************************/
-
-static inline void PossiblyUpdateChain(
-        struct tx_descriptor *d,
-        gcptr G, gcptr R, gcptr R_Container, size_t offset)
-{
-  if (R != G && --d->readonly_updates < 0)
-    {
-      volatile revision_t *vp;
-      d->readonly_updates = 148;   /* XXX tweak */
-      // compress the chain one step (cannot compress the whole chain!)
-      // avoid emitting a pointless "write", too.
-      vp = (volatile revision_t *)&G->h_revision;
-      if (*vp != (revision_t)R)
-        *vp = (revision_t)R;
-      // update the original field
-      if (R_Container != NULL)
-        {
-          gcptr *ref = (gcptr *)(((char *)R_Container) + offset);
-          *ref = R;
-        }
-    }
-}
-
-static gcptr LatestGlobalRevision(struct tx_descriptor *d, gcptr G,
-                                  gcptr R_Container, size_t offset)
-{
-  gcptr R = G;
-  revision_t v;
-  volatile revision_t *vp;
- retry:
-  while (1)
-    {
-      vp = (volatile revision_t *)&R->h_revision;
-      v = *vp;
-      if (v & 1)  // "is not a pointer", i.e.
-        break;    //      "doesn't have a more recent revision"
-      R = (gcptr)v;
-    }
-  if (__builtin_expect(v > d->start_time, 0))   // object too recent?
-    {
-      if (v >= LOCKED)
-        {
-          SpinLoop(1);     // spinloop until it is no longer LOCKED
-          goto retry;
-        }
-      ValidateNow(d);                  // try to move start_time forward
-      goto retry;                      // restart searching from R
-    }
-  PossiblyUpdateChain(d, G, R, R_Container, offset);
-  return R;
-}
-
-static inline gcptr AddInReadSet(struct tx_descriptor *d, gcptr R)
-{
-  d->count_reads++;
-  if (!fxcache_add(&d->recent_reads_cache, R)) {
-      /* not in the cache: it may be the first time we see it,
-       * so insert it into the list */
-      gcptrlist_insert(&d->list_of_read_objects, R);
-  }
-      //      break;
-
-      //  case 2:
-      /* already in the cache, and FX_THRESHOLD reached */
-      //      return Localize(d, R);
-      //  }
-  return R;
-}
-
-static inline gcptr _direct_read_barrier(gcptr G, gcptr R_Container,
-                                         size_t offset)
-{
-  gcptr R;
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d->active);
-  if (!(G->h_tid & GCFLAG_POSSIBLY_OUTDATED))
-    {
-      R = G;
-    }
-  else
-    {
-      R = LatestGlobalRevision(d, G, R_Container, offset);
-      if (R->h_tid & GCFLAG_POSSIBLY_OUTDATED)
-        {
-          wlog_t *entry;
-          gcptr L;
-          G2L_FIND(d->global_to_local, R, entry, goto not_found);
-          L = entry->val;
-          assert(L->h_revision == (revision_t)R);
-          if (R_Container && !(R_Container->h_tid & GCFLAG_GLOBAL))
-            {    /* R_Container is a local object */
-              gcptr *ref = (gcptr *)(((char *)R_Container) + offset);
-              *ref = L;   /* fix in-place */
-            }
-          return L;
-
-        not_found:;
-        }
-    }
-  R = AddInReadSet(d, R);
-  return R;
-}
-
-void *stm_DirectReadBarrier(void *G1)
-{
-  return _direct_read_barrier((gcptr)G1, NULL, 0);
-}
-
-void *stm_DirectReadBarrierFromR(void *G1, void *R_Container1, size_t offset)
-{
-  return _direct_read_barrier((gcptr)G1, (gcptr)R_Container1, offset);
-}
-
-void *stm_RepeatReadBarrier(void *O1)
-{
-  // LatestGlobalRevision(O) would either return O or abort
-  // the whole transaction, so omitting it is not wrong
-  struct tx_descriptor *d = thread_descriptor;
-  gcptr O = (gcptr)O1;
-  gcptr L;
-  wlog_t *entry;
-  G2L_FIND(d->global_to_local, O, entry, return O);
-  L = entry->val;
-  assert(L->h_revision == (revision_t)O);
-  return L;
-}
-
-#if 0
-gcptr _NonTransactionalReadBarrier(gcptr P)
-{
-  /* testing only: use this outside transactions to check the state */
-  revision_t v;
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d == NULL || !d->active);
-
-  assert(P->h_tid & GCFLAG_GLOBAL);
-
-  while (!((v = P->h_revision) & 1))   // "is a pointer", i.e.
-    {                                  //   "has a more recent revision"
-      assert(P->h_tid & GCFLAG_POSSIBLY_OUTDATED);
-      fprintf(stderr, "[%p->%p]\n", P, (gcptr)v);
-      P = (gcptr)v;
-    }
-  if (P->h_tid & GCFLAG_POSSIBLY_OUTDATED)
-    fprintf(stderr, "[---%p possibly outdated---]\n", P);
-  return P;
-}
-#endif
-
-static gcptr Localize(struct tx_descriptor *d, gcptr R)
-{
-  wlog_t *entry;
-  gcptr L;
-  G2L_FIND(d->global_to_local, R, entry, goto not_found);
-  L = entry->val;
-  assert(L->h_revision == (revision_t)R);
-  return L;
-
- not_found:
-  L = pypy_g__stm_duplicate(R);
-  assert(!(L->h_tid & GCFLAG_GLOBAL));   /* must be removed by stm_duplicate */
-  assert(!(L->h_tid & GCFLAG_POSSIBLY_OUTDATED));    /* must be removed by ^ */
-  assert(L->h_tid & GCFLAG_LOCAL_COPY);    /* must be added by stm_duplicate */
-  assert(L->h_tid & GCFLAG_NOT_WRITTEN); /* must not be set in the 1st place */
-  L->h_revision = (revision_t)R;     /* back-reference to the original */
-  g2l_insert(&d->global_to_local, R, L);
-  d->count_reads++;
-  gcptrlist_insert(&d->list_of_read_objects, R);
-  return L;
-}
-
-void *stm_WriteBarrier(void *P1)
-{
-  gcptr P = (gcptr)P1;
-  gcptr R, W;
-  if (!(P->h_tid & GCFLAG_GLOBAL))
-    {
-      W = P;
-      R = (gcptr)W->h_revision;
-    }
-  else
-    {
-      struct tx_descriptor *d = thread_descriptor;
-      assert(d->active);
-      if (P->h_tid & GCFLAG_POSSIBLY_OUTDATED)
-        R = LatestGlobalRevision(d, P, NULL, 0);
-      else
-        R = P;
-      W = Localize(d, R);
-    }
-  W->h_tid &= ~GCFLAG_NOT_WRITTEN;
-  R->h_tid |= GCFLAG_POSSIBLY_OUTDATED;
-  return W;
-}
-
-void *stm_WriteBarrierFromReady(void *R1)
-{
-  gcptr R = (gcptr)R1;
-  gcptr W;
-  if (!(R->h_tid & GCFLAG_GLOBAL))
-    {
-      W = R;
-      R = (gcptr)W->h_revision;
-    }
-  else
-    {
-      struct tx_descriptor *d = thread_descriptor;
-      assert(d->active);
-      W = Localize(d, R);
-    }
-  W->h_tid &= ~GCFLAG_NOT_WRITTEN;
-  R->h_tid |= GCFLAG_POSSIBLY_OUTDATED;
-  return W;
-}
-
-/************************************************************/
-
-static revision_t GetGlobalCurTime(struct tx_descriptor *d)
-{
-  assert(!is_inevitable(d));    // must not be myself inevitable
-  return global_cur_time & ~1;
-}
-
-static _Bool ValidateDuringTransaction(struct tx_descriptor *d,
-                                       _Bool during_commit)
-{
-  long i, size = d->list_of_read_objects.size;
-  gcptr *items = d->list_of_read_objects.items;
-
-  for (i=0; i<size; i++)
-    {
-      gcptr R = items[i];
-      revision_t v;
-      volatile revision_t *vp;
-    retry:
-      vp = (volatile revision_t *)&R->h_revision;
-      v = *vp;
-      if (!(v & 1))               // "is a pointer", i.e.
-        return 0;                 //   "has a more recent revision"
-      if (v >= LOCKED)            // locked
-        {
-          if (!during_commit)
-            {
-              assert(v != d->my_lock);    // we don't hold any lock
-              SpinLoop(3);
-              goto retry;
-            }
-          else
-            {
-              if (v != d->my_lock)         // not locked by me: conflict
-                return 0;
-            }
-        }
-    }
-  return 1;
-}
-
-static void ValidateNow(struct tx_descriptor *d)
-{
-  d->start_time = GetGlobalCurTime(d);   // copy from the global time
-  if (!ValidateDuringTransaction(d, 0))
-    AbortTransaction(1);
-}
-
-/************************************************************/
-
-static void SpinLoop(int num)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d->active);
-  assert(num < SPINLOOP_REASONS);
-  d->num_spinloops[num]++;
-  spinloop();
-}
-
-size_t _stm_decode_abort_info(struct tx_descriptor *d, long long elapsed_time,
-                              int abort_reason, char *output);
-
-static void AbortTransaction(int num)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  unsigned long limit;
-  size_t size;
-  struct timespec now;
-  long long elapsed_time;
-  int err;
-
-  assert(d->active);
-  assert(!is_inevitable(d));
-  assert(num < ABORT_REASONS);
-  d->num_aborts[num]++;
-
-  CancelLocks(d);
-
-  /* compute the elapsed time */
-  if (d->start_real_time.tv_nsec != -1 &&
-      clock_gettime(CLOCK_MONOTONIC, &now) >= 0) {
-    elapsed_time = now.tv_sec - d->start_real_time.tv_sec;
-    elapsed_time *= 1000000000;
-    elapsed_time += now.tv_nsec - d->start_real_time.tv_nsec;
-    if (elapsed_time < 1)
-      elapsed_time = 1;
-  }
-  else {
-    elapsed_time = 1;
-  }
-
-  if (elapsed_time >= d->longest_abort_info_time)
-    {
-      /* decode the 'abortinfo' and produce a human-readable summary in
-         the string 'longest_abort_info' */
-      size = _stm_decode_abort_info(d, elapsed_time, num, NULL);
-      free(d->longest_abort_info);
-      d->longest_abort_info = malloc(size);
-      if (d->longest_abort_info == NULL)
-        d->longest_abort_info_time = 0;   /* out of memory! */
-      else
-        {
-          if (_stm_decode_abort_info(d, elapsed_time,
-                                     num, d->longest_abort_info) != size)
-            {
-              fprintf(stderr,
-                      "during stm abort: object mutated unexpectedly\n");
-              abort();
-            }
-          d->longest_abort_info_time = elapsed_time;
-        }
-    }
-
-  /* run the undo log in reverse order, cancelling the values set by
-     stm_ThreadLocalRef_LLSet(). */
-  if (d->undolog.size > 0) {
-      gcptr *item = d->undolog.items;
-      long i;
-      for (i=d->undolog.size; i>=0; i-=2) {
-          void **addr = (void **)(item[i-2]);
-          void *oldvalue = (void *)(item[i-1]);
-          *addr = oldvalue;
-      }
-  }
-
-  /* upon abort, set the reads size limit to 94% of how much was read
-     so far.  This should ensure that, assuming the retry does the same
-     thing, it will commit just before it reaches the conflicting point.
-     Note that we should never *increase* the read length limit here. */
-  limit = d->count_reads;
-  if (limit > d->reads_size_limit_nonatomic) {  /* can occur if atomic */
-      limit = d->reads_size_limit_nonatomic;
-  }
-  if (limit > 0) {
-      limit -= (limit >> 4);
-      d->reads_size_limit_nonatomic = limit;
-  }
-
-  gcptrlist_clear(&d->list_of_read_objects);
-  gcptrlist_clear(&d->gcroots);
-  g2l_clear(&d->global_to_local);
-
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_START("stm-abort");
-  if (PYPY_HAVE_DEBUG_PRINTS)
-      fprintf(PYPY_DEBUG_FILE, "thread %lx aborting %d\n",
-                               (long)d->my_lock, num);
-  PYPY_DEBUG_STOP("stm-abort");
-#endif
-
-  // notifies the CPU that we're potentially in a spin loop
-  SpinLoop(0);
-  // jump back to the setjmp_buf (this call does not return)
-  d->active = 0;
-  err = pthread_rwlock_unlock(&rwlock_in_transaction);
-  assert(err == 0);
-  longjmp(*d->setjmp_buf, 1);
-}
-
-/************************************************************/
-
-static void update_reads_size_limit(struct tx_descriptor *d)
-{
-  /* 'reads_size_limit' is set to ULONG_MAX if we are atomic; else
-     we copy the value from reads_size_limit_nonatomic. */
-  d->reads_size_limit = d->atomic ? ULONG_MAX : d->reads_size_limit_nonatomic;
-}
-
-static void init_transaction(struct tx_descriptor *d)
-{
-  int err;
-  assert(d->active == 0);
-  err = pthread_rwlock_rdlock(&rwlock_in_transaction);
-  assert(err == 0);
-
-  if (clock_gettime(CLOCK_MONOTONIC, &d->start_real_time) < 0) {
-    d->start_real_time.tv_nsec = -1;
-  }
-  assert(d->list_of_read_objects.size == 0);
-  assert(d->gcroots.size == 0);
-  assert(!g2l_any_entry(&d->global_to_local));
-  d->count_reads = 1;
-  fxcache_clear(&d->recent_reads_cache);
-  gcptrlist_clear(&d->undolog);
-  gcptrlist_clear(&d->abortinfo);
-}
-
-void BeginTransaction(jmp_buf* buf)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  init_transaction(d);
-  d->active = 1;
-  d->setjmp_buf = buf;
-  d->start_time = GetGlobalCurTime(d);
-  update_reads_size_limit(d);
-}
-
-static void AcquireLocks(struct tx_descriptor *d)
-{
-  revision_t my_lock = d->my_lock;
-  gcptr *item = d->gcroots.items;
-  // gcroots should be sorted in some deterministic order by construction
-
-  while (item[0] != NULL)
-    {
-      gcptr R = (gcptr)item[0]->h_revision;
-      revision_t v;
-      volatile revision_t *vp;
-    retry:
-      vp = (volatile revision_t *)&R->h_revision;
-      v = *vp;
-      if (!(v & 1))            // "is a pointer", i.e.
-        AbortTransaction(0);   //   "has a more recent revision"
-      if (v >= LOCKED)         // already locked by someone else
-        {
-          // we can always spinloop here: deadlocks should be impossible,
-          // because FindRootsForLocalCollect's G2L_LOOP_FORWARD should
-          // ensure a consistent ordering of the R's.
-          SpinLoop(2);
-          goto retry;
-        }
-      if (!bool_cas(vp, v, my_lock))
-        goto retry;
-
-      item[1] = (gcptr)v;
-      item += 2;
-    }
-}
-
-static void CancelLocks(struct tx_descriptor *d)
-{
-  long i, lastitem = d->gcroots.size - 2;
-  gcptr *items = d->gcroots.items;
-  for (i=0; i<=lastitem; i+=2)
-    {
-      gcptr R = (gcptr)items[i]->h_revision;
-      revision_t v = (revision_t)items[i+1];
-      if (v == 0)
-        break;
-      R->h_revision = v;
-      // if we're going to retry later, and abort,
-      // then we must not re-cancel the same entries
-      items[i+1] = (gcptr)0;
-    }
-}
-
-static void UpdateChainHeads(struct tx_descriptor *d, revision_t cur_time)
-{
-  gcptr *item, *itemstart = d->gcroots.items;
-  revision_t new_revision = cur_time + 1;     // make an odd number
-  assert(new_revision & 1);
-
-  for (item = itemstart; item[0] != NULL; item += 2)
-    {
-      gcptr L = item[0];
-      assert((L->h_tid & (GCFLAG_GLOBAL |
-                          GCFLAG_NOT_WRITTEN |
-                          GCFLAG_POSSIBLY_OUTDATED)) ==
-              (GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN));
-      item[1] = (gcptr)L->h_revision;    /* old value: pointer to R */
-      L->h_revision = new_revision;
-    }
-  smp_wmb();
-  for (item = itemstart; item[0] != NULL; item += 2)
-    {
-      gcptr L = item[0];
-      gcptr R = item[1];
-      assert((R->h_tid & (GCFLAG_GLOBAL |
-                          GCFLAG_NOT_WRITTEN |
-                          GCFLAG_POSSIBLY_OUTDATED)) ==
-              (GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN | GCFLAG_POSSIBLY_OUTDATED));
-      R->h_revision = (revision_t)L;
-    }
-}
-
-static void FindRootsForLocalCollect(struct tx_descriptor *d)
-{
-  wlog_t *item;
-  assert(d->gcroots.size == 0);
-
-  G2L_LOOP_FORWARD(d->global_to_local, item)
-    {
-      gcptr R = item->addr;
-      gcptr L = item->val;
-      assert(L->h_revision == (revision_t)R);
-      assert((L->h_tid & GCFLAG_GLOBAL) == 0);
-      assert(L->h_tid & GCFLAG_LOCAL_COPY);
-      assert((L->h_tid & GCFLAG_POSSIBLY_OUTDATED) == 0);
-
-      /* GCFLAG_VISITED is only used by stmgc, but it's convenient to
-       * remove that flag from here */
-      L->h_tid &= ~(GCFLAG_VISITED | GCFLAG_LOCAL_COPY);
-      if (L->h_tid & GCFLAG_NOT_WRITTEN)
-        {
-          L->h_tid |= GCFLAG_GLOBAL | GCFLAG_POSSIBLY_OUTDATED;
-        }
-      else
-        {
-          L->h_tid |= GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN;
-          gcptrlist_insert2(&d->gcroots, L, (gcptr)0);
-        }
-
-    } G2L_LOOP_END;
-  gcptrlist_insert(&d->gcroots, NULL);
-  g2l_clear(&d->global_to_local);
-}
-
-#if 0
-int _FakeReach(gcptr P)
-{
-  if (P->h_tid & GCFLAG_GLOBAL)
-    return 0;
-  P->h_tid |= GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN;
-  if ((P->h_tid & GCFLAG_LOCAL_COPY) == 0)
-    P->h_revision = 1;
-  else
-    P->h_tid &= ~GCFLAG_LOCAL_COPY;
-  return 1;
-}
-#endif
-
-void CommitTransaction(void)
-{
-  revision_t cur_time;
-  struct tx_descriptor *d = thread_descriptor;
-  int err;
-  assert(d->active != 0);
-
-  FindRootsForLocalCollect(d);
-  AcquireLocks(d);
-
-  if (is_inevitable(d))
-    {
-      // no-one else can have changed global_cur_time if I'm inevitable
-      cur_time = d->start_time;
-      if (!bool_cas(&global_cur_time, cur_time + 1, cur_time + 2))
-        {
-          assert(!"global_cur_time modified even though we are inev.");
-          abort();
-        }
-      inev_mutex_release();
-    }
-  else
-    {
-      while (1)
-        {
-          cur_time = global_cur_time;
-          if (cur_time & 1)
-            {                    // there is another inevitable transaction
-              CancelLocks(d);
-              inev_mutex_acquire();   // wait until released
-              inev_mutex_release();
-              AcquireLocks(d);
-              continue;
-            }
-          if (bool_cas(&global_cur_time, cur_time, cur_time + 2))
-            break;
-        }
-      // validate (but skip validation if nobody else committed)
-      if (cur_time != d->start_time)
-        if (!ValidateDuringTransaction(d, 1))
-          AbortTransaction(2);
-    }
-  /* we cannot abort any more from here */
-  d->setjmp_buf = NULL;
-  gcptrlist_clear(&d->list_of_read_objects);
-
-  UpdateChainHeads(d, cur_time);
-
-  gcptrlist_clear(&d->gcroots);
-  d->num_commits++;
-  d->active = 0;
-  err = pthread_rwlock_unlock(&rwlock_in_transaction);
-  assert(err == 0);
-}
-
-/************************************************************/
-
-static void make_inevitable(struct tx_descriptor *d)
-{
-  d->setjmp_buf = NULL;
-  d->active = 2;
-  d->reads_size_limit_nonatomic = 0;
-  update_reads_size_limit(d);
-}
-
-static revision_t acquire_inev_mutex_and_mark_global_cur_time(void)
-{
-  revision_t cur_time;
-
-  inev_mutex_acquire();
-  while (1)
-    {
-      cur_time = global_cur_time;
-      assert((cur_time & 1) == 0);
-      if (bool_cas(&global_cur_time, cur_time, cur_time + 1))
-        break;
-      /* else try again */
-    }
-  return cur_time;
-}
-
-void BecomeInevitable(const char *why)
-{
-  revision_t cur_time;
-  struct tx_descriptor *d = thread_descriptor;
-  if (d == NULL || d->active != 1)
-    return;  /* I am already inevitable, or not in a transaction at all
-                (XXX statically we should know when we're outside
-                a transaction) */
-
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_START("stm-inevitable");
-  if (PYPY_HAVE_DEBUG_PRINTS)
-    {
-      fprintf(PYPY_DEBUG_FILE, "%s\n", why);
-    }
-#endif
-
-  cur_time = acquire_inev_mutex_and_mark_global_cur_time();
-  if (d->start_time != cur_time)
-    {
-      d->start_time = cur_time;
-      if (!ValidateDuringTransaction(d, 0))
-        {
-          global_cur_time = cur_time;   // revert from (cur_time + 1)
-          inev_mutex_release();
-          AbortTransaction(3);
-        }
-    }
-  make_inevitable(d);    /* cannot abort any more */
-
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_STOP("stm-inevitable");
-#endif
-}
-
-void BeginInevitableTransaction(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  revision_t cur_time;
-
-  init_transaction(d);
-  cur_time = acquire_inev_mutex_and_mark_global_cur_time();
-  d->start_time = cur_time;
-  make_inevitable(d);
-}
-
-/************************************************************/
-
-inline static _Bool _PtrEq_Globals(gcptr G1, gcptr G2)
-{
-  /* This is a mess, because G1 and G2 can be different pointers to "the
-     same" object, and it's hard to determine that.  Description of the
-     idealized problem: we have chained lists following the 'h_revision'
-     field from G1 and from G2, and we must return True if the chained
-     lists end in the same object G and False if they don't.
-
-     It's possible that G1 != G2 but G1->h_revision == G2->h_revision.
-     This occurs because of random updates done by PossiblyUpdateChain().
-
-     But the real mess comes from other threads doing concurrent
-     updates.  They can be either done by PossiblyUpdateChain() from
-     other threads, changing 'h_revision' without warning to point to
-     what used to be 'h_revision-> ... -> h_revision'.  Alternatively,
-     they can be adding new objects to the so-far root of the tree:
-     consider the G that used to be the end of both chained lists.  Its
-     'h_revision' used to be not-a-pointer.  But it can suddenly
-     suddenly turns into a LOCK value, and then later to a pointer to
-     some new object, making the chained list longer.  Worse, once the
-     list is longer, a PossiblyUpdateChain() might make shortcuts in
-     it...
-
-     So it means even the following algorithm is flawed: "follow
-     h_revisions from G1 until we reach G; then follow h_revisions from
-     G2, and return True/False if we reach/do not reach the same G".
-
-     For now we go with: try to move forward G1 and G2 alternatively,
-     until we failed three times to move forward one of these.  Let's
-     say the last three steps were "G1 is already at the end", "G2 is
-     already at the end", "G1 is already at the end".  Then we can
-     return (G1 == G2).  This is because we know that G1 was at the end
-     of the chained list for the whole time during which we checked G2.
-
-     WARNING: THIS ONLY WORKS ON THE X86 MEMORY MODEL, AND NOT ON WEAKER
-     MODELS!
-  */
-  int blocked_steps = 0;
-  volatile revision_t *vp;
-  revision_t v;
-  gcptr G3;
-
-  while (1)
-    {
-      /* try to move G1 forward */
-      vp = (volatile revision_t *)&G1->h_revision;
-      v = *vp;
-      if (v & 1)
-        {
-          blocked_steps++;
-          if (blocked_steps == 3)
-            break;
-        }
-      else
-        {
-          G1 = (gcptr)v;    /* success */
-          blocked_steps = 0;
-        }
-      /* swap the roles of G1 and G2, then loop */
-      G3 = G1; G1 = G2; G2 = G3;
-    }
-  return G1 == G2;
-}
-
-_Bool stm_PtrEq(gcptr P1, gcptr P2)
-{
-  if (P1 == P2)
-    return 1;
-  else if (P1 == NULL || P2 == NULL)   /* and not P1 == P2 == NULL */
-    return 0;
-
-  if (P1->h_tid & GCFLAG_GLOBAL)
-    {
-      if (P2->h_tid & GCFLAG_GLOBAL)
-        return _PtrEq_Globals(P1, P2);
-      else if (P2->h_tid & GCFLAG_LOCAL_COPY)
-        return _PtrEq_Globals(P1, (gcptr)P2->h_revision);
-      else
-        return 0;   /* P1 is global, P2 is new */
-    }
-  /* P1 is local, i.e. either new or a local copy */
-  if (P2->h_tid & GCFLAG_GLOBAL)
-    {
-      if (P1->h_tid & GCFLAG_LOCAL_COPY)
-        return _PtrEq_Globals((gcptr)P1->h_revision, P2);
-      else
-        return 0;   /* P1 is new, P2 is global */
-    }
-  /* P1 and P2 are both locals (and P1 != P2) */
-  return 0;
-}
-
-/************************************************************/
-
-void stm_ThreadLocalRef_LLSet(void **addr, void *newvalue)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  gcptrlist_insert2(&d->undolog, (gcptr)addr, (gcptr)*addr);
-  *addr = newvalue;
-}
-
-/************************************************************/
-
-int DescriptorInit(void)
-{
-  if (thread_descriptor == NULL)
-    {
-      struct tx_descriptor *d = malloc(sizeof(struct tx_descriptor));
-      memset(d, 0, sizeof(struct tx_descriptor));
-
-#ifdef RPY_STM_DEBUG_PRINT
-      PYPY_DEBUG_START("stm-init");
-#endif
-
-      /* initialize 'my_lock' to be a unique odd number >= LOCKED */
-      while (1)
-        {
-          d->my_lock = next_locked_value;
-          if (bool_cas(&next_locked_value, d->my_lock, d->my_lock + 2))
-            break;
-        }
-      if (d->my_lock < LOCKED || d->my_lock == (revision_t)-1)
-        {
-          /* XXX fix this limitation */
-          fprintf(stderr, "XXX error: too many threads ever created "
-                          "in this process");
-          abort();
-        }
-      assert(d->my_lock & 1);
-      thread_descriptor = d;
-
-#ifdef RPY_STM_DEBUG_PRINT
-      if (PYPY_HAVE_DEBUG_PRINTS)
-        fprintf(PYPY_DEBUG_FILE, "thread %lx starting (pthread=%lx)\n",
-                (long)d->my_lock, (long)pthread_self());
-      PYPY_DEBUG_STOP("stm-init");
-#endif
-      return 1;
-    }
-  else
-    return 0;
-}
-
-void DescriptorDone(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d != NULL);
-  assert(d->active == 0);
-
-  thread_descriptor = NULL;
-
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_START("stm-done");
-  if (PYPY_HAVE_DEBUG_PRINTS) {
-    int num_aborts = 0, num_spinloops = 0;
-    int i;
-    char line[256], *p = line;
-
-    for (i=0; i<ABORT_REASONS; i++)
-      num_aborts += d->num_aborts[i];
-    for (i=0; i<SPINLOOP_REASONS; i++)
-      num_spinloops += d->num_spinloops[i];
-
-    p += sprintf(p, "thread %lx: %d commits, %d aborts\n",
-                 (long)d->my_lock,
-                 d->num_commits,
-                 num_aborts);
-
-    for (i=0; i<ABORT_REASONS; i++)
-      p += sprintf(p, "%c%d", i == 0 ? '[' : ',',
-                   d->num_aborts[i]);
-
-    for (i=1; i<SPINLOOP_REASONS; i++)  /* num_spinloops[0] == num_aborts */
-      p += sprintf(p, "%c%d", i == 1 ? '|' : ',',
-                   d->num_spinloops[i]);
-
-    p += sprintf(p, "]\n");
-    fwrite(line, 1, p - line, PYPY_DEBUG_FILE);
-  }
-  PYPY_DEBUG_STOP("stm-done");
-#endif
-
-  free(d);
-}
-
-#include "rpyintf.c"
diff --git a/rpython/translator/stm/src_stm/et.h b/rpython/translator/stm/src_stm/et.h
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/et.h
+++ /dev/null
@@ -1,140 +0,0 @@
-/*** Extendable Timestamps
- *
- * Documentation:
- * https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/stm2012/stmimpl.rst
- *
- * This is very indirectly based on rstm_r5/stm/et.hpp.
- * See http://www.cs.rochester.edu/research/synchronization/rstm/api.shtml
- *
- * Stand-alone version of these files, including random stress-tests:
- * https://bitbucket.org/arigo/arigo/raw/default/hack/stm/c2
- *
- */
-
-#ifndef _ET_H
-#define _ET_H
-
-#include <stddef.h>
-#include <setjmp.h>
-
-
-/* These are partly the same flags as defined in stmgc.py, as well as
-   nogcstm.py.  Keep in sync! */
-enum {
-  _first_gcflag            = 1L << (PYPY_LONG_BIT / 2),
-  GCFLAG_GLOBAL            = _first_gcflag << 0,
-  GCFLAG_POSSIBLY_OUTDATED = _first_gcflag << 1,
-  GCFLAG_NOT_WRITTEN       = _first_gcflag << 2,
-  GCFLAG_LOCAL_COPY        = _first_gcflag << 3,
-  GCFLAG_VISITED           = _first_gcflag << 4,
-
-  GCFLAG_PREBUILT          = GCFLAG_GLOBAL|GCFLAG_NOT_WRITTEN,
-  REV_INITIAL              = 1,
-};
-
-typedef struct pypy_header0 *gcptr;
-/*declared in structdef.h as {
-    Signed h_tid;
-    void *h_revision;
-}*/
-
-#define STM_BARRIER_P2R(P)                                              \
-    (__builtin_expect((((gcptr)(P))->h_tid & GCFLAG_GLOBAL) == 0, 1) ?  \
-     (P) : (typeof(P))stm_DirectReadBarrier(P))
-
-#define STM_BARRIER_G2R(G)                                          \
-    (assert(((gcptr)(G))->h_tid & GCFLAG_GLOBAL),                   \
-     (typeof(G))stm_DirectReadBarrier(G))
-
-#define STM_BARRIER_O2R(O)                                              \
-    (__builtin_expect((((gcptr)(O))->h_tid & GCFLAG_POSSIBLY_OUTDATED) == 0, \
-                      1) ?                                              \
-     (O) : (typeof(O))stm_RepeatReadBarrier(O))
-
-/*#define STM_READ_BARRIER_P_FROM_R(P, R_container, offset)             \
-    (__builtin_expect((((gcptr)(P))->h_tid & GCFLAG_GLOBAL) == 0, 1) ?  \
-     (P) : (typeof(P))stm_DirectReadBarrierFromR((P),            \
-                                              (R_container),     \
-                                              offset))*/
-
-#define STM_BARRIER_P2W(P)                                                  \
-    (__builtin_expect((((gcptr)(P))->h_tid & GCFLAG_NOT_WRITTEN) == 0, 1) ? \
-     (P) : (typeof(P))stm_WriteBarrier(P))
-
-#define STM_BARRIER_G2W(G)                              \
-    (assert(((gcptr)(G))->h_tid & GCFLAG_GLOBAL),       \
-     (typeof(G))stm_WriteBarrier(G))
-
-#define STM_BARRIER_R2W(R)                                                  \
-    (__builtin_expect((((gcptr)(R))->h_tid & GCFLAG_NOT_WRITTEN) == 0, 1) ? \
-     (R) : (typeof(R))stm_WriteBarrierFromReady(R))
-
-#define STM_BARRIER_O2W(R)  STM_BARRIER_R2W(R)   /* same logic works here */
-
-#define STM_PTR_EQ(P1, P2)                      \
-    stm_PtrEq((gcptr)(P1), (gcptr)(P2))
-
-#define OP_STM_THREADLOCALREF_LLSET(P, X, IGNORED)          \
-    stm_ThreadLocalRef_LLSet((void **)(P), (void *)(X))
-
-/* special usage only */
-#define OP_STM_READ_BARRIER(P, R)   R = STM_BARRIER_P2R(P)
-#define OP_STM_WRITE_BARRIER(P, W)   W = STM_BARRIER_P2W(P)
-
-
-void BeginTransaction(jmp_buf *);
-void BeginInevitableTransaction(void);
-//int _FakeReach(gcptr);
-void CommitTransaction(void);
-void BecomeInevitable(const char *why);
-int DescriptorInit(void);
-void DescriptorDone(void);
-
-//gcptr Allocate(size_t size, int gctid);
-_Bool stm_PtrEq(gcptr P1, gcptr P2);
-
-void *stm_DirectReadBarrier(void *);
-void *stm_DirectReadBarrierFromR(void *, void *, size_t);
-void *stm_RepeatReadBarrier(void *);
-void *stm_WriteBarrier(void *);
-void *stm_WriteBarrierFromReady(void *);
-//gcptr _NonTransactionalReadBarrier(gcptr);
-
-void stm_ThreadLocalRef_LLSet(void **P, void *X);
-
-
-extern void *pypy_g__stm_duplicate(void *);
-extern void pypy_g__stm_enum_callback(void *, void *);
-void *stm_set_tls(void *newtls);
-void *stm_get_tls(void);
-void stm_del_tls(void);
-gcptr stm_tldict_lookup(gcptr);     /* for tests only */
-void stm_tldict_add(gcptr, gcptr);  /* for tests only */
-void stm_tldict_enum(void);
-void stm_tldict_enum_external(void *l_thread_descriptor);
-long stm_in_transaction(void);
-long stm_is_inevitable(void);
-void stm_add_atomic(long delta);
-long stm_get_atomic(void);
-long stm_should_break_transaction(void);
-void stm_set_transaction_length(long length_max);
-void stm_perform_transaction(long(*callback)(void*, long), void *arg,
-                             void *save_and_restore);
-void stm_abort_and_retry(void);
-void stm_abort_info_push(void *, void *);
-void stm_abort_info_pop(long);
-char *stm_inspect_abort_info(void);
-long stm_extraref_llcount(void);
-gcptr *stm_extraref_lladdr(long);
-
-#ifdef USING_NO_GC_AT_ALL
-# define OP_GC_ADR_OF_ROOT_STACK_TOP(r)   r = NULL
-void stm_nogc_start_transaction(void);
-void stm_nogc_stop_transaction(void);
-gcptr stm_nogc_allocate(size_t);
-# undef OP_BOEHM_ZERO_MALLOC
-# define OP_BOEHM_ZERO_MALLOC(size, r, restype, is_atomic, is_varsize)  \
-    r = (restype) stm_nogc_allocate(size)
-#endif
-
-#endif  /* _ET_H */
diff --git a/rpython/translator/stm/src_stm/lists.c b/rpython/translator/stm/src_stm/lists.c
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/lists.c
+++ /dev/null
@@ -1,328 +0,0 @@
-/* -*- c-basic-offset: 2 -*- */
-
-#include <limits.h>
-
-/************************************************************/
-
-/* The g2l_xx functions ("global_to_local") are implemented as a tree,
-   supporting very high performance in G2L_FIND in the common case where
-   there are no or few elements in the tree, but scaling correctly
-   if the number of items becomes large. */
-
-#define TREE_BITS   4
-#define TREE_ARITY  (1 << TREE_BITS)
-
-#define TREE_DEPTH_MAX   ((sizeof(void*)*8 - 2 + TREE_BITS-1) / TREE_BITS)
-/* sizeof(void*) = total number of bits
-   2 = bits that we ignore anyway (2 or 3, conservatively 2)
-   (x + TREE_BITS-1) / TREE_BITS = divide by TREE_BITS, rounding up
-*/
-
-#define TREE_MASK   ((TREE_ARITY - 1) * sizeof(void*))
-
-typedef struct {
-  gcptr addr;
-  gcptr val;
-} wlog_t;
-
-typedef struct {
-  char *items[TREE_ARITY];
-} wlog_node_t;
-
-struct G2L {
-  char *raw_start, *raw_current, *raw_end;
-  wlog_node_t toplevel;
-};
-
-static void _g2l_clear_node(wlog_node_t *node)
-{
-  memset(node, 0, sizeof(wlog_node_t));
-}
-
-static void g2l_clear(struct G2L *g2l)
-{
-  if (g2l->raw_current != g2l->raw_start)
-    {
-      _g2l_clear_node(&g2l->toplevel);
-      g2l->raw_current = g2l->raw_start;
-    }
-}
-
-static int g2l_any_entry(struct G2L *g2l)
-{
-  return g2l->raw_current != g2l->raw_start;
-}
-
-#define _G2L_LOOP(g2l, item, INITIAL, _PLUS_)                           \
-{                                                                       \
-  struct { char **next; char **end; } _stack[TREE_DEPTH_MAX], *_stackp; \
-  char **_next, **_end, *_entry;                                        \
-  /* initialization */                                                  \
-  _stackp = _stack;      /* empty stack */                              \
-  _next = (g2l).toplevel.items + INITIAL;                               \
-  _end = _next _PLUS_ TREE_ARITY;                                       \
-  /* loop */                                                            \
-  while (1)                                                             \
-    {                                                                   \
-      if (_next == _end)                                                \
-        {                                                               \
-          if (_stackp == _stack)                                        \
-            break;   /* done */                                         \
-          /* finished with this level, go to the next one */            \
-          _stackp--;                                                    \
-          _next = _stackp->next;                                        \
-          _end = _stackp->end;                                          \
-          continue;                                                     \
-        }                                                               \
-      _entry = *_next;                                                  \
-      _next = _next _PLUS_ 1;                                           \
-      if (_entry == NULL)   /* empty entry */                           \
-        continue;                                                       \
-      if (((long)_entry) & 1)                                           \
-        {  /* points to a further level: enter it */                    \
-          _stackp->next = _next;                                        \
-          _stackp->end = _end;                                          \
-          _stackp++;                                                    \
-          _next = ((wlog_node_t *)(_entry - 1))->items + INITIAL;       \
-          _end = _next _PLUS_ TREE_ARITY;                               \
-          continue;                                                     \
-        }                                                               \
-      /* points to a wlog_t item */                                     \
-      item = (wlog_t *)_entry;
-
-#define G2L_LOOP_FORWARD(g2l, item)                             \
-                       _G2L_LOOP(g2l, item, 0, +)
-#define G2L_LOOP_BACKWARD(g2l, item)                            \
-                       _G2L_LOOP(g2l, item, (TREE_ARITY-1), -)
-#define G2L_LOOP_END     } }
-
-#define G2L_FIND(g2l, addr1, result, goto_not_found)            \
-{                                                               \
-  unsigned long _key = (unsigned long)(addr1);                  \
-  char *_p = (char *)((g2l).toplevel.items);                    \
-  char *_entry = *(char **)(_p + (_key & TREE_MASK));           \
-  if (__builtin_expect(_entry == NULL, 1))                      \
-    goto_not_found;    /* common case, hopefully */             \
-  result = _g2l_find(_entry, addr1);                            \
-  if (result == NULL || result->addr != (addr1))                \
-    goto_not_found;                                             \
-}
-
-static wlog_t *_g2l_find(char *entry, gcptr addr)
-{
-  unsigned long key = (unsigned long)addr;
-  while (((long)entry) & 1)
-    {   /* points to a further level */
-      key >>= TREE_BITS;
-      entry = *(char **)((entry - 1) + (key & TREE_MASK));
-    }
-  return (wlog_t *)entry;   /* may be NULL */
-}
-
-static void g2l_insert(struct G2L *g2l, gcptr addr, gcptr val);
-
-static void _g2l_grow(struct G2L *g2l, long extra)
-{
-  struct G2L newg2l;
-  wlog_t *item;
-  long alloc = g2l->raw_end - g2l->raw_start;
-  long newalloc = (alloc + extra + (alloc >> 2) + 31) & ~15;
-  //fprintf(stderr, "growth: %ld\n", newalloc);
-  char *newitems = malloc(newalloc);
-  newg2l.raw_start = newitems;
-  newg2l.raw_current = newitems;
-  newg2l.raw_end = newitems + newalloc;
-  _g2l_clear_node(&newg2l.toplevel);
-  G2L_LOOP_FORWARD(*g2l, item)
-    {
-      g2l_insert(&newg2l, item->addr, item->val);
-    } G2L_LOOP_END;
-  free(g2l->raw_start);
-  *g2l = newg2l;
-}
-
-static char *_g2l_grab(struct G2L *g2l, long size)
-{
-  char *result;
-  result = g2l->raw_current;
-  g2l->raw_current += size;
-  if (g2l->raw_current > g2l->raw_end)
-    {
-      _g2l_grow(g2l, size);
-      return NULL;
-    }
-  return result;
-}
-
-static void g2l_insert(struct G2L *g2l, gcptr addr, gcptr val)
-{
- retry:;
-  wlog_t *wlog;
-  unsigned long key = (unsigned long)addr;
-  int shift = 0;
-  char *p = (char *)(g2l->toplevel.items);
-  char *entry;
-  assert((key & (sizeof(void*)-1)) == 0);   /* only for aligned keys */
-  while (1)
-    {
-      p += (key >> shift) & TREE_MASK;
-      shift += TREE_BITS;
-      entry = *(char **)p;
-      if (entry == NULL)
-        break;
-      else if (((long)entry) & 1)
-        {   /* points to a further level */
-          p = entry - 1;
-        }
-      else
-        {
-          wlog_t *wlog1 = (wlog_t *)entry;
-          /* the key must not already be present */
-          assert(wlog1->addr != addr);
-          /* collision: there is already a different wlog here */
-          wlog_node_t *node = (wlog_node_t *)
-                _g2l_grab(g2l, sizeof(wlog_node_t));
-          if (node == NULL) goto retry;
-          _g2l_clear_node(node);
-          unsigned long key1 = (unsigned long)(wlog1->addr);
-          char *p1 = (char *)(node->items);
-          *(wlog_t **)(p1 + ((key1 >> shift) & TREE_MASK)) = wlog1;
-          *(char **)p = ((char *)node) + 1;
-          p = p1;
-        }
-    }
-  wlog = (wlog_t *)_g2l_grab(g2l, sizeof(wlog_t));
-  if (wlog == NULL) goto retry;
-  wlog->addr = addr;
-  wlog->val = val;
-  *(char **)p = (char *)wlog;
-}
-
-/************************************************************/
-
-/* The gcptrlist_xx functions are implemented as an array that grows
-   as needed. */
-
-struct GcPtrList {
-  long size, alloc;
-  gcptr *items;
-};
-
-static void gcptrlist_clear(struct GcPtrList *gcptrlist)
-{
-  gcptrlist->size = 0;
-}
-
-static void _gcptrlist_grow(struct GcPtrList *gcptrlist)
-{
-  long newalloc = gcptrlist->alloc + (gcptrlist->alloc >> 1) + 16;
-
-  PYPY_DEBUG_START("stm-growth");
-  if (PYPY_HAVE_DEBUG_PRINTS)
-    {
-      fprintf(PYPY_DEBUG_FILE, "%ld KB\n", newalloc * sizeof(gcptr) / 1024);
-    }
-
-  gcptr *newitems = malloc(newalloc * sizeof(gcptr));
-  long i;
-  for (i=0; i<gcptrlist->size; i++)
-    newitems[i] = gcptrlist->items[i];
-  free(gcptrlist->items);
-  gcptrlist->items = newitems;
-  gcptrlist->alloc = newalloc;
-
-  PYPY_DEBUG_STOP("stm-growth");
-}
-
-static inline void gcptrlist_insert(struct GcPtrList *gcptrlist, gcptr newitem)
-{
-  if (__builtin_expect(gcptrlist->size == gcptrlist->alloc, 0))
-    _gcptrlist_grow(gcptrlist);
-  gcptrlist->items[gcptrlist->size++] = newitem;
-}
-
-static void gcptrlist_insert2(struct GcPtrList *gcptrlist, gcptr newitem1,
-                              gcptr newitem2)
-{
-  gcptr *items;
-  long i = gcptrlist->size;
-  if (__builtin_expect((gcptrlist->alloc - i) < 2, 0))
-    _gcptrlist_grow(gcptrlist);
-  items = gcptrlist->items;
-  items[i+0] = newitem1;
-  items[i+1] = newitem2;
-  gcptrlist->size = i + 2;
-}
-
-static void gcptrlist_reduce_size(struct GcPtrList *gcptrlist, long newsize)
-{
-  gcptrlist->size = newsize;
-}
-
-/************************************************************/
-
-/* The fxcache_xx functions implement a fixed-size set of gcptr's.
-   Moreover the gcptr's in the set are mapped to small integers.  In case
-   of collisions, old items are discarded.  The cache uses 3-way caching
-   and the cache entries are aligned to two, which means that the items
-   0 and 2 can collide with the bordering cache rows, but not item 1.
-
-   The cache itself uses a total of FX_ENTRIES+1 entries in the 'cache'
-   array below, starting at 'cache_start'.  The reason it is bigger is
-   that fxcache_clear() simply shifts 'cache_start', making any previous
-   entries invalid by not being in the correct position any more.
-*/
-
-#define FX_ENTRIES   8192
-#define FX_TOTAL     (FX_ENTRIES * 2)
-
-struct FXCache {
-  char *cache_start;
-  revision_t nextadd;
-  revision_t shift;
-  revision_t cache[FX_TOTAL];
-};
-
-static void fxcache_clear(struct FXCache *fxcache)
-{
-  fxcache->shift += 4;
-  /* FX_ENTRIES+1 entries are needed */
-  if (fxcache->shift + FX_ENTRIES + 1 > FX_TOTAL) {
-    memset(fxcache->cache, 0, sizeof(fxcache->cache));
-    fxcache->shift = 0;
-  }
-  fxcache->cache_start = (char *)(fxcache->cache + fxcache->shift);
-}
-
-static inline int fxcache_add(struct FXCache *fxcache, gcptr item)
-{
-  /* If 'item' is not in the cache, add it and returns 0.
-     If it is already, return 1.
-     */
-  revision_t uitem = (revision_t)item;
-  /* 'entry' points to 'cache_start[mask of uitem, even-valued]' */
-  revision_t *entry = (revision_t *)
-    (fxcache->cache_start + (uitem & ((FX_ENTRIES-2) * sizeof(revision_t))));
-  revision_t current;
-
-  current = entry[1];   /* first look here, the cache-private entry */
-  if (current == uitem)
-    return 1;
-
-  if (entry[0] == uitem) {
-    entry[0] = current;    /* move from this collidable entry to */
-    entry[1] = uitem;      /*    the cache-private entry         */
-    return 1;
-  }
-  if (entry[2] == uitem) {
-    entry[2] = current;    /* move from this collidable entry to */
-    entry[1] = uitem;      /*    the cache-private entry         */
-    return 1;
-  }
-
-  entry[fxcache->nextadd] = uitem;
-  fxcache->nextadd ^= 2;
-  return 0;
-}
-
-/************************************************************/
diff --git a/rpython/translator/stm/src_stm/rpyintf.c b/rpython/translator/stm/src_stm/rpyintf.c
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/rpyintf.c
+++ /dev/null
@@ -1,461 +0,0 @@
-
-static __thread void *rpython_tls_object;
-
-void *stm_set_tls(void *newtls)
-{
-  rpython_tls_object = newtls;
-  return (void *)thread_descriptor;
-}
-
-void *stm_get_tls(void)
-{
-  return rpython_tls_object;
-}
-
-void stm_del_tls(void)
-{
-  rpython_tls_object = NULL;
-}
-
-gcptr stm_tldict_lookup(gcptr key)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  wlog_t* found;
-  G2L_FIND(d->global_to_local, key, found, goto not_found);
-  return found->val;
-
- not_found:
-  return NULL;
-}
-
-void stm_tldict_add(gcptr key, gcptr value)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d != NULL);
-  g2l_insert(&d->global_to_local, key, value);
-}
-
-void stm_tldict_enum(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  wlog_t *item;
-  void *tls = stm_get_tls();
-
-  G2L_LOOP_FORWARD(d->global_to_local, item)
-    {
-      gcptr R = item->addr;
-      gcptr L = item->val;
-      assert(L->h_revision == (revision_t)R);
-      if ((L->h_tid & GCFLAG_NOT_WRITTEN) == 0)
-        pypy_g__stm_enum_callback(tls, L);
-    } G2L_LOOP_END;
-}
-
-void stm_tldict_enum_external(void *l_thread_descriptor)
-{
-  struct tx_descriptor *d = (struct tx_descriptor *)l_thread_descriptor;
-  wlog_t *item;
-
-  G2L_LOOP_FORWARD(d->global_to_local, item)
-    {
-      gcptr R = item->addr;
-      gcptr L = item->val;
-      assert(L->h_revision == (revision_t)R);
-      pypy_g__stm_enum_external_callback(R, L);
-    } G2L_LOOP_END;
-}
-
-long stm_in_transaction(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  return d->active;
-}
-
-long stm_is_inevitable(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  return is_inevitable(d);
-}
-
-static unsigned long stm_regular_length_limit = ULONG_MAX;
-
-/* sync_required is either 0 or 0xffffffff */
-#define SYNC_REQUIRED  ((unsigned long)-1)
-static volatile unsigned long sync_required = 0;
-static void reached_safe_point(void);
-
-void stm_add_atomic(long delta)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  d->atomic += delta;
-  update_reads_size_limit(d);
-}
-
-long stm_get_atomic(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  return d->atomic;
-}
-
-long stm_should_break_transaction(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-
-  /* a single comparison to handle all cases:
-
-     - first, if sync_required == 0xffffffff, this should return True.
-
-     - if d->atomic, then we should return False.  This is done by
-       forcing reads_size_limit to ULONG_MAX as soon as atomic > 0.
-
-     - otherwise, if is_inevitable(), then we should return True.
-       This is done by forcing both reads_size_limit and
-       reads_size_limit_nonatomic to 0 in that case.
-
-     - finally, the default case: return True if
-       d->count_reads is
-       greater than reads_size_limit == reads_size_limit_nonatomic.
-  */
-#if 0   /* ifdef RPY_STM_ASSERT */
-  /* reads_size_limit is ULONG_MAX if d->atomic, or else it is equal to
-     reads_size_limit_nonatomic. */
-  assert(d->reads_size_limit == (d->atomic ? ULONG_MAX :
-                                     d->reads_size_limit_nonatomic));
-  /* if is_inevitable(), reads_size_limit_nonatomic should be 0
-     (and thus reads_size_limit too, if !d->atomic.) */
-  if (is_inevitable(d))
-    assert(d->reads_size_limit_nonatomic == 0);
-#endif
-
-  return (sync_required | d->count_reads) >= d->reads_size_limit;
-}
-
-void stm_set_transaction_length(long length_max)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  BecomeInevitable("set_transaction_length");
-  if (length_max <= 0)
-    length_max = 1;
-  stm_regular_length_limit = length_max;
-}
-
-#define END_MARKER   ((void*)-8)   /* keep in sync with stmframework.py */
-
-void stm_perform_transaction(long(*callback)(void*, long), void *arg,
-                             void *save_and_restore)
-{
-  jmp_buf _jmpbuf;
-  long volatile v_counter = 0;
-  void **volatile v_saved_value;
-  long volatile v_atomic = thread_descriptor->atomic;
-  assert((!thread_descriptor->active) == (!v_atomic));
-#ifndef USING_NO_GC_AT_ALL
-  v_saved_value = *(void***)save_and_restore;
-#endif
-  PYPY_DEBUG_START("stm-perform-transaction");
-  /***/
-  setjmp(_jmpbuf);
-  /* After setjmp(), the local variables v_* are preserved because they
-   * are volatile.  The other variables are only declared here. */
-  struct tx_descriptor *d = thread_descriptor;
-  long counter, result;
-  void **restore_value;
-  counter = v_counter;
-  d->atomic = v_atomic;
-#ifndef USING_NO_GC_AT_ALL
-  restore_value = v_saved_value;
-  if (!d->atomic)
-    {
-      /* In non-atomic mode, we are now between two transactions.
-         It means that in the next transaction's collections we know
-         that we won't need to access the shadows stack beyond its
-         current position.  So we add an end marker. */
-      *restore_value++ = END_MARKER;
-    }
-  *(void***)save_and_restore = restore_value;
-#endif
-
-  do
-    {
-      v_counter = counter + 1;
-      /* If counter==0, initialize 'reads_size_limit_nonatomic' from the
-         configured length limit.  If counter>0, we did an abort, which
-         has configured 'reads_size_limit_nonatomic' to a smaller value.
-         When such a shortened transaction succeeds, the next one will
-         see its length limit doubled, up to the maximum. */
-      if (counter == 0) {
-          unsigned long limit = d->reads_size_limit_nonatomic;
-          if (limit != 0 && limit < (stm_regular_length_limit >> 1))
-              limit = (limit << 1) | 1;
-          else
-              limit = stm_regular_length_limit;
-          d->reads_size_limit_nonatomic = limit;
-      }
-      if (!d->atomic)
-        BeginTransaction(&_jmpbuf);
-      else
-        {
-          /* atomic transaction: a common case is that callback() returned
-             even though we are atomic because we need a major GC.  For
-             that case, release and require the rw lock here. */
-          if (sync_required)
-            reached_safe_point();
-        }
-
-      /* invoke the callback in the new transaction */
-      result = callback(arg, counter);
-
-      v_atomic = d->atomic;
-      if (!d->atomic)
-        CommitTransaction();
-      counter = 0;
-    }
-  while (result == 1);  /* also stops if we got an RPython exception */
-
-  if (d->atomic && d->setjmp_buf == &_jmpbuf)
-    BecomeInevitable("perform_transaction left with atomic");
-
-#ifndef USING_NO_GC_AT_ALL
-  *(void***)save_and_restore = v_saved_value;
-#endif
-  PYPY_DEBUG_STOP("stm-perform-transaction");
-}
-
-static struct tx_descriptor *in_single_thread = NULL;  /* for debugging */
-
-void stm_start_single_thread(void)
-{
-  /* Called by the GC, just after a minor collection, when we need to do
-     a major collection.  When it returns, it acquired the "write lock"
-     which prevents any other thread from running in a transaction.
-     Warning, may block waiting for rwlock_in_transaction while another
-     thread runs a major GC itself! */
-  int err;
-  sync_required = SYNC_REQUIRED;
-  err = pthread_rwlock_unlock(&rwlock_in_transaction);
-  assert(err == 0);
-  err = pthread_rwlock_wrlock(&rwlock_in_transaction);
-  assert(err == 0);
-  sync_required = 0;
-
-  assert(in_single_thread == NULL);
-  in_single_thread = thread_descriptor;
-  assert(in_single_thread != NULL);
-}
-
-void stm_stop_single_thread(void)
-{
-  /* Warning, may block waiting for rwlock_in_transaction while another
-     thread runs a major GC */
-  int err;
-
-  assert(in_single_thread == thread_descriptor);
-  in_single_thread = NULL;
-
-  err = pthread_rwlock_unlock(&rwlock_in_transaction);
-  assert(err == 0);
-  err = pthread_rwlock_rdlock(&rwlock_in_transaction);
-  assert(err == 0);
-}
-
-static void reached_safe_point(void)
-{
-  /* Warning, may block waiting for rwlock_in_transaction while another
-     thread runs a major GC */
-  int err;
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d->active);
-  assert(in_single_thread != d);
-
-  err = pthread_rwlock_unlock(&rwlock_in_transaction);
-  assert(err == 0);
-  /* another thread should be waiting in pthread_rwlock_wrlock(),
-     which takes priority here */
-  err = pthread_rwlock_rdlock(&rwlock_in_transaction);
-  assert(err == 0);
-}
-
-void stm_abort_and_retry(void)
-{
-  AbortTransaction(4);    /* manual abort */
-}
-
-void stm_abort_info_push(void *obj, void *fieldoffsets)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    gcptr P = (gcptr)obj;
-    if ((P->h_tid & GCFLAG_GLOBAL) &&
-        (P->h_tid & GCFLAG_POSSIBLY_OUTDATED)) {
-        P = LatestGlobalRevision(d, P, NULL, 0);
-    }
-    gcptrlist_insert2(&d->abortinfo, P, (gcptr)fieldoffsets);
-}
-
-void stm_abort_info_pop(long count)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    long newsize = d->abortinfo.size - 2 * count;
-    gcptrlist_reduce_size(&d->abortinfo, newsize < 0 ? 0 : newsize);
-}
-
-size_t _stm_decode_abort_info(struct tx_descriptor *d, long long elapsed_time,
-                              int abort_reason, char *output)
-{
-    /* re-encodes the abort info as a single string.
-       For convenience (no escaping needed, no limit on integer
-       sizes, etc.) we follow the bittorrent format. */
-    size_t totalsize = 0;
-    long i;
-    char buffer[32];
-    size_t res_size;
-#define WRITE(c)   { totalsize++; if (output) *output++=(c); }
-#define WRITE_BUF(p, sz)  { totalsize += (sz);                          \
-                            if (output) {                               \
-                                 memcpy(output, (p), (sz)); output += (sz); \
-                             }                                          \
-                           }
-    WRITE('l');
-    WRITE('l');
-    res_size = sprintf(buffer, "i%llde", (long long)elapsed_time);
-    WRITE_BUF(buffer, res_size);
-    res_size = sprintf(buffer, "i%de", (int)abort_reason);
-    WRITE_BUF(buffer, res_size);
-    res_size = sprintf(buffer, "i%lde", (long)(d->my_lock - LOCKED));
-    WRITE_BUF(buffer, res_size);
-    res_size = sprintf(buffer, "i%lde", (long)d->atomic);
-    WRITE_BUF(buffer, res_size);
-    res_size = sprintf(buffer, "i%de", (int)d->active);
-    WRITE_BUF(buffer, res_size);
-    res_size = sprintf(buffer, "i%lue", (unsigned long)d->count_reads);
-    WRITE_BUF(buffer, res_size);
-    res_size = sprintf(buffer, "i%lue",
-                       (unsigned long)d->reads_size_limit_nonatomic);
-    WRITE_BUF(buffer, res_size);
-    WRITE('e');
-    for (i=0; i<d->abortinfo.size; i+=2) {
-        char *object = (char *)stm_RepeatReadBarrier(d->abortinfo.items[i+0]);
-        long *fieldoffsets = (long*)d->abortinfo.items[i+1];
-        long kind, offset;
-        size_t rps_size;
-        RPyString *rps;
-
-        while (1) {
-            kind = *fieldoffsets++;
-            if (kind <= 0) {
-                if (kind == -2) {
-                    WRITE('l');    /* '[', start of sublist */
-                    continue;
-                }
-                if (kind == -1) {
-                    WRITE('e');    /* ']', end of sublist */
-                    continue;
-                }
-                break;   /* 0, terminator */
-            }
-            offset = *fieldoffsets++;
-            switch(kind) {
-            case 1:    /* signed */
-                res_size = sprintf(buffer, "i%lde",
-                                   *(long*)(object + offset));
-                WRITE_BUF(buffer, res_size);
-                break;
-            case 2:    /* unsigned */
-                res_size = sprintf(buffer, "i%lue",
-                                   *(unsigned long*)(object + offset));
-                WRITE_BUF(buffer, res_size);
-                break;
-            case 3:    /* pointer to STR */
-                rps = *(RPyString **)(object + offset);
-                if (rps) {
-                    rps_size = RPyString_Size(rps);
-                    res_size = sprintf(buffer, "%zu:", rps_size);
-                    WRITE_BUF(buffer, res_size);
-                    WRITE_BUF(_RPyString_AsString(rps), rps_size);
-                }
-                else {
-                    WRITE_BUF("0:", 2);
-                }
-                break;
-            default:
-                fprintf(stderr, "Fatal RPython error: corrupted abort log\n");
-                abort();
-            }
-        }
-    }
-    WRITE('e');
-    WRITE('\0');   /* final null character */
-#undef WRITE
-    return totalsize;
-}
-
-char *stm_inspect_abort_info(void)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    if (d->longest_abort_info_time <= 0)
-        return NULL;
-    d->longest_abort_info_time = 0;
-    return d->longest_abort_info;
-}
-
-long stm_extraref_llcount(void)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    return d->abortinfo.size / 2;
-}
-
-gcptr *stm_extraref_lladdr(long index)
-{
-    struct tx_descriptor *d = thread_descriptor;
-    return &d->abortinfo.items[index * 2];
-}
-
-#ifdef USING_NO_GC_AT_ALL
-static __thread gcptr stm_nogc_chained_list;
-void stm_nogc_start_transaction(void)
-{
-    stm_nogc_chained_list = NULL;
-}
-gcptr stm_nogc_allocate(size_t size)
-{
-    gcptr W = calloc(1, size);
-    if (!W) {
-        fprintf(stderr, "out of memory!\n");
-        abort();
-    }
-    W->h_size = size;
-    W->h_revision = (revision_t)stm_nogc_chained_list;
-    stm_nogc_chained_list = W;
-    return W;
-}
-void stm_nogc_stop_transaction(void)
-{
-    gcptr W = stm_nogc_chained_list;
-    stm_nogc_chained_list = NULL;
-    while (W) {
-        gcptr W_next = (gcptr)W->h_revision;
-        assert((W->h_tid & (GCFLAG_GLOBAL |
-                            GCFLAG_NOT_WRITTEN |
-                            GCFLAG_LOCAL_COPY)) == 0);
-        W->h_tid |= GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN;
-        W->h_revision = 1;

From noreply at buildbot.pypy.org  Fri Jun 28 20:55:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:23 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: A tool similar to
	pypy/tool/import_cffi.py.
Message-ID: <20130628185523.216411C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65079:787c3e1796bd
Date: 2013-06-28 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/787c3e1796bd/

Log:	A tool similar to pypy/tool/import_cffi.py.

diff --git a/rpython/translator/stm/import_stmgc.py b/rpython/translator/stm/import_stmgc.py
new file mode 100755
--- /dev/null
+++ b/rpython/translator/stm/import_stmgc.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+""" A simple tool for importing the current version of the external stmgc
+repository into pypy; should sync whatever version you provide.  Usage:
+
+import_stmgc.py <path-to-stmgc-working-copy>
+
+The working copy comes from:  hg clone https://bitbucket.org/pypy/stmgc
+"""
+
+import sys, py
+
+def mangle(lines):
+    yield "/* Imported by rpython/translator/stm/import_stmgc.py */\n"
+    for line in lines:
+        yield line
+
+def main(stmgc_dir):
+    stmgc_dir = py.path.local(stmgc_dir).join('c4')
+    stmgc_dest = py.path.local(__file__).join('..', 'src_stm')
+    plist = stmgc_dir.visit(rec=lambda p: False)
+    for p in sorted(plist):
+        if not (p.basename.endswith('.c') or p.basename.endswith('.h')):
+            continue
+        if p.basename.startswith('demo'):
+            continue
+        path = stmgc_dest.join(p.relto(stmgc_dir))
+        print path
+        path.join('..').ensure(dir=1)
+        if path.check():
+            path.remove()
+        path.write(''.join(mangle(p.readlines())))
+        path.chmod(0444)
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print __doc__
+        sys.exit(2)
+    main(sys.argv[1])

From noreply at buildbot.pypy.org  Fri Jun 28 20:55:24 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:24 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Initial import.
Message-ID: <20130628185524.5EB7B1C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65080:02a14eab89f6
Date: 2013-06-28 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/02a14eab89f6/

Log:	Initial import.

diff too long, truncating to 2000 out of 5134 lines

diff --git a/rpython/translator/stm/src_stm/atomic_ops.h b/rpython/translator/stm/src_stm/atomic_ops.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/atomic_ops.h
@@ -0,0 +1,121 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+#ifndef _SRCSTM_ATOMIC_OPS_
+#define _SRCSTM_ATOMIC_OPS_
+
+#include <assert.h>
+#define IMPLIES(a, b)   (!(a) || (b))
+
+/* Ask the compiler to really reload the revision_t argument from memory.
+   That's all that this macro does; it does not imply any type of barrier.
+   Consider it as meaning: I want to read (or possibly write) a shared
+   value out of explicit synchronization now. */
+#define ACCESS_ONCE(x) (*(volatile revision_t *)&(x))
+
+#define UNLIKELY(test)  __builtin_expect(test, 0)
+
+
+#if defined(__amd64__) || defined(__i386__)
+#  define smp_wmb()       asm volatile ("":::"memory")
+#  define smp_spinloop()  asm volatile ("pause":::"memory")
+#elif defined(__powerpc__)
+#  define smp_wmb()       asm volatile ("lwsync":::"memory")
+#  define smp_spinloop()  asm volatile ("":::"memory")   /* fill me? */
+#else
+#  error "Define smp_wmb() for your architecture"
+#endif
+
+
+#ifdef __llvm__
+#  define HAS_SYNC_BOOL_COMPARE_AND_SWAP
+#  define HAS_SYNC_FETCH_AND_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_SYNC_BOOL_COMPARE_AND_SWAP
+#    define HAS_SYNC_FETCH_AND_ADD
+#  endif
+#endif
+
+
+#ifdef HAS_SYNC_BOOL_COMPARE_AND_SWAP
+#  define bool_cas __sync_bool_compare_and_swap
+#else
+/* x86 (32 bits and 64 bits) */
+static inline _Bool
+bool_cas(revision_t *ptr, revision_t old, revision_t _new)
+{
+    revision_t prev;
+#if defined(__amd64__)
+    assert(sizeof(revision_t) == 8);
+#elif defined(__i386__)
+    assert(sizeof(revision_t) == 4);
+#else
+#   error "the custom version of bool_cas() is only for x86 or x86-64"
+#endif
+    asm volatile("lock;"
+#if defined(__amd64__)
+                 "cmpxchgq %1, %2;"
+#else
+                 "cmpxchgl %1, %2;"
+#endif
+                 : "=a"(prev)
+                 : "q"(_new), "m"(*ptr), "a"(old)
+                 : "memory");
+    return prev == old;
+}
+/* end */
+#endif
+
+#ifdef HAS_SYNC_FETCH_AND_ADD
+#  define fetch_and_add __sync_fetch_and_add
+#else
+/* x86 (32 bits and 64 bits) */
+static inline revision_t
+fetch_and_add(revision_t *ptr, revision_t value)
+{
+    revision_t prev;
+#if defined(__amd64__)
+    assert(sizeof(revision_t) == 8);
+#elif defined(__i386__)
+    assert(sizeof(revision_t) == 4);
+#else
+#   error "the custom version of fetch_and_add() is only for x86 or x86-64"
+#endif
+    asm volatile("lock;"
+#if defined(__amd64__)
+                 "xaddq %1, %2;"
+#else
+                 "xaddl %1, %2;"
+#endif
+                 : "=r"(prev)
+                 : "0"(value), "m"(*ptr)
+                 : "memory");
+    return prev;
+}
+/* end */
+#endif
+
+
+#if 0    /* fprinting versions */
+# define spinlock_acquire(lock, targetvalue)                            \
+    do { if (bool_cas(&(lock), 0, (targetvalue))) {                     \
+             dprintf(("<<< locked %d\n", (int)targetvalue));            \
+             break;                                                     \
+         }                                                              \
+         do { smp_spinloop(); } while (ACCESS_ONCE(lock));              \
+    } while (1)
+# define spinlock_release(lock)                                         \
+    do { dprintf(("unlocked >>>\n")); smp_wmb();                        \
+         assert((lock) != 0); (lock) = 0; } while (0)
+#else
+# define spinlock_acquire(lock, targetvalue)                    \
+    do { if (bool_cas(&(lock), 0, (targetvalue))) break;        \
+         do { smp_spinloop(); } while (ACCESS_ONCE(lock));      \
+    } while (1)
+# define spinlock_release(lock)                                 \
+    do { smp_wmb(); assert((lock) != 0); (lock) = 0; } while (0)
+#endif
+
+
+#endif  /* _SRCSTM_ATOMIC_OPS_ */
diff --git a/rpython/translator/stm/src_stm/dbgmem.c b/rpython/translator/stm/src_stm/dbgmem.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/dbgmem.c
@@ -0,0 +1,131 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+#include "stmimpl.h"
+#include <sys/mman.h>
+
+
+#define PAGE_SIZE  4096
+
+
+#ifdef _GC_DEBUG
+/************************************************************/
+
+#define MMAP_TOTAL  671088640   /* 640MB */
+
+static pthread_mutex_t malloc_mutex = PTHREAD_MUTEX_INITIALIZER;
+static char *zone_start, *zone_current = NULL, *zone_end = NULL;
+static signed char accessible_pages[MMAP_TOTAL / PAGE_SIZE] = {0};
+
+
+static void _stm_dbgmem(void *p, size_t sz, int prot)
+{
+    if (sz == 0)
+        return;
+
+    assert((ssize_t)sz > 0);
+    intptr_t align = ((intptr_t)p) & (PAGE_SIZE-1);
+    p = ((char *)p) - align;
+    sz += align;
+    dprintf(("dbgmem: %p, %ld, %d\n", p, (long)sz, prot));
+    int err = mprotect(p, sz, prot);
+    assert(err == 0);
+}
+
+void *stm_malloc(size_t sz)
+{
+    pthread_mutex_lock(&malloc_mutex);
+
+    if (zone_current == NULL) {
+        zone_start = mmap(NULL, MMAP_TOTAL, PROT_NONE,
+                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (zone_start == NULL || zone_start == MAP_FAILED) {
+            stm_fatalerror("not enough memory: mmap() failed\n");
+        }
+        zone_current = zone_start;
+        zone_end = zone_start + MMAP_TOTAL;
+        assert((MMAP_TOTAL % PAGE_SIZE) == 0);
+        _stm_dbgmem(zone_start, MMAP_TOTAL, PROT_NONE);
+    }
+
+    size_t nb_pages = (sz + PAGE_SIZE - 1) / PAGE_SIZE + 1;
+    char *result = zone_current;
+    zone_current += nb_pages * PAGE_SIZE;
+    if (zone_current > zone_end) {
+        stm_fatalerror("dbgmem.c: %ld MB of memory have been exausted\n",
+                       (long)(MMAP_TOTAL / (1024*1024)));
+    }
+    pthread_mutex_unlock(&malloc_mutex);
+
+    result += (-sz) & (PAGE_SIZE-1);
+    assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
+    _stm_dbgmem(result, sz, PROT_READ | PROT_WRITE);
+
+    long i, base = (result - zone_start) / PAGE_SIZE;
+    for (i = 0; i < nb_pages; i++)
+        accessible_pages[base + i] = 42;
+
+    dprintf(("stm_malloc(%zu): %p\n", sz, result));
+    assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
+    return result;
+}
+
+void stm_free(void *p, size_t sz)
+{
+    assert(((intptr_t)((char *)p + sz) & (PAGE_SIZE-1)) == 0);
+
+    size_t nb_pages = (sz + PAGE_SIZE - 1) / PAGE_SIZE + 1;
+    long i, base = ((char *)p - zone_start) / PAGE_SIZE;
+    assert(0 <= base && base < (MMAP_TOTAL / PAGE_SIZE));
+    for (i = 0; i < nb_pages; i++) {
+        assert(accessible_pages[base + i] == 42);
+        accessible_pages[base + i] = -1;
+    }
+    memset(p, 0xDD, sz);
+    _stm_dbgmem(p, sz, PROT_NONE);
+}
+
+int _stm_can_access_memory(char *p)
+{
+    long base = ((char *)p - zone_start) / PAGE_SIZE;
+    assert(0 <= base && base < (MMAP_TOTAL / PAGE_SIZE));
+    return accessible_pages[base] == 42;
+}
+
+/************************************************************/
+#endif
+
+
+void stm_clear_large_memory_chunk(void *base, size_t size,
+                                  size_t already_cleared)
+{
+    char *baseaddr = base;
+
+    if (size > 2 * PAGE_SIZE) {
+        int lowbits = ((intptr_t)baseaddr) & (PAGE_SIZE-1);
+        if (lowbits) {   /*  clear the initial misaligned part, if any */
+            int partpage = PAGE_SIZE - lowbits;
+            memset(baseaddr, 0, partpage);
+            baseaddr += partpage;
+            size -= partpage;
+        }
+        /* 'already_cleared' bytes at the end are assumed to be already
+           cleared.  Reduce 'size' accordingly, but avoid getting a
+           misaligned 'size'. */
+        size_t length = size & (-PAGE_SIZE);
+        if (already_cleared > (size - length)) {
+            already_cleared -= (size - length);
+            already_cleared &= -PAGE_SIZE;
+            length -= already_cleared;
+            size = length;
+            already_cleared = 0;
+        }
+
+        int err = madvise(baseaddr, length, MADV_DONTNEED);
+        if (err == 0) {   /*  madvise() worked */
+            baseaddr += length;
+            size -= length;
+        }
+    }
+    if (size > already_cleared) { /* clear the final misaligned part, if any */
+        memset(baseaddr, 0, size - already_cleared);
+    }
+}
diff --git a/rpython/translator/stm/src_stm/dbgmem.h b/rpython/translator/stm/src_stm/dbgmem.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/dbgmem.h
@@ -0,0 +1,22 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+#ifndef _SRCSTM_DBGMEM_H
+#define _SRCSTM_DBGMEM_H
+
+
+#ifdef _GC_DEBUG
+
+void *stm_malloc(size_t);
+void stm_free(void *, size_t);
+int _stm_can_access_memory(char *);
+
+#else
+
+#define stm_malloc(sz)    malloc(sz)
+#define stm_free(p,sz)    free(p)
+
+#endif
+
+void stm_clear_large_memory_chunk(void *, size_t, size_t);
+
+
+#endif
diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/et.c
@@ -0,0 +1,1619 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+/* -*- c-basic-offset: 2 -*- */
+
+/* XXX assumes that time never wraps around (in a 'long'), which may be
+ * correct on 64-bit machines but not on 32-bit machines if the process
+ * runs for long enough.
+ */
+#include "stmimpl.h"
+
+
+__thread struct tx_descriptor *thread_descriptor = NULL;
+
+/* 'global_cur_time' is normally a multiple of 2, except when we turn
+   a transaction inevitable: we then add 1 to it. */
+static revision_t global_cur_time = 2;
+
+/* a negative odd number that identifies the currently running
+   transaction within the thread. */
+__thread revision_t stm_private_rev_num;
+
+
+revision_t stm_global_cur_time(void)  /* for tests */
+{
+  return global_cur_time;
+}
+revision_t get_private_rev_num(void)        /* for tests */
+{
+  return stm_private_rev_num;
+}
+struct tx_descriptor *stm_thread_descriptor(void)  /* for tests */
+{
+  return thread_descriptor;
+}
+static int is_private(gcptr P)
+{
+  return (P->h_revision == stm_private_rev_num) ||
+    (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+}
+int _stm_is_private(gcptr P)
+{
+  return is_private(P);
+}
+void stm_clear_read_cache(void)
+{
+  fxcache_clear(&thread_descriptor->recent_reads_cache);
+}
+
+/************************************************************/
+
+static void ValidateNow(struct tx_descriptor *);
+static void CancelLocks(struct tx_descriptor *d);
+
+static _Bool is_inevitable(struct tx_descriptor *d)
+{
+  /* Assert that we are running a transaction.
+   *      Returns True if this transaction is inevitable. */
+  assert(d->active == 1 + !d->setjmp_buf);
+  return d->active == 2;
+}
+
+static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
+
+static void inev_mutex_release(void)
+{
+  pthread_mutex_unlock(&mutex_inevitable);
+}
+
+static void inev_mutex_acquire(struct tx_descriptor *d)
+{   /* must save roots around this call */
+  stm_stop_sharedlock();
+  pthread_mutex_lock(&mutex_inevitable);
+  stm_start_sharedlock();
+
+  if (d->active < 0)
+    {
+      inev_mutex_release();
+      AbortNowIfDelayed();
+      abort();   /* unreachable */
+    }
+}
+
+/************************************************************/
+
+gcptr stm_DirectReadBarrier(gcptr G)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  gcptr P = G;
+  revision_t v;
+
+ restart_all:
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+    {
+      assert(IS_POINTER(P->h_revision));   /* pointer to the backup copy */
+
+      /* check P->h_revision->h_revision: if a pointer, then it means
+         the backup copy has been stolen into a public object and then
+         modified by some other thread.  Abort. */
+      if (IS_POINTER(((gcptr)P->h_revision)->h_revision))
+        AbortTransaction(ABRT_STOLEN_MODIFIED);
+
+      goto add_in_recent_reads_cache;
+    }
+  /* else, for the rest of this function, we can assume that P was not
+     a private copy */
+
+  if (P->h_tid & GCFLAG_PUBLIC)
+    {
+      /* follow the chained list of h_revision's as long as they are
+         regular pointers.  We will only find more public objects
+         along this chain.
+      */
+    restart_all_public:
+      assert(P->h_tid & GCFLAG_PUBLIC);
+      v = ACCESS_ONCE(P->h_revision);
+      if (IS_POINTER(v))  /* "is a pointer", "has a more recent revision" */
+        {
+        retry:
+          if (v & 2)
+            goto follow_stub;
+
+          gcptr P_prev = P;
+          P = (gcptr)v;
+          assert((P->h_tid & GCFLAG_PUBLIC) ||
+                 (P_prev->h_tid & GCFLAG_NURSERY_MOVED));
+
+          v = ACCESS_ONCE(P->h_revision);
+
+          if (IS_POINTER(v))
+            {
+              if (v & 2)
+                goto follow_stub;
+
+              /* we update P_prev->h_revision as a shortcut */
+              /* XXX check if this really gives a worse performance than only
+                 doing this write occasionally based on a counter in d */
+              P_prev->h_revision = v;
+              P = (gcptr)v;
+              v = ACCESS_ONCE(P->h_revision);
+              if (IS_POINTER(v))
+                goto retry;
+            }
+
+          /* We reach this point if P != G only.  Check again the
+             read_barrier_cache: if P now hits the cache, just return it
+          */
+          if (FXCACHE_AT(P) == P)
+            {
+              dprintf(("read_barrier: %p -> %p fxcache\n", G, P));
+              return P;
+            }
+        }
+
+      /* If we land on a P with GCFLAG_PUBLIC_TO_PRIVATE, it might be
+         because *we* have an entry in d->public_to_private.  (It might
+         also be someone else.)
+      */
+      if (P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE)
+        {
+          wlog_t *item;
+        retry_public_to_private:;
+          G2L_FIND(d->public_to_private, P, item, goto no_private_obj);
+
+          /* We have a key in 'public_to_private'.  The value is the
+             corresponding private object. */
+          P = item->val;
+          assert(!(P->h_tid & GCFLAG_PUBLIC));
+          assert(is_private(P));
+          dprintf(("read_barrier: %p -> %p public_to_private\n", G, P));
+          return P;
+
+        no_private_obj:
+          /* Key not found.  It might be because there really is none, or
+             because we still have it waiting in 'stolen_objects'. */
+          if (d->public_descriptor->stolen_objects.size > 0)
+            {
+              spinlock_acquire(d->public_descriptor->collection_lock, 'N');
+              stm_normalize_stolen_objects(d);
+              spinlock_release(d->public_descriptor->collection_lock);
+              goto retry_public_to_private;
+            }
+        }
+
+      /* The answer is a public object.  Is it too recent? */
+      if (UNLIKELY(v > d->start_time))
+        {
+          if (v >= LOCKED)
+            {
+              SpinLoop(SPLP_LOCKED_INFLIGHT);
+              goto restart_all_public; // spinloop until it is no longer LOCKED
+            }
+          ValidateNow(d);                  // try to move start_time forward
+          goto restart_all_public;         // restart searching from P
+        }
+      dprintf(("read_barrier: %p -> %p public\n", G, P));
+    }
+  else
+    {
+      /* Not private and not public: it's a protected object
+       */
+      dprintf(("read_barrier: %p -> %p protected\n", G, P));
+
+      /* The risks are not high, but in parallel it's possible for the
+         object to be stolen by another thread and become public, after
+         which it can be outdated by another commit.  So the following
+         assert can actually fail in that case. */
+      /*assert(P->h_revision & 1);*/
+    }
+
+  dprintf(("readobj: %p\n", P));
+  assert(!(P->h_tid & GCFLAG_STUB));
+  gcptrlist_insert(&d->list_of_read_objects, P);
+
+ add_in_recent_reads_cache:
+  /* The risks are that the following assert fails, because the flag was
+     added just now by a parallel thread during stealing... */
+  /*assert(!(P->h_tid & GCFLAG_NURSERY_MOVED));*/
+  fxcache_add(&d->recent_reads_cache, P);
+  return P;
+
+ follow_stub:;
+  /* We know that P is a stub object, because only stubs can have
+     an h_revision that is == 2 mod 4.
+  */
+  struct tx_public_descriptor *foreign_pd = STUB_THREAD(P);
+  if (foreign_pd == d->public_descriptor)
+    {
+      /* Same thread: dereference the pointer directly.  It's possible
+         we reach any kind of object, even a public object, in case it
+         was stolen.  So we just repeat the whole procedure. */
+      P = (gcptr)(v - 2);
+      dprintf(("read_barrier: %p -> %p via stub\n  ", G, P));
+
+      if (UNLIKELY((P->h_revision != stm_private_rev_num) &&
+                   (FXCACHE_AT(P) != P)))
+        goto restart_all;
+
+      return P;
+    }
+  else
+    {
+      /* stealing */
+      dprintf(("read_barrier: %p -> stealing %p...\n  ", G, P));
+      stm_steal_stub(P);
+
+      assert(P->h_tid & GCFLAG_PUBLIC);
+      goto restart_all_public;
+    }
+}
+
+static gcptr _match_public_to_private(gcptr P, gcptr pubobj, gcptr privobj,
+                                      int from_stolen)
+{
+  gcptr org_pubobj = pubobj;
+  while ((pubobj->h_revision & 3) == 0)
+    {
+      assert(pubobj != P);
+      pubobj = (gcptr)pubobj->h_revision;
+    }
+  if (pubobj == P || ((P->h_revision & 3) == 2 &&
+                      pubobj->h_revision == P->h_revision))
+    {
+      assert(!(org_pubobj->h_tid & GCFLAG_STUB));
+      assert(!(privobj->h_tid & GCFLAG_PUBLIC));
+      assert(is_private(privobj));
+      if (P != org_pubobj)
+        dprintf(("| actually %p ", org_pubobj));
+      if (from_stolen)
+        dprintf(("-stolen"));
+      else
+        assert(org_pubobj->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
+      dprintf(("-public_to_private-> %p private\n", privobj));
+      return privobj;
+    }
+  return NULL;
+}
+
+static gcptr _find_public_to_private(gcptr P)
+{
+  gcptr R;
+  wlog_t *item;
+  struct tx_descriptor *d = thread_descriptor;
+
+  G2L_LOOP_FORWARD(d->public_to_private, item)
+    {
+      assert(item->addr->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
+      R = _match_public_to_private(P, item->addr, item->val, 0);
+      if (R != NULL)
+        return R;
+
+    } G2L_LOOP_END;
+
+  long i, size = d->public_descriptor->stolen_objects.size;
+  gcptr *items = d->public_descriptor->stolen_objects.items;
+
+  for (i = 0; i < size; i += 2)
+    {
+      if (items[i + 1] == NULL)
+        continue;
+      R = _match_public_to_private(P, items[i], items[i + 1], 1);
+      if (R != NULL)
+        return R;
+    }
+
+  return NULL;
+}
+
+static void _check_flags(gcptr P)
+{
+#ifndef NDEBUG
+  struct tx_descriptor *d = thread_descriptor;
+  if (P->h_tid & GCFLAG_STUB)
+    {
+      dprintf(("S"));
+    }
+  int is_old = (P->h_tid & GCFLAG_OLD) != 0;
+  int in_nurs = (d->nursery_base <= (char*)P && ((char*)P) < d->nursery_end);
+  if (in_nurs)
+    {
+      assert(!is_old);
+      dprintf(("Y "));
+    }
+  else
+    {
+      assert(is_old);
+      dprintf(("O "));
+    }
+#endif
+}
+
+gcptr _stm_nonrecord_barrier(gcptr P)
+{
+  /* follows the logic in stm_DirectReadBarrier() */
+  struct tx_descriptor *d = thread_descriptor;
+  revision_t v;
+
+  dprintf(("_stm_nonrecord_barrier: %p ", P));
+  _check_flags(P);
+
+ restart_all:
+  if (P->h_revision == stm_private_rev_num)
+    {
+      /* private */
+      dprintf(("private\n"));
+      return P;
+    }
+
+  if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+    {
+      /* private too, with a backup copy */
+      assert(IS_POINTER(P->h_revision));
+      dprintf(("private_from_protected\n"));
+      return P;
+    }
+
+  if (P->h_tid & GCFLAG_PUBLIC)
+    {
+      dprintf(("public "));
+
+      while (v = P->h_revision, IS_POINTER(v))
+        {
+          if (P->h_tid & GCFLAG_NURSERY_MOVED)
+            dprintf(("nursery_moved "));
+
+          if (v & 2)
+            {
+              dprintf(("stub "));
+              gcptr L = _find_public_to_private(P);
+              if (L != NULL)
+                return L;
+              goto follow_stub;
+            }
+
+          P = (gcptr)v;
+          assert(P->h_tid & GCFLAG_PUBLIC);
+          dprintf(("-> %p public ", P));
+          _check_flags(P);
+        }
+
+      gcptr L = _find_public_to_private(P);
+      if (L != NULL)
+        return L;
+
+      if (UNLIKELY(v > d->start_time))
+        {
+          dprintf(("too recent!\n"));
+          return NULL;   // object too recent
+        }
+      dprintf(("\n"));
+    }
+  else
+    {
+      dprintf(("protected\n"));
+    }
+  return P;
+
+ follow_stub:;
+  if (STUB_THREAD(P) == d->public_descriptor)
+    {
+      P = (gcptr)(v - 2);
+      dprintf(("-> %p ", P));
+      _check_flags(P);
+    }
+  else
+    {
+      P = (gcptr)(v - 2);
+      /* cannot _check_flags(P): foreign! */
+      dprintf(("-foreign-> %p ", P));
+      if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+        {
+          P = (gcptr)P->h_revision;     /* the backup copy */
+          /* cannot _check_flags(P): foreign! */
+          dprintf(("-backup-> %p ", P));
+        }
+      if (!(P->h_tid & GCFLAG_PUBLIC))
+        {
+          dprintf(("protected by someone else!\n"));
+          return (gcptr)-1;
+        }
+    }
+  /* cannot _check_flags(P): foreign! */
+  goto restart_all;
+}
+
+#if 0
+void *stm_DirectReadBarrierFromR(void *G1, void *R_Container1, size_t offset)
+{
+  return _direct_read_barrier((gcptr)G1, (gcptr)R_Container1, offset);
+}
+#endif
+
+gcptr stm_RepeatReadBarrier(gcptr O)
+{
+  abort();//XXX
+#if 0
+  // LatestGlobalRevision(O) would either return O or abort
+  // the whole transaction, so omitting it is not wrong
+  struct tx_descriptor *d = thread_descriptor;
+  gcptr L;
+  wlog_t *entry;
+  G2L_FIND(d->global_to_local, O, entry, return O);
+  L = entry->val;
+  assert(L->h_revision == stm_local_revision);
+  return L;
+#endif
+}
+
+static gcptr LocalizeProtected(struct tx_descriptor *d, gcptr P)
+{
+  gcptr B;
+
+  assert(P->h_revision != stm_private_rev_num);
+  assert(P->h_revision & 1);
+  assert(!(P->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
+  assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
+  assert(!(P->h_tid & GCFLAG_STUB));
+  assert(!(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
+
+  B = stmgc_duplicate_old(P);
+  B->h_tid |= GCFLAG_BACKUP_COPY;
+  B->h_tid &= ~GCFLAG_HAS_ID;
+  if (!(P->h_original) && (P->h_tid & GCFLAG_OLD)) {
+    /* if P is old, it must be the original
+       if P is young, it will create a shadow original later
+       or it's getting decided when backup gets stolen.
+    */
+    B->h_original = (revision_t)P;
+  }
+  
+  P->h_tid |= GCFLAG_PRIVATE_FROM_PROTECTED;
+  P->h_revision = (revision_t)B;
+
+  gcptrlist_insert(&d->private_from_protected, P);
+  dprintf(("private_from_protected: insert %p (backup %p)\n", P, B));
+
+  return P;   /* always returns its arg: the object is converted in-place */
+}
+
+static gcptr LocalizePublic(struct tx_descriptor *d, gcptr R)
+{
+  assert(R->h_tid & GCFLAG_PUBLIC);
+  assert(!(R->h_tid & GCFLAG_NURSERY_MOVED));
+
+#ifdef _GC_DEBUG
+  wlog_t *entry;
+  G2L_FIND(d->public_to_private, R, entry, goto not_found);
+  stm_fatalerror("R is already in public_to_private\n");
+ not_found:
+#endif
+
+  R->h_tid |= GCFLAG_PUBLIC_TO_PRIVATE;
+
+  /* note that stmgc_duplicate() usually returns a young object, but may
+     return an old one if the nursery is full at this moment. */
+  gcptr L = stmgc_duplicate(R);
+  if (!(L->h_original) || L->h_tid & GCFLAG_PREBUILT_ORIGINAL) {
+    /* if we don't have an original object yet, it must be the 
+       old public R 
+       Also, prebuilt objects may have a predefined hash stored
+       in the h_original. -> point to the original copy on copies
+       of the prebuilt.
+    */
+    assert(R->h_tid & GCFLAG_OLD); // if not, force stm_id??
+    L->h_original = (revision_t)R;
+  }
+
+  assert(!(L->h_tid & GCFLAG_BACKUP_COPY));
+  assert(!(L->h_tid & GCFLAG_STUB));
+  assert(!(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED));
+  L->h_tid &= ~(GCFLAG_VISITED           |
+                GCFLAG_PUBLIC            |
+                GCFLAG_PREBUILT_ORIGINAL |
+                GCFLAG_PUBLIC_TO_PRIVATE |
+                GCFLAG_WRITE_BARRIER     |
+                0);
+  L->h_revision = stm_private_rev_num;
+  assert(stm_private_rev_num < 0);
+  assert(stm_private_rev_num & 1);
+  g2l_insert(&d->public_to_private, R, L);
+  dprintf(("write_barrier: adding %p -> %p to public_to_private\n",
+           R, L));
+
+  /* must remove R from the read_barrier_cache, because returning R is no
+     longer a valid result */
+  fxcache_remove(&d->recent_reads_cache, R);
+
+  return L;
+}
+
+static inline void record_write_barrier(gcptr P)
+{
+  if (P->h_tid & GCFLAG_WRITE_BARRIER)
+    {
+      P->h_tid &= ~GCFLAG_WRITE_BARRIER;
+      gcptrlist_insert(&thread_descriptor->old_objects_to_trace, P);
+    }
+}
+
+gcptr stm_WriteBarrier(gcptr P)
+{
+  if (is_private(P))
+    {
+      /* If we have GCFLAG_WRITE_BARRIER in P, then list it into
+         old_objects_to_trace: it's a private object that may be
+         modified by the program after we return, and the mutation may
+         be to write young pointers (in fact it's a common case).
+      */
+      record_write_barrier(P);
+      return P;
+    }
+
+  gcptr R, W;
+  R = stm_read_barrier(P);
+
+  if (is_private(R))
+    {
+      record_write_barrier(R);
+      return R;
+    }
+
+  struct tx_descriptor *d = thread_descriptor;
+  assert(d->active >= 1);
+
+  /* We need the collection_lock for the sequel; this is required notably
+     because we're about to edit flags on a protected object.
+  */
+  spinlock_acquire(d->public_descriptor->collection_lock, 'L');
+  if (d->public_descriptor->stolen_objects.size != 0)
+    stm_normalize_stolen_objects(d);
+
+  if (R->h_tid & GCFLAG_PUBLIC)
+    {
+      /* Make and return a new (young) private copy of the public R.
+         Add R into the list 'public_with_young_copy', unless W is
+         actually an old object, in which case we need to record W.
+      */
+      if (R->h_tid & GCFLAG_NURSERY_MOVED)
+        {
+          /* Bah, the object turned into this kind of stub, possibly
+             while we were waiting for the collection_lock, because it
+             was stolen by someone else.  Use R->h_revision instead. */
+          assert(IS_POINTER(R->h_revision));
+          R = (gcptr)R->h_revision;
+          assert(R->h_tid & GCFLAG_PUBLIC);
+        }
+      assert(R->h_tid & GCFLAG_OLD);
+      W = LocalizePublic(d, R);
+      assert(is_private(W));
+
+      if (W->h_tid & GCFLAG_OLD)
+        gcptrlist_insert(&d->old_objects_to_trace, W);
+      else
+        gcptrlist_insert(&d->public_with_young_copy, R);
+    }
+  else
+    {
+      /* Turn the protected copy in-place into a private copy.  If it's
+         an old object that still has GCFLAG_WRITE_BARRIER, then we must
+         also record it in the list 'old_objects_to_trace'. */
+      W = LocalizeProtected(d, R);
+      assert(is_private(W));
+      record_write_barrier(W);
+    }
+
+  spinlock_release(d->public_descriptor->collection_lock);
+
+  dprintf(("write_barrier: %p -> %p -> %p\n", P, R, W));
+
+  return W;
+}
+
+gcptr stm_get_private_from_protected(long index)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  if (index < gcptrlist_size(&d->private_from_protected))
+    return d->private_from_protected.items[index];
+  return NULL;
+}
+
+gcptr stm_get_read_obj(long index)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  if (index < gcptrlist_size(&d->list_of_read_objects))
+    return d->list_of_read_objects.items[index];
+  return NULL;
+}
+
+/************************************************************/
+
+static revision_t GetGlobalCurTime(struct tx_descriptor *d)
+{
+  assert(!is_inevitable(d));    // must not be myself inevitable
+  return ACCESS_ONCE(global_cur_time) & ~1;
+}
+
+static _Bool ValidateDuringTransaction(struct tx_descriptor *d,
+                                       _Bool during_commit)
+{
+  long i, size = d->list_of_read_objects.size;
+  gcptr *items = d->list_of_read_objects.items;
+
+  for (i=0; i<size; i++)
+    {
+      gcptr R = items[i];
+      revision_t v;
+    retry:
+      v = ACCESS_ONCE(R->h_revision);
+      if (IS_POINTER(v))  /* "is a pointer", i.e. has a more recent revision */
+        {
+          if (R->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+            {
+              /* such an object R might be listed in list_of_read_objects
+                 before it was turned from protected to private */
+              if (((gcptr)v)->h_tid & GCFLAG_PUBLIC)
+                {
+                  /* The backup was stolen, but maybe not modified
+                     afterwards.  Check it. */
+                  R = (gcptr)v;
+                  goto retry;
+                }
+              else
+                {
+                  /* The backup was not stolen, everything's fine */
+                  continue;
+                }
+            }
+          else if ((R->h_tid & (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
+                            == (GCFLAG_PUBLIC | GCFLAG_NURSERY_MOVED))
+            {
+              /* such an object is identical to the one it points to
+               (stolen protected young object with h_revision pointing
+               to the new copy) */
+              R = (gcptr)v;
+              goto retry;
+            }
+          else
+            {
+              dprintf(("validation failed: "
+                       "%p has a more recent revision\n", R));
+              return 0;
+            }
+        }
+      if (v >= LOCKED)            // locked
+        {
+          if (!during_commit)
+            {
+              assert(v != d->my_lock);    // we don't hold any lock
+              /* spinloop until the other thread releases its lock */
+              SpinLoop(SPLP_LOCKED_VALIDATE);
+              goto retry;
+            }
+          else
+            {
+              if (v != d->my_lock)         // not locked by me: conflict
+                {
+                  /* It's delicate here to do a spinloop rather than
+                     just aborting.
+
+                     A case that can occur: two threads A and B are both
+                     committing, thread A locked object a, thread B
+                     locked object b, and then thread A tries to
+                     validate the reads it did on object b and
+                     vice-versa.  In this case both threads cannot
+                     commit, but if they both enter the SpinLoop()
+                     here, then they will livelock.
+
+                     Another case: thread A might be blocked in this
+                     spinloop, while thread B is blocked in the
+                     SpinLoop(SPLP_LOCKED_COMMIT) below.
+
+                     For now we always abort.
+                  */
+                  dprintf(("validation failed: "
+                           "%p is locked by another thread\n", R));
+                  return 0;
+                }
+            }
+        }
+    }
+  return 1;
+}
+
+static void ValidateNow(struct tx_descriptor *d)
+{
+  d->start_time = GetGlobalCurTime(d);   // copy from the global time
+  dprintf(("et.c: ValidateNow: %ld\n", (long)d->start_time));
+
+  /* subtle: we have to normalize stolen objects, because doing so
+     might add a few extra objects in the list_of_read_objects */
+  if (d->public_descriptor->stolen_objects.size != 0)
+    {
+      spinlock_acquire(d->public_descriptor->collection_lock, 'N');
+      stm_normalize_stolen_objects(d);
+      spinlock_release(d->public_descriptor->collection_lock);
+    }
+
+  if (!ValidateDuringTransaction(d, 0))
+    AbortTransaction(ABRT_VALIDATE_INFLIGHT);
+}
+
+/************************************************************/
+
+void SpinLoop(int num)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  assert(d->active >= 1);
+  assert(num < SPINLOOP_REASONS);
+  d->num_spinloops[num]++;
+  smp_spinloop();
+}
+
+#if 0
+size_t _stm_decode_abort_info(struct tx_descriptor *d, long long elapsed_time,
+                              int abort_reason, char *output);
+#endif
+
+void AbortPrivateFromProtected(struct tx_descriptor *d);
+
+void AbortTransaction(int num)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  unsigned long limit;
+  struct timespec now;
+  long long elapsed_time;
+
+  /* acquire the lock, but don't double-acquire it if already committing */
+  if (d->public_descriptor->collection_lock != 'C') {
+    spinlock_acquire(d->public_descriptor->collection_lock, 'C');
+    if (d->public_descriptor->stolen_objects.size != 0)
+      stm_normalize_stolen_objects(d);
+  }
+
+
+  assert(d->active != 0);
+  assert(!is_inevitable(d));
+  assert(num < ABORT_REASONS);
+  d->num_aborts[num]++;
+
+  CancelLocks(d);
+
+  /* compute the elapsed time */
+  if (d->start_real_time.tv_nsec != -1 &&
+      clock_gettime(CLOCK_MONOTONIC, &now) >= 0) {
+    elapsed_time = now.tv_sec - d->start_real_time.tv_sec;
+    elapsed_time *= 1000000000;
+    elapsed_time += now.tv_nsec - d->start_real_time.tv_nsec;
+    if (elapsed_time < 1)
+      elapsed_time = 1;
+  }
+  else {
+    elapsed_time = 1;
+  }
+
+#if 0
+  size_t size;
+  if (elapsed_time >= d->longest_abort_info_time)
+    {
+      /* decode the 'abortinfo' and produce a human-readable summary in
+         the string 'longest_abort_info' */
+      size = _stm_decode_abort_info(d, elapsed_time, num, NULL);
+      free(d->longest_abort_info);
+      d->longest_abort_info = malloc(size);
+      if (d->longest_abort_info == NULL)
+        d->longest_abort_info_time = 0;   /* out of memory! */
+      else
+        {
+          if (_stm_decode_abort_info(d, elapsed_time,
+                                     num, d->longest_abort_info) != size)
+            stm_fatalerror("during stm abort: object mutated unexpectedly\n");
+
+          d->longest_abort_info_time = elapsed_time;
+        }
+    }
+#endif
+
+#if 0
+  /* run the undo log in reverse order, cancelling the values set by
+     stm_ThreadLocalRef_LLSet(). */
+  if (d->undolog.size > 0) {
+      gcptr *item = d->undolog.items;
+      long i;
+      for (i=d->undolog.size; i>=0; i-=2) {
+          void **addr = (void **)(item[i-2]);
+          void *oldvalue = (void *)(item[i-1]);
+          *addr = oldvalue;
+      }
+  }
+#endif
+
+  /* upon abort, set the reads size limit to 94% of how much was read
+     so far.  This should ensure that, assuming the retry does the same
+     thing, it will commit just before it reaches the conflicting point.
+     Note that we should never *increase* the read length limit here. */
+  limit = d->count_reads;
+  if (limit > d->reads_size_limit_nonatomic) {  /* can occur if atomic */
+      limit = d->reads_size_limit_nonatomic;
+  }
+  if (limit > 0) {
+      limit -= (limit >> 4);
+      d->reads_size_limit_nonatomic = limit;
+  }
+
+  AbortPrivateFromProtected(d);
+  gcptrlist_clear(&d->list_of_read_objects);
+  g2l_clear(&d->public_to_private);
+
+  /* 'old_thread_local_obj' contains the old value from stm_thread_local_obj,
+     but only when the transaction can be aborted; when it is inevitable
+     old_thread_local_obj will be reset to NULL. */
+  assert(d->thread_local_obj_ref = &stm_thread_local_obj);
+  stm_thread_local_obj = d->old_thread_local_obj;
+  d->old_thread_local_obj = NULL;
+
+  /* release the lock */
+  spinlock_release(d->public_descriptor->collection_lock);
+
+  dprintf(("\n"
+          "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+          "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+          "!!!!!!!!!!!!!!!!!!!!!  [%lx] abort %d\n"
+          "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+          "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+          "\n", (long)d->public_descriptor_index, num));
+  if (num != ABRT_MANUAL && d->max_aborts >= 0 && !d->max_aborts--)
+    stm_fatalerror("unexpected abort!\n");
+
+  // notifies the CPU that we're potentially in a spin loop
+  SpinLoop(SPLP_ABORT);
+  // jump back to the setjmp_buf (this call does not return)
+  d->active = 0;
+  stm_stop_sharedlock();
+  longjmp(*d->setjmp_buf, 1);
+}
+
+void AbortTransactionAfterCollect(struct tx_descriptor *d, int reason)
+{
+  if (d->active >= 0)
+    {
+      dprintf(("abort %d after collect!\n", reason));
+      assert(d->active == 1);   /* not 2, which means inevitable */
+      d->active = -reason;
+    }
+  assert(d->active < 0);
+}
+
+void AbortNowIfDelayed(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  if (d->active < 0)
+    {
+      int reason = -d->active;
+      d->active = 1;
+      AbortTransaction(reason);
+    }
+}
+
+/************************************************************/
+
+static void update_reads_size_limit(struct tx_descriptor *d)
+{
+  /* 'reads_size_limit' is set to ULONG_MAX if we are atomic; else
+     we copy the value from reads_size_limit_nonatomic. */
+  d->reads_size_limit = d->atomic ? ULONG_MAX : d->reads_size_limit_nonatomic;
+}
+
+static void init_transaction(struct tx_descriptor *d)
+{
+  assert(d->active == 0);
+  stm_start_sharedlock();
+  assert(d->active == 0);
+
+  if (clock_gettime(CLOCK_MONOTONIC, &d->start_real_time) < 0) {
+    d->start_real_time.tv_nsec = -1;
+  }
+  assert(d->list_of_read_objects.size == 0);
+  assert(d->private_from_protected.size == 0);
+  assert(d->num_private_from_protected_known_old == 0);
+  assert(d->num_read_objects_known_old == 0);
+  assert(!g2l_any_entry(&d->public_to_private));
+  assert(d->old_thread_local_obj == NULL);
+
+  d->count_reads = 1;
+  fxcache_clear(&d->recent_reads_cache);
+#if 0
+  gcptrlist_clear(&d->undolog);
+  gcptrlist_clear(&d->abortinfo);
+#endif
+}
+
+void BeginTransaction(jmp_buf* buf)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  init_transaction(d);
+  d->active = 1;
+  d->setjmp_buf = buf;
+  d->old_thread_local_obj = stm_thread_local_obj;
+  d->start_time = GetGlobalCurTime(d);
+  update_reads_size_limit(d);
+}
+
+static void AcquireLocks(struct tx_descriptor *d)
+{
+  revision_t my_lock = d->my_lock;
+  wlog_t *item;
+
+  assert(d->public_descriptor->stolen_objects.size == 0);
+
+  if (!g2l_any_entry(&d->public_to_private))
+    return;
+
+  G2L_LOOP_FORWARD(d->public_to_private, item)
+    {
+      gcptr R = item->addr;
+      revision_t v;
+    retry:
+      assert(R->h_tid & GCFLAG_PUBLIC);
+      v = ACCESS_ONCE(R->h_revision);
+      if (IS_POINTER(v))     /* "has a more recent revision" */
+        {
+          assert(v != 0);
+          AbortTransaction(ABRT_COMMIT);
+        }
+      if (v >= LOCKED)         // already locked by someone else
+        {
+          // we can always spinloop here: deadlocks should be impossible,
+          // because G2L_LOOP_FORWARD should ensure a consistent ordering
+          // of the R's.
+          assert(v != my_lock);
+          SpinLoop(SPLP_LOCKED_COMMIT);
+          goto retry;
+        }
+      if (!bool_cas(&R->h_revision, v, my_lock))
+        goto retry;
+
+      gcptr L = item->val;
+      assert(L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED ?
+             L->h_revision == (revision_t)R :
+             L->h_revision == stm_private_rev_num);
+      assert(v != stm_private_rev_num);
+      assert(v & 1);
+      L->h_revision = v;   /* store temporarily this value here */
+
+    } G2L_LOOP_END;
+}
+
+static void CancelLocks(struct tx_descriptor *d)
+{
+  wlog_t *item;
+
+  if (!g2l_any_entry(&d->public_to_private))
+    return;
+
+  G2L_LOOP_FORWARD(d->public_to_private, item)
+    {
+      gcptr R = item->addr;
+      gcptr L = item->val;
+      if (L == NULL)
+        continue;
+
+      revision_t expected, v = L->h_revision;
+
+      if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
+        expected = (revision_t)R;
+      else
+        expected = stm_private_rev_num;
+
+      if (v == expected)
+        {
+          assert(R->h_revision != d->my_lock);
+          break;    /* done */
+        }
+
+      L->h_revision = expected;
+
+#ifdef DUMP_EXTRA
+      dprintf(("%p->h_revision = %p (CancelLocks)\n", R, (gcptr)v));
+#endif
+      assert(R->h_revision == d->my_lock);
+      ACCESS_ONCE(R->h_revision) = v;
+
+    } G2L_LOOP_END;
+}
+
+static pthread_mutex_t mutex_prebuilt_gcroots = PTHREAD_MUTEX_INITIALIZER;
+
+static void UpdateChainHeads(struct tx_descriptor *d, revision_t cur_time,
+                             revision_t localrev)
+{
+  wlog_t *item;
+  revision_t new_revision = cur_time + 1;     // make an odd number
+  assert(new_revision & 1);
+
+  if (!g2l_any_entry(&d->public_to_private))
+    return;
+
+  G2L_LOOP_FORWARD(d->public_to_private, item)
+    {
+      gcptr L = item->val;
+      assert(!(L->h_tid & GCFLAG_VISITED));
+      assert(!(L->h_tid & GCFLAG_PUBLIC_TO_PRIVATE));
+      assert(!(L->h_tid & GCFLAG_PREBUILT_ORIGINAL));
+      assert(!(L->h_tid & GCFLAG_NURSERY_MOVED));
+      assert(L->h_revision != localrev);   /* modified by AcquireLocks() */
+
+#ifdef DUMP_EXTRA
+      dprintf(("%p->h_revision = %p (UpdateChainHeads)\n",
+               L, (gcptr)new_revision));
+#endif
+      L->h_revision = new_revision;
+
+      gcptr stub = stm_stub_malloc(d->public_descriptor);
+      stub->h_tid = (L->h_tid & STM_USER_TID_MASK) | GCFLAG_PUBLIC
+                                                   | GCFLAG_STUB
+                                                   | GCFLAG_OLD;
+      stub->h_revision = ((revision_t)L) | 2;
+
+      assert(!(L->h_tid & GCFLAG_HAS_ID));
+      if (L->h_original) {
+        stub->h_original = L->h_original;
+      }
+      else if (L->h_tid & GCFLAG_OLD) {
+        stub->h_original = (revision_t)L;
+      }
+      else {
+        /* There shouldn't be a public, young object without
+           a h_original. They only come from stealing which
+           always sets h_original */
+        assert(0);
+        /* L->h_original = (revision_t)stub; */
+        /* if (L->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) { */
+        /*   ((gcptr)L->h_revision)->h_original = (revision_t)stub; */
+        /* } */
+      }
+      
+      item->val = stub;
+
+    } G2L_LOOP_END;
+
+  smp_wmb(); /* a memory barrier: make sure the new L->h_revisions are visible
+                from other threads before we change the R->h_revisions */
+
+  G2L_LOOP_FORWARD(d->public_to_private, item)
+    {
+      gcptr R = item->addr;
+      revision_t v = (revision_t)item->val;
+
+      assert(R->h_tid & GCFLAG_PUBLIC);
+      assert(R->h_tid & GCFLAG_PUBLIC_TO_PRIVATE);
+      assert(!(R->h_tid & GCFLAG_NURSERY_MOVED));
+      assert(R->h_revision != localrev);
+
+#ifdef DUMP_EXTRA
+      dprintf(("%p->h_revision = %p (stub to %p)\n",
+               R, (gcptr)v, (gcptr)item->val->h_revision));
+#endif
+      ACCESS_ONCE(R->h_revision) = v;
+
+      if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)
+        {
+          /* cannot possibly get here more than once for a given value of R */
+          pthread_mutex_lock(&mutex_prebuilt_gcroots);
+          gcptrlist_insert(&stm_prebuilt_gcroots, R);
+          pthread_mutex_unlock(&mutex_prebuilt_gcroots);
+        }
+
+    } G2L_LOOP_END;
+
+  g2l_clear(&d->public_to_private);
+}
+
+void CommitPrivateFromProtected(struct tx_descriptor *d, revision_t cur_time)
+{
+  long i, size = d->private_from_protected.size;
+  gcptr *items = d->private_from_protected.items;
+  revision_t new_revision = cur_time + 1;     // make an odd number
+  assert(new_revision & 1);
+  assert(d->public_descriptor->stolen_objects.size == 0);
+
+  for (i = 0; i < size; i++)
+    {
+      gcptr P = items[i];
+      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+      P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
+
+      if (!IS_POINTER(P->h_revision))
+        {
+          /* This case occurs when a GCFLAG_PRIVATE_FROM_PROTECTED object
+             is stolen: it ends up as a value in 'public_to_private'.
+             Its h_revision is then mangled by AcquireLocks(). */
+          assert(P->h_revision != stm_private_rev_num);
+          continue;
+        }
+
+      gcptr B = (gcptr)P->h_revision;
+      P->h_revision = new_revision;
+
+      if (B->h_tid & GCFLAG_PUBLIC)
+        {
+          assert(!(P->h_tid & GCFLAG_HAS_ID));
+
+          /* B was stolen */
+          while (1)
+            {
+              revision_t v = ACCESS_ONCE(B->h_revision);
+              if (IS_POINTER(v))    /* "was modified" */
+                AbortTransaction(ABRT_STOLEN_MODIFIED);
+
+              if (bool_cas(&B->h_revision, v, (revision_t)P))
+                break;
+            }
+        }      
+      else
+        {
+          stmgcpage_free(B);
+          dprintf(("commit: free backup at %p\n", B));
+        }
+    };
+  gcptrlist_clear(&d->private_from_protected);
+  d->num_private_from_protected_known_old = 0;
+  d->num_read_objects_known_old = 0;
+  dprintf(("private_from_protected: clear (commit)\n"));
+}
+
+void AbortPrivateFromProtected(struct tx_descriptor *d)
+{
+  long i, size = d->private_from_protected.size;
+  gcptr *items = d->private_from_protected.items;
+
+  for (i = 0; i < size; i++)
+    {
+      gcptr P = items[i];
+      assert(P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED);
+      assert(IS_POINTER(P->h_revision));
+      P->h_tid &= ~GCFLAG_PRIVATE_FROM_PROTECTED;
+
+      gcptr B = (gcptr)P->h_revision;
+      assert(B->h_tid & GCFLAG_OLD);
+
+      if (B->h_tid & GCFLAG_PUBLIC)
+        {
+          assert(!(B->h_tid & GCFLAG_BACKUP_COPY));
+          P->h_tid |= GCFLAG_PUBLIC;
+          assert(!(P->h_tid & GCFLAG_HAS_ID));
+          if (!(P->h_tid & GCFLAG_OLD)) P->h_tid |= GCFLAG_NURSERY_MOVED;
+          /* P becomes a public outdated object.  It may create an
+             exception documented in doc-objects.txt: a public but young
+             object.  It's still fine because it should only be seen by
+             other threads during stealing, and as it's outdated,
+             stealing will follow its h_revision (to B).
+          */
+        }
+      else
+        {
+          /* copy the backup copy B back over the now-protected object P,
+             and then free B, which will not be used any more. */
+          size_t size = stmgc_size(B);
+          assert(B->h_tid & GCFLAG_BACKUP_COPY);
+          memcpy(((char *)P) + offsetof(struct stm_object_s, h_revision),
+                 ((char *)B) + offsetof(struct stm_object_s, h_revision),
+                 size - offsetof(struct stm_object_s, h_revision));
+          assert(!(P->h_tid & GCFLAG_BACKUP_COPY));
+          stmgcpage_free(B);
+          dprintf(("abort: free backup at %p\n", B));
+        }
+    };
+  gcptrlist_clear(&d->private_from_protected);
+  d->num_private_from_protected_known_old = 0;
+  d->num_read_objects_known_old = 0;
+  dprintf(("private_from_protected: clear (abort)\n"));
+}
+
+void CommitTransaction(void)
+{   /* must save roots around this call */
+  revision_t cur_time;
+  struct tx_descriptor *d = thread_descriptor;
+  assert(d->active >= 1);
+
+  spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
+  if (d->public_descriptor->stolen_objects.size != 0)
+    stm_normalize_stolen_objects(d);
+  AcquireLocks(d);
+
+  if (is_inevitable(d))
+    {
+      // no-one else can have changed global_cur_time if I'm inevitable
+      cur_time = d->start_time;
+      if (!bool_cas(&global_cur_time, cur_time + 1, cur_time + 2))
+        {
+          stm_fatalerror("global_cur_time modified even though we are inev\n");
+        }
+      inev_mutex_release();
+    }
+  else
+    {
+      while (1)
+        {
+          cur_time = ACCESS_ONCE(global_cur_time);
+          if (cur_time & 1)
+            {                    // there is another inevitable transaction
+              CancelLocks(d);
+              spinlock_release(d->public_descriptor->collection_lock);
+              inev_mutex_acquire(d);   // wait until released
+              inev_mutex_release();
+              spinlock_acquire(d->public_descriptor->collection_lock, 'C');
+              if (d->public_descriptor->stolen_objects.size != 0)
+                stm_normalize_stolen_objects(d);
+
+              AcquireLocks(d);
+              continue;
+            }
+          if (bool_cas(&global_cur_time, cur_time, cur_time + 2))
+            break;
+        }
+      // validate (but skip validation if nobody else committed)
+      if (cur_time != d->start_time)
+        if (!ValidateDuringTransaction(d, 1))
+          AbortTransaction(ABRT_VALIDATE_COMMIT);
+    }
+
+  CommitPrivateFromProtected(d, cur_time);
+
+  /* we cannot abort any more from here */
+  d->setjmp_buf = NULL;
+  d->old_thread_local_obj = NULL;
+  gcptrlist_clear(&d->list_of_read_objects);
+
+  dprintf(("\n"
+          "*************************************\n"
+          "**************************************  committed %ld\n"
+          "*************************************\n",
+           (long)cur_time));
+
+  revision_t localrev = stm_private_rev_num;
+  //UpdateProtectedChainHeads(d, cur_time, localrev);
+  //smp_wmb();
+
+  revision_t newrev = -(cur_time + 1);
+  assert(newrev & 1);
+  ACCESS_ONCE(stm_private_rev_num) = newrev;
+  dprintf(("%p: stm_local_revision = %ld\n", d, (long)newrev));
+  assert(d->private_revision_ref == &stm_private_rev_num);
+  assert(d->read_barrier_cache_ref == &stm_read_barrier_cache);
+
+  UpdateChainHeads(d, cur_time, localrev);
+
+  spinlock_release(d->public_descriptor->collection_lock);
+  d->num_commits++;
+  d->active = 0;
+  stm_stop_sharedlock();
+}
+
+/************************************************************/
+
+static void make_inevitable(struct tx_descriptor *d)
+{
+  d->setjmp_buf = NULL;
+  d->old_thread_local_obj = NULL;
+  d->active = 2;
+  d->reads_size_limit_nonatomic = 0;
+  update_reads_size_limit(d);
+}
+
+static revision_t acquire_inev_mutex_and_mark_global_cur_time(
+                      struct tx_descriptor *d)
+{   /* must save roots around this call */
+  revision_t cur_time;
+
+  inev_mutex_acquire(d);
+  while (1)
+    {
+      cur_time = ACCESS_ONCE(global_cur_time);
+      assert((cur_time & 1) == 0);
+      if (bool_cas(&global_cur_time, cur_time, cur_time + 1))
+        break;
+      /* else try again */
+    }
+  return cur_time;
+}
+
+void BecomeInevitable(const char *why)
+{   /* must save roots around this call */
+  revision_t cur_time;
+  struct tx_descriptor *d = thread_descriptor;
+  if (d == NULL || d->active != 1)
+    return;  /* I am already inevitable, or not in a transaction at all
+                (XXX statically we should know when we're outside
+                a transaction) */
+
+  dprintf(("[%lx] inevitable: %s\n",
+           (long)d->public_descriptor_index, why));
+
+  cur_time = acquire_inev_mutex_and_mark_global_cur_time(d);
+  if (d->start_time != cur_time)
+    {
+      d->start_time = cur_time;
+      if (!ValidateDuringTransaction(d, 0))
+        {
+          global_cur_time = cur_time;   // revert from (cur_time + 1)
+          inev_mutex_release();
+          AbortTransaction(ABRT_VALIDATE_INEV);
+        }
+    }
+  make_inevitable(d);    /* cannot abort any more */
+}
+
+void BeginInevitableTransaction(void)
+{   /* must save roots around this call */
+  struct tx_descriptor *d = thread_descriptor;
+  revision_t cur_time;
+
+  init_transaction(d);
+  cur_time = acquire_inev_mutex_and_mark_global_cur_time(d);
+  d->start_time = cur_time;
+  make_inevitable(d);
+}
+
+/************************************************************/
+
+#if 0
+static _Bool _PtrEq_Globals(gcptr G1, gcptr G2)
+{
+  /* This is a mess, because G1 and G2 can be different pointers to "the
+     same" object, and it's hard to determine that.  Description of the
+     idealized problem: we have chained lists following the 'h_revision'
+     field from G1 and from G2, and we must return True if the chained
+     lists end in the same object G and False if they don't.
+
+     It's possible that G1 != G2 but G1->h_revision == G2->h_revision.
+     More complicated cases are also possible.  This occurs because of
+     random updates done in LatestGlobalRevision().  Note also that
+     other threads can do concurrent updates.
+
+     For now we simply use LatestGlobalRevision() and abort the current
+     transaction if we see a too recent object.
+  */
+  struct tx_descriptor *d = thread_descriptor;
+  if (G1->h_tid & GCFLAG_POSSIBLY_OUTDATED)
+    G1 = LatestGlobalRevision(d, G1);
+  if (G2->h_tid & GCFLAG_POSSIBLY_OUTDATED)
+    G2 = LatestGlobalRevision(d, G2);
+  return G1 == G2;
+}
+
+static _Bool _PtrEq_GlobalLocalCopy(gcptr G, gcptr L)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  wlog_t *entry;
+  gcptr R;
+
+  if (G->h_tid & GCFLAG_POSSIBLY_OUTDATED)
+    R = LatestGlobalRevision(d, G);
+  else
+    R = G;
+
+  G2L_FIND(d->global_to_local, R, entry, return 0);
+  return L == entry->val;
+}
+
+_Bool stm_PtrEq(gcptr P1, gcptr P2)
+{
+  if (P1 == P2)
+    return 1;
+  else if (P1 == NULL || P2 == NULL)   /* and not P1 == P2 == NULL */
+    return 0;
+
+  if (P1->h_revision != stm_local_revision)
+    {
+      if (P2->h_revision != stm_local_revision)
+        {
+          /* P1 and P2 are two globals */
+          return _PtrEq_Globals(P1, P2);
+        }
+      else if (P2->h_tid & GCFLAG_LOCAL_COPY)
+        {
+          /* P1 is a global, P2 is a local copy */
+          return _PtrEq_GlobalLocalCopy(P1, P2);
+        }
+      else
+        return 0;   /* P1 is global, P2 is new */
+    }
+  /* P1 is local, i.e. either new or a local copy */
+  if (P2->h_revision != stm_local_revision)
+    {
+      if (P1->h_tid & GCFLAG_LOCAL_COPY)
+        {
+          /* P2 is a global, P1 is a local copy */
+          return _PtrEq_GlobalLocalCopy(P2, P1);
+        }
+      else
+        return 0;   /* P1 is new, P2 is global */
+    }
+  /* P1 and P2 are both locals (and P1 != P2) */
+  return 0;
+}
+#endif
+_Bool stm_PtrEq(gcptr P1, gcptr P2)
+{
+  abort();//XXX
+}
+
+/************************************************************/
+
+#if 0
+void stm_ThreadLocalRef_LLSet(void **addr, void *newvalue)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  gcptrlist_insert2(&d->undolog, (gcptr)addr, (gcptr)*addr);
+  *addr = newvalue;
+}
+#endif
+
+/************************************************************/
+
+struct tx_descriptor *stm_tx_head = NULL;
+struct tx_public_descriptor *stm_descriptor_array[MAX_THREADS] = {0};
+static revision_t descriptor_array_free_list = 0;
+
+void _stm_test_forget_previous_state(void)
+{
+  /* debug: reset all global states, between tests */
+  dprintf(("=======================================================\n"));
+  assert(thread_descriptor == NULL);
+  memset(stm_descriptor_array, 0, sizeof(stm_descriptor_array));
+  descriptor_array_free_list = 0;
+  stm_tx_head = NULL;
+  stmgcpage_count(2);  /* reset */
+}
+
+struct tx_public_descriptor *stm_get_free_public_descriptor(revision_t *pindex)
+{
+  if (*pindex < 0)
+    *pindex = descriptor_array_free_list;
+
+  struct tx_public_descriptor *pd = stm_descriptor_array[*pindex];
+  if (pd != NULL)
+    {
+      *pindex = pd->free_list_next;
+      assert(*pindex >= 0);
+    }
+  return pd;
+}
+
+__thread gcptr stm_thread_local_obj;
+
+int DescriptorInit(void)
+{
+  if (GCFLAG_PREBUILT != PREBUILT_FLAGS)
+    {
+      stm_fatalerror("fix PREBUILT_FLAGS in stmgc.h by giving "
+                     "it the same value as GCFLAG_PREBUILT!\n");
+    }
+
+  if (thread_descriptor == NULL)
+    {
+      revision_t i;
+      struct tx_descriptor *d = stm_malloc(sizeof(struct tx_descriptor));
+      memset(d, 0, sizeof(struct tx_descriptor));
+      stmgcpage_acquire_global_lock();
+
+      struct tx_public_descriptor *pd;
+      i = descriptor_array_free_list;
+      pd = stm_descriptor_array[i];
+      if (pd != NULL) {
+          /* we are reusing 'pd' */
+          descriptor_array_free_list = pd->free_list_next;
+          assert(descriptor_array_free_list >= 0);
+          assert(pd->stolen_objects.size == 0);
+          assert(pd->stolen_young_stubs.size == 0);
+          /* there may be a thread holding the collection lock
+             because it steals a stub belonging to the thread
+             that previously owned this descriptor.
+          */
+      }
+      else {
+          /* no item in the free list */
+          descriptor_array_free_list = i + 1;
+          if (descriptor_array_free_list == MAX_THREADS) {
+              stm_fatalerror("error: too many threads at the same time "
+                             "in this process");
+          }
+          pd = stm_malloc(sizeof(struct tx_public_descriptor));
+          memset(pd, 0, sizeof(struct tx_public_descriptor));
+          stm_descriptor_array[i] = pd;
+      }
+      pd->free_list_next = -1;
+
+      d->public_descriptor = pd;
+      d->public_descriptor_index = i;
+      d->my_lock = LOCKED + 2 * i;
+      assert(d->my_lock & 1);
+      assert(d->my_lock >= LOCKED);
+      stm_private_rev_num = -d->my_lock;
+      d->private_revision_ref = &stm_private_rev_num;
+      d->read_barrier_cache_ref = &stm_read_barrier_cache;
+      stm_thread_local_obj = NULL;
+      d->thread_local_obj_ref = &stm_thread_local_obj;
+      d->max_aborts = -1;
+      d->tx_prev = NULL;
+      d->tx_next = stm_tx_head;
+      if (d->tx_next != NULL) d->tx_next->tx_prev = d;
+      stm_tx_head = d;
+      thread_descriptor = d;
+
+      dprintf(("[%lx] pthread %lx starting\n",
+               (long)d->public_descriptor_index, (long)pthread_self()));
+
+      stmgcpage_init_tls();
+      stmgcpage_release_global_lock();
+      return 1;
+    }
+  else
+    return 0;
+}
+
+void DescriptorDone(void)
+{
+    revision_t i;
+    struct tx_descriptor *d = thread_descriptor;
+    assert(d != NULL);
+    assert(d->active == 0);
+    stmgcpage_acquire_global_lock();
+
+    /* our nursery is empty at this point.  The list 'stolen_objects'
+       should have been emptied at the previous minor collection and
+       should remain empty because we don't have any young object. */
+    assert(d->public_descriptor->stolen_objects.size == 0);
+    assert(d->public_descriptor->stolen_young_stubs.size == 0);
+    gcptrlist_delete(&d->public_descriptor->stolen_objects);
+    gcptrlist_delete(&d->public_descriptor->stolen_young_stubs);
+
+    stmgcpage_done_tls();
+    i = d->public_descriptor_index;
+    assert(stm_descriptor_array[i] == d->public_descriptor);
+    d->public_descriptor->free_list_next = descriptor_array_free_list;
+    descriptor_array_free_list = i;
+    if (d->tx_prev != NULL) d->tx_prev->tx_next = d->tx_next;
+    if (d->tx_next != NULL) d->tx_next->tx_prev = d->tx_prev;
+    if (d == stm_tx_head) stm_tx_head = d->tx_next;
+    stmgcpage_release_global_lock();
+
+    thread_descriptor = NULL;
+
+    stm_thread_local_obj = (gcptr)0xBB;   /* to detect misuses */
+
+    g2l_delete(&d->public_to_private);
+    assert(d->private_from_protected.size == 0);
+    gcptrlist_delete(&d->private_from_protected);
+    gcptrlist_delete(&d->list_of_read_objects);
+#if 0
+    gcptrlist_delete(&d->abortinfo);
+    free(d->longest_abort_info);
+    gcptrlist_delete(&d->undolog);
+#endif
+
+    int num_aborts = 0, num_spinloops = 0;
+    char line[256], *p = line;
+
+    for (i=0; i<ABORT_REASONS; i++)
+        num_aborts += d->num_aborts[i];
+    for (i=0; i<SPINLOOP_REASONS; i++)
+        num_spinloops += d->num_spinloops[i];
+
+    p += sprintf(p, "[%lx] finishing: %d commits, %d aborts ",
+                 (long)d->public_descriptor_index,
+                 d->num_commits,
+                 num_aborts);
+
+    for (i=0; i<ABORT_REASONS; i++)
+        p += sprintf(p, "%c%d", i == 0 ? '[' : ',',
+                     d->num_aborts[i]);
+
+    for (i=1; i<SPINLOOP_REASONS; i++)  /* num_spinloops[0] == num_aborts */
+        p += sprintf(p, "%c%d", i == 1 ? '|' : ',',
+                     d->num_spinloops[i]);
+
+    p += sprintf(p, "]\n");
+    dprintf(("%s", line));
+
+    stm_free(d, sizeof(struct tx_descriptor));
+}
diff --git a/rpython/translator/stm/src_stm/et.h b/rpython/translator/stm/src_stm/et.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/et.h
@@ -0,0 +1,199 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+/*** Extendable Timestamps
+ *
+ * Documentation:
+ * doc-*.txt
+ *
+ * This is very indirectly based on rstm_r5/stm/et.hpp.
+ * See http://www.cs.rochester.edu/research/synchronization/rstm/api.shtml
+ */
+
+#ifndef _SRCSTM_ET_H
+#define _SRCSTM_ET_H
+
+
+#define MAX_THREADS         1024
+#define LOCKED              (INTPTR_MAX - 2*(MAX_THREADS-1))
+
+/* Description of the flags
+ * ------------------------
+ *
+ * Objects are either "young" or "old" depending on the generational garbage
+ * collection: "young" objects are the ones in the nursery (plus a few big
+ * ones outside) and will be collected by the following minor collection.
+ *
+ * Additionally, objects are either "public", "protected" or "private".
+ *
+ * GCFLAG_OLD is set on old objects.
+ *
+ * GCFLAG_VISITED is used temporarily during major collections.
+ *
+ * GCFLAG_PUBLIC is set on public objects.
+ *
+ * GCFLAG_BACKUP_COPY means the object is a (protected) backup copy.
+ * For debugging.
+ *
+ * GCFLAG_PUBLIC_TO_PRIVATE is added to a *public* object that has got a
+ * *private* copy.  It is sticky, reset only at the next major collection.
+ *
+ * GCFLAG_PREBUILT_ORIGINAL is only set on the original version of
+ * prebuilt objects.
+ *
+ * GCFLAG_WRITE_BARRIER is set on *old* objects to track old-to-young
+ * pointers.  It is only useful on private objects, and on protected
+ * objects (which may be turned private again).  It may be left set on
+ * public objects but is ignored there, because such objects are read-only.
+ * The flag is removed once a write occurs and the object is recorded in
+ * the list 'old_objects_to_trace'; it is set again at the next minor
+ * collection.
+ *
+ * GCFLAG_NURSERY_MOVED is used temporarily during minor collections.
+ *
+ * GCFLAG_STUB is set for debugging on stub objects made by stealing or
+ * by major collections.  'p_stub->h_revision' might be a value
+ * that is == 2 (mod 4): in this case they point to a protected/private
+ * object that belongs to the thread 'STUB_THREAD(p_stub)'.
+ *
+ * GCFLAG_PRIVATE_FROM_PROTECTED is set on objects that are private
+ * but converted from a protected.  These are precisely the objects
+ * that have a backup copy (in h_revision), which gives a copy of the
+ * original protected object.
+ * 
+ * GCFLAG_HAS_ID is set on young objects that have an old reserved
+ * memory to be copied to in minor collections (obj->h_original)
+ */
+static const revision_t GCFLAG_OLD                    = STM_FIRST_GCFLAG << 0;
+static const revision_t GCFLAG_VISITED                = STM_FIRST_GCFLAG << 1;
+static const revision_t GCFLAG_PUBLIC                 = STM_FIRST_GCFLAG << 2;
+static const revision_t GCFLAG_PREBUILT_ORIGINAL      = STM_FIRST_GCFLAG << 3;
+static const revision_t GCFLAG_PUBLIC_TO_PRIVATE      = STM_FIRST_GCFLAG << 4;
+// in stmgc.h:          GCFLAG_WRITE_BARRIER          = STM_FIRST_GCFLAG << 5;
+static const revision_t GCFLAG_NURSERY_MOVED          = STM_FIRST_GCFLAG << 6;
+static const revision_t GCFLAG_BACKUP_COPY  /*debug*/ = STM_FIRST_GCFLAG << 7;
+static const revision_t GCFLAG_STUB         /*debug*/ = STM_FIRST_GCFLAG << 8;
+static const revision_t GCFLAG_PRIVATE_FROM_PROTECTED = STM_FIRST_GCFLAG << 9;
+static const revision_t GCFLAG_HAS_ID                 = STM_FIRST_GCFLAG << 10;
+
+/* this value must be reflected in PREBUILT_FLAGS in stmgc.h */
+#define GCFLAG_PREBUILT  (GCFLAG_VISITED           | \
+                          GCFLAG_PREBUILT_ORIGINAL | \
+                          GCFLAG_OLD               | \
+                          GCFLAG_PUBLIC)
+

From noreply at buildbot.pypy.org  Fri Jun 28 20:55:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill the ctypes (and
 ctypes_config_cache) version of syslog, 
Message-ID: <20130628185525.882AE1C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65081:f379c0cd7d76
Date: 2013-06-28 20:51 +0200
http://bitbucket.org/pypy/pypy/changeset/f379c0cd7d76/

Log:	Kill the ctypes (and ctypes_config_cache) version of syslog, replace
	it with a cffi version.

diff --git a/lib_pypy/ctypes_config_cache/syslog.ctc.py b/lib_pypy/ctypes_config_cache/syslog.ctc.py
deleted file mode 100644
--- a/lib_pypy/ctypes_config_cache/syslog.ctc.py
+++ /dev/null
@@ -1,75 +0,0 @@
-"""
-'ctypes_configure' source for syslog.py.
-Run this to rebuild _syslog_cache.py.
-"""
-
-from ctypes_configure.configure import (configure,
-    ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger)
-import dumpcache
-
-
-_CONSTANTS = (
-    'LOG_EMERG',
-    'LOG_ALERT',
-    'LOG_CRIT',
-    'LOG_ERR',
-    'LOG_WARNING',
-    'LOG_NOTICE',
-    'LOG_INFO',
-    'LOG_DEBUG',
-
-    'LOG_PID',
-    'LOG_CONS',
-    'LOG_NDELAY',
-
-    'LOG_KERN',
-    'LOG_USER',
-    'LOG_MAIL',
-    'LOG_DAEMON',
-    'LOG_AUTH',
-    'LOG_LPR',
-    'LOG_LOCAL0',
-    'LOG_LOCAL1',
-    'LOG_LOCAL2',
-    'LOG_LOCAL3',
-    'LOG_LOCAL4',
-    'LOG_LOCAL5',
-    'LOG_LOCAL6',
-    'LOG_LOCAL7',
-)
-_OPTIONAL_CONSTANTS = (
-    'LOG_NOWAIT',
-    'LOG_PERROR',
-
-    'LOG_SYSLOG',
-    'LOG_CRON',
-    'LOG_UUCP',
-    'LOG_NEWS',
-)
-
-# Constant aliases if there are not defined
-_ALIAS = (
-    ('LOG_SYSLOG', 'LOG_DAEMON'),
-    ('LOG_CRON', 'LOG_DAEMON'),
-    ('LOG_NEWS', 'LOG_MAIL'),
-    ('LOG_UUCP', 'LOG_MAIL'),
-)
-
-class SyslogConfigure:
-    _compilation_info_ = ExternalCompilationInfo(includes=['sys/syslog.h'])
-for key in _CONSTANTS:
-    setattr(SyslogConfigure, key, ConstantInteger(key))
-for key in _OPTIONAL_CONSTANTS:
-    setattr(SyslogConfigure, key, DefinedConstantInteger(key))
-
-config = configure(SyslogConfigure)
-for key in _OPTIONAL_CONSTANTS:
-    if config[key] is None:
-        del config[key]
-for alias, key in _ALIAS:
-    config.setdefault(alias, config[key])
-
-all_constants = config.keys()
-all_constants.sort()
-config['ALL_CONSTANTS'] = tuple(all_constants)
-dumpcache.dumpcache2('syslog', config)
diff --git a/lib_pypy/syslog.py b/lib_pypy/syslog.py
--- a/lib_pypy/syslog.py
+++ b/lib_pypy/syslog.py
@@ -1,3 +1,4 @@
+# this cffi version was rewritten based on the
 # ctypes implementation: Victor Stinner, 2008-05-08
 """
 This module provides an interface to the Unix syslog library routines.
@@ -9,34 +10,84 @@
 if sys.platform == 'win32':
     raise ImportError("No syslog on Windows")
 
-# load the platform-specific cache made by running syslog.ctc.py
-from ctypes_config_cache._syslog_cache import *
-
-from ctypes_support import standard_c_lib as libc
-from ctypes import c_int, c_char_p
+from cffi import FFI
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
 
+ffi = FFI()
 
-# Real prototype is:
-# void syslog(int priority, const char *format, ...);
-# But we also need format ("%s") and one format argument (message)
-_syslog = libc.syslog
-_syslog.argtypes = (c_int, c_char_p, c_char_p)
-_syslog.restype = None
+ffi.cdef("""
+/* mandatory constants */
+#define LOG_EMERG ...
+#define LOG_ALERT ...
+#define LOG_CRIT ...
+#define LOG_ERR ...
+#define LOG_WARNING ...
+#define LOG_NOTICE ...
+#define LOG_INFO ...
+#define LOG_DEBUG ...
 
-_openlog = libc.openlog
-_openlog.argtypes = (c_char_p, c_int, c_int)
-_openlog.restype = None
+#define LOG_PID ...
+#define LOG_CONS ...
+#define LOG_NDELAY ...
 
-_closelog = libc.closelog
-_closelog.argtypes = None
-_closelog.restype = None
+#define LOG_KERN ...
+#define LOG_USER ...
+#define LOG_MAIL ...
+#define LOG_DAEMON ...
+#define LOG_AUTH ...
+#define LOG_LPR ...
+#define LOG_LOCAL0 ...
+#define LOG_LOCAL1 ...
+#define LOG_LOCAL2 ...
+#define LOG_LOCAL3 ...
+#define LOG_LOCAL4 ...
+#define LOG_LOCAL5 ...
+#define LOG_LOCAL6 ...
+#define LOG_LOCAL7 ...
 
-_setlogmask = libc.setlogmask
-_setlogmask.argtypes = (c_int,)
-_setlogmask.restype = c_int
+/* optional constants, gets defined to -919919 if missing */
+#define LOG_NOWAIT ...
+#define LOG_PERROR ...
+
+/* aliased constants, gets defined as some other constant if missing */
+#define LOG_SYSLOG ...
+#define LOG_CRON ...
+#define LOG_UUCP ...
+#define LOG_NEWS ...
+
+/* functions */
+void openlog(const char *ident, int option, int facility);
+void syslog(int priority, const char *format, const char *string);
+// NB. the signature of syslog() is specialized to the only case we use
+void closelog(void);
+int setlogmask(int mask);
+""")
+
+lib = ffi.verify("""
+#include <syslog.h>
+
+#ifndef LOG_NOWAIT
+#define LOG_NOWAIT -919919
+#endif
+#ifndef LOG_PERROR
+#define LOG_PERROR -919919
+#endif
+#ifndef LOG_SYSLOG
+#define LOG_SYSLOG LOG_DAEMON
+#endif
+#ifndef LOG_CRON
+#define LOG_CRON LOG_DAEMON
+#endif
+#ifndef LOG_UUCP
+#define LOG_UUCP LOG_MAIL
+#endif
+#ifndef LOG_NEWS
+#define LOG_NEWS LOG_MAIL
+#endif
+""")
+
 
 _S_log_open = False
 _S_ident_o = None
@@ -52,12 +103,17 @@
     return None
 
 @builtinify
-def openlog(ident=None, logoption=0, facility=LOG_USER):
+def openlog(ident=None, logoption=0, facility=lib.LOG_USER):
     global _S_ident_o, _S_log_open
     if ident is None:
         ident = _get_argv()
-    _S_ident_o = c_char_p(ident)    # keepalive
-    _openlog(_S_ident_o, logoption, facility)
+    if ident is None:
+        _S_ident_o = ffi.NULL
+    elif isinstance(ident, str):
+        _S_ident_o = ffi.new("char[]", ident)    # keepalive
+    else:
+        raise TypeError("'ident' must be a string or None")
+    lib.openlog(_S_ident_o, logoption, facility)
     _S_log_open = True
 
 @builtinify
@@ -69,19 +125,19 @@
     # if log is not opened, open it now
     if not _S_log_open:
         openlog()
-    _syslog(priority, "%s", message)
+    lib.syslog(priority, "%s", message)
 
 @builtinify
 def closelog():
     global _S_log_open, S_ident_o
     if _S_log_open:
-        _closelog()
+        lib.closelog()
         _S_log_open = False
         _S_ident_o = None
 
 @builtinify
 def setlogmask(mask):
-    return _setlogmask(mask)
+    return lib.setlogmask(mask)
 
 @builtinify
 def LOG_MASK(pri):
@@ -91,8 +147,15 @@
 def LOG_UPTO(pri):
     return (1 << (pri + 1)) - 1
 
-__all__ = ALL_CONSTANTS + (
+__all__ = []
+
+for name in sorted(lib.__dict__):
+    if name.startswith('LOG_'):
+        value = getattr(lib, name)
+        if value != -919919:
+            globals()[name] = value
+            __all__.append(name)
+
+__all__ = tuple(__all__) + (
     'openlog', 'syslog', 'closelog', 'setlogmask',
     'LOG_MASK', 'LOG_UPTO')
-
-del ALL_CONSTANTS
diff --git a/pypy/module/test_lib_pypy/test_syslog.py b/pypy/module/test_lib_pypy/test_syslog.py
--- a/pypy/module/test_lib_pypy/test_syslog.py
+++ b/pypy/module/test_lib_pypy/test_syslog.py
@@ -7,9 +7,5 @@
 
 # XXX very minimal test
 
-from lib_pypy.ctypes_config_cache import rebuild
-rebuild.rebuild_one('syslog.ctc.py')
-
-
 def test_syslog():
     assert hasattr(syslog, 'LOG_ALERT')

From noreply at buildbot.pypy.org  Fri Jun 28 20:55:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 20:55:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: Force cffi to build syslog here.
Message-ID: <20130628185526.A7B291C0351@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65082:27138960ee13
Date: 2013-06-28 20:54 +0200
http://bitbucket.org/pypy/pypy/changeset/27138960ee13/

Log:	Force cffi to build syslog here.

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -69,6 +69,7 @@
     subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
     if not sys.platform == 'win32':
         subprocess.check_call([str(pypy_c), '-c', 'import _curses'])
+        subprocess.check_call([str(pypy_c), '-c', 'import syslog'])
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
         rename_pypy_c += '.exe'
     binaries = [(pypy_c, rename_pypy_c)]

From noreply at buildbot.pypy.org  Fri Jun 28 21:16:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Jun 2013 21:16:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: More tests and small fixes in 'grp'.
Message-ID: <20130628191652.308401C01E5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65083:ae8a5b721a0d
Date: 2013-06-28 21:16 +0200
http://bitbucket.org/pypy/pypy/changeset/ae8a5b721a0d/

Log:	More tests and small fixes in 'grp'.

diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -8,6 +8,7 @@
 
 from ctypes import Structure, c_char_p, c_int, POINTER
 from ctypes_support import standard_c_lib as libc
+import _structseq
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -23,32 +24,13 @@
         ('gr_mem', POINTER(c_char_p)),
         )
 
-class Group(object):
-    def __init__(self, gr_name, gr_passwd, gr_gid, gr_mem):
-        self.gr_name = gr_name
-        self.gr_passwd = gr_passwd
-        self.gr_gid = gr_gid
-        self.gr_mem = gr_mem
+class struct_group:
+    __metaclass__ = _structseq.structseqtype
 
-    def __getitem__(self, item):
-        if item == 0:
-            return self.gr_name
-        elif item == 1:
-            return self.gr_passwd
-        elif item == 2:
-            return self.gr_gid
-        elif item == 3:
-            return self.gr_mem
-        else:
-            raise IndexError(item)
-
-    def __len__(self):
-        return 4
-
-    def __repr__(self):
-        return str((self.gr_name, self.gr_passwd, self.gr_gid, self.gr_mem))
-
-    # whatever else...
+    gr_name   = _structseq.structseqfield(0)
+    gr_passwd = _structseq.structseqfield(1)
+    gr_gid    = _structseq.structseqfield(2)
+    gr_mem    = _structseq.structseqfield(3)
 
 libc.getgrgid.argtypes = [gid_t]
 libc.getgrgid.restype = POINTER(GroupStruct)
@@ -71,8 +53,8 @@
     while res.contents.gr_mem[i]:
         mem.append(res.contents.gr_mem[i])
         i += 1
-    return Group(res.contents.gr_name, res.contents.gr_passwd,
-                 res.contents.gr_gid, mem)
+    return struct_group((res.contents.gr_name, res.contents.gr_passwd,
+                         res.contents.gr_gid, mem))
 
 @builtinify
 def getgrgid(gid):
diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -5,6 +5,22 @@
 except ImportError:
     py.test.skip("No grp module on this platform")
 
+def test_basic():
+    g = grp.getgrnam("root")
+    assert g.gr_gid == 0
+    assert g.gr_mem == ['root']
+    assert g.gr_name == 'root'
+    assert isinstance(g.gr_passwd, str)    # usually just 'x', don't hope :-)
+
 def test_extra():
     py.test.raises(TypeError, grp.getgrnam, False)
     py.test.raises(TypeError, grp.getgrnam, None)
+
+def test_struct_group():
+    g = grp.struct_group((10, 20, 30, 40))
+    assert len(g) == 4
+    assert list(g) == [10, 20, 30, 40]
+    assert g.gr_name == 10
+    assert g.gr_passwd == 20
+    assert g.gr_gid == 30
+    assert g.gr_mem == 40

From noreply at buildbot.pypy.org  Fri Jun 28 22:50:07 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 28 Jun 2013 22:50:07 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: In python3.2,
	the import statement is absolute, 
Message-ID: <20130628205007.5AAF11C08E4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65084:d2cb525d01f4
Date: 2013-06-28 22:18 +0200
http://bitbucket.org/pypy/pypy/changeset/d2cb525d01f4/

Log:	In python3.2, the import statement is absolute, but __import__()
	still tries both relative and absolute. Test and fix.

	This is needed by IDLE.

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -145,6 +145,9 @@
 
     if ctxt_package is not None:
         # __package__ is set, so use it
+        if ctxt_package == '' and level < 0:
+            return None, 0
+
         dot_position = _get_dot_position(ctxt_package, level - 1)
         if dot_position < 0:
             if len(ctxt_package) == 0:
@@ -226,9 +229,9 @@
 
 @unwrap_spec(name='str0', level=int)
 def importhook(space, name, w_globals=None,
-               w_locals=None, w_fromlist=None, level=0):
+               w_locals=None, w_fromlist=None, level=-1):
     modulename = name
-    if not modulename and level == 0:
+    if not modulename and level < 0:
         raise OperationError(
             space.w_ValueError,
             space.wrap("Empty module name"))
@@ -240,15 +243,31 @@
         fromlist_w = None
 
     rel_modulename = None
-    if (level > 0 and
+    if (level != 0 and
         w_globals is not None and
         space.isinstance_w(w_globals, space.w_dict)):
 
         rel_modulename, rel_level = _get_relative_name(space, modulename, level, w_globals)
 
         if rel_modulename:
-            w_mod = absolute_import(space, rel_modulename, rel_level,
-                                    fromlist_w, tentative=False)
+            # if no level was set, ignore import errors, and
+            # fall back to absolute import at the end of the
+            # function.
+            if level == -1:
+                # This check is a fast path to avoid redoing the
+                # following absolute_import() in the common case
+                w_mod = check_sys_modules_w(space, rel_modulename)
+                if w_mod is not None and space.is_w(w_mod, space.w_None):
+                    # if we already find space.w_None, it means that we
+                    # already tried and failed and fell back to the
+                    # end of this function.
+                    w_mod = None
+                else:
+                    w_mod = absolute_import(space, rel_modulename, rel_level,
+                                            fromlist_w, tentative=True)
+            else:
+                w_mod = absolute_import(space, rel_modulename, rel_level,
+                                        fromlist_w, tentative=False)
             if w_mod is not None:
                 return w_mod
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -59,6 +59,9 @@
              relative_c = "from __future__ import absolute_import\nfrom .struct import inpackage",
              relative_f = "from .imp import get_magic",
              relative_g = "import imp; from .imp import get_magic",
+             inpackage  = "inpackage = 1",
+             function_a = "g = {'__name__': 'pkg.a'}; __import__('inpackage', g); print(g)",
+             function_b = "g = {'__name__': 'not.a'}; __import__('inpackage', g); print(g)",
              )
     setuppkg("pkg.pkg1",
              __init__   = 'from . import a',
@@ -552,6 +555,16 @@
         check_absolute()
         raises(ValueError, check_relative)
 
+    def test_import_function(self):
+        # More tests for __import__
+        import sys
+        if sys.version < '3.3':
+            from pkg import function_a
+            assert function_a.g['__package__'] == 'pkg'
+            raises(ImportError, "from pkg import function_b")
+        else:
+            raises(ImportError, "from pkg import function_a")
+
     def test_universal_newlines(self):
         import pkg_univnewlines
         assert pkg_univnewlines.a == 5

From noreply at buildbot.pypy.org  Fri Jun 28 23:21:52 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 28 Jun 2013 23:21:52 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add co_kwonlyargcount to builtin code
	objects.
Message-ID: <20130628212152.0343C1C0883@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65085:cd74547a33c5
Date: 2013-06-28 23:16 +0200
http://bitbucket.org/pypy/pypy/changeset/cd74547a33c5/

Log:	Add co_kwonlyargcount to builtin code objects. This fixes
	inspect.getfullargspec(''.join), and IDLE can display tooltips for
	builtin functions.

diff --git a/pypy/interpreter/test/test_code.py b/pypy/interpreter/test/test_code.py
--- a/pypy/interpreter/test/test_code.py
+++ b/pypy/interpreter/test/test_code.py
@@ -40,6 +40,7 @@
                 (abs.__code__, {'co_name': 'abs',
                                  'co_varnames': ('val',),
                                  'co_argcount': 1,
+                                 'co_kwonlyargcount': 0,
                                  'co_flags': 0,
                                  'co_consts': ("abs(number) -> number\n\nReturn the absolute value of the argument.",),
                                  }),
@@ -47,6 +48,7 @@
                                 {#'co_name': '__init__',   XXX getting descr__init__
                                  'co_varnames': ('obj', 'args', 'keywords'),
                                  'co_argcount': 1,
+                                 'co_kwonlyargcount': 0,
                                  'co_flags': 0x000C,  # VARARGS|VARKEYWORDS
                                  }),
                 ]
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -666,6 +666,9 @@
 def fget_co_argcount(space, code): # unwrapping through unwrap_spec
     return space.wrap(code.signature().num_argnames())
 
+def fget_zero(space, code):
+    return space.wrap(0)
+
 def fget_co_flags(space, code): # unwrapping through unwrap_spec
     sig = code.signature()
     flags = 0
@@ -711,6 +714,7 @@
     co_name = interp_attrproperty('co_name', cls=Code),
     co_varnames = GetSetProperty(fget_co_varnames, cls=Code),
     co_argcount = GetSetProperty(fget_co_argcount, cls=Code),
+    co_kwonlyargcount = GetSetProperty(fget_zero, cls=Code),
     co_flags = GetSetProperty(fget_co_flags, cls=Code),
     co_consts = GetSetProperty(fget_co_consts, cls=Code),
     )
@@ -721,6 +725,7 @@
     co_name = interp_attrproperty('co_name', cls=BuiltinCode),
     co_varnames = GetSetProperty(fget_co_varnames, cls=BuiltinCode),
     co_argcount = GetSetProperty(fget_co_argcount, cls=BuiltinCode),
+    co_kwonlyargcount = GetSetProperty(fget_zero, cls=BuiltinCode),
     co_flags = GetSetProperty(fget_co_flags, cls=BuiltinCode),
     co_consts = GetSetProperty(fget_co_consts, cls=BuiltinCode),
     )

From noreply at buildbot.pypy.org  Fri Jun 28 23:21:53 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 28 Jun 2013 23:21:53 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Port _tkinter-on-cffi to py3k.
Message-ID: <20130628212153.4C62B1C0883@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65086:e228809dd947
Date: 2013-06-28 23:20 +0200
http://bitbucket.org/pypy/pypy/changeset/e228809dd947/

Log:	Port _tkinter-on-cffi to py3k.

diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -16,8 +16,8 @@
 
 from .app import TkApp
 
-TK_VERSION = tkffi.string(tklib.get_tk_version())
-TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+TK_VERSION = tkffi.string(tklib.get_tk_version()).decode('utf-8')
+TCL_VERSION = tkffi.string(tklib.get_tcl_version()).decode('utf-8')
 
 READABLE = tklib.TCL_READABLE
 WRITABLE = tklib.TCL_WRITABLE
@@ -26,7 +26,7 @@
 def create(screenName=None, baseName=None, className=None,
            interactive=False, wantobjects=False, wantTk=True,
            sync=False, use=None):
-    return TkApp(screenName, baseName, className,
+    return TkApp(screenName, className,
                  interactive, wantobjects, wantTk, sync, use)
 
 def _flatten(item):
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -9,15 +9,15 @@
 def varname_converter(input):
     if isinstance(input, TclObject):
         return input.string
-    return input
+    return input.encode('utf-8')
 
 
 def Tcl_AppInit(app):
     if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
         app.raiseTclError()
     skip_tk_init = tklib.Tcl_GetVar(
-        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
-    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        app.interp, b"_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == b"1":
         return
 
     if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
@@ -38,7 +38,8 @@
         self = tkffi.from_handle(clientData)
         assert self.app.interp == interp
         try:
-            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            args = [tkffi.string(arg).decode('utf-8')
+                    for arg in argv[1:argc]]
             result = self.func(*args)
             obj = AsObj(result)
             tklib.Tcl_SetObjResult(interp, obj)
@@ -58,7 +59,7 @@
 
 
 class TkApp(object):
-    def __new__(cls, screenName, baseName, className,
+    def __new__(cls, screenName, className,
                 interactive, wantobjects, wantTk, sync, use):
         if not wantobjects:
             raise NotImplementedError("wantobjects=True only")
@@ -66,7 +67,7 @@
         self.interp = tklib.Tcl_CreateInterp()
         self._wantobjects = wantobjects
         self.threaded = bool(tklib.Tcl_GetVar2Ex(
-            self.interp, "tcl_platform", "threaded",
+            self.interp, b"tcl_platform", b"threaded",
             tklib.TCL_GLOBAL_ONLY))
         self.thread_id = tklib.Tcl_GetCurrentThread()
         self.dispatching = False
@@ -77,26 +78,27 @@
         self._commands = {}
 
         # Delete the 'exit' command, which can screw things up
-        tklib.Tcl_DeleteCommand(self.interp, "exit")
+        tklib.Tcl_DeleteCommand(self.interp, b"exit")
 
         if screenName is not None:
-            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+            tklib.Tcl_SetVar2(self.interp, b"env", b"DISPLAY",
+                              screenName.encode('utf-8'),
                               tklib.TCL_GLOBAL_ONLY)
 
         if interactive:
-            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+            tklib.Tcl_SetVar(self.interp, b"tcl_interactive", b"1",
                              tklib.TCL_GLOBAL_ONLY)
         else:
-            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+            tklib.Tcl_SetVar(self.interp, b"tcl_interactive", b"0",
                              tklib.TCL_GLOBAL_ONLY)
 
         # This is used to get the application class for Tk 4.1 and up
-        argv0 = className.lower()
-        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+        argv0 = className.lower().encode('utf-8')
+        tklib.Tcl_SetVar(self.interp, b"argv0", argv0,
                          tklib.TCL_GLOBAL_ONLY)
 
         if not wantTk:
-            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+            tklib.Tcl_SetVar(self.interp, b"_tkinter_skip_tk_init", b"1",
                              tklib.TCL_GLOBAL_ONLY)
 
         # some initial arguments need to be in argv
@@ -123,8 +125,9 @@
     def raiseTclError(self):
         if self.errorInCmd:
             self.errorInCmd = False
-            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
-        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+            raise self.exc_info[1].with_traceback(self.exc_info[2])
+        raise TclError(tkffi.string(
+                tklib.Tcl_GetStringResult(self.interp)).decode('utf-8'))
 
     def wantobjects(self):
         return self._wantobjects
@@ -135,11 +138,11 @@
 
     def loadtk(self):
         # We want to guard against calling Tk_Init() multiple times
-        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        err = tklib.Tcl_Eval(self.interp, b"info exists     tk_version")
         if err == tklib.TCL_ERROR:
             self.raiseTclError()
         tk_exists = tklib.Tcl_GetStringResult(self.interp)
-        if not tk_exists or tkffi.string(tk_exists) != "1":
+        if not tk_exists or tkffi.string(tk_exists) != b"1":
             err = tklib.Tk_Init(self.interp)
             if err == tklib.TCL_ERROR:
                 self.raiseTclError()
@@ -220,7 +223,7 @@
             raise NotImplementedError("Call from another thread")
 
         res = tklib.Tcl_CreateCommand(
-            self.interp, cmdName, _CommandData.PythonCmd,
+            self.interp, cmdName.encode('utf-8'), _CommandData.PythonCmd,
             clientData, _CommandData.PythonCmdDelete)
         if not res:
             raise TclError("can't create Tcl command")
@@ -229,7 +232,7 @@
         if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
             raise NotImplementedError("Call from another thread")
 
-        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName.encode('utf-8'))
         if res == -1:
             raise TclError("can't delete Tcl command")
 
@@ -280,17 +283,19 @@
 
     def eval(self, script):
         self._check_tcl_appartment()
-        res = tklib.Tcl_Eval(self.interp, script)
+        res = tklib.Tcl_Eval(self.interp, script.encode('utf-8'))
         if res == tklib.TCL_ERROR:
             self.raiseTclError()
-        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        result = tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        return result.decode('utf-8')
 
     def evalfile(self, filename):
         self._check_tcl_appartment()
-        res = tklib.Tcl_EvalFile(self.interp, filename)
+        res = tklib.Tcl_EvalFile(self.interp, filename.encode('utf-8'))
         if res == tklib.TCL_ERROR:
             self.raiseTclError()
-        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        result = tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        return result.decode('utf-8')
 
     def split(self, arg):
         if isinstance(arg, tuple):
@@ -301,7 +306,7 @@
     def splitlist(self, arg):
         if isinstance(arg, tuple):
             return arg
-        if isinstance(arg, unicode):
+        if isinstance(arg, str):
             arg = arg.encode('utf8')
 
         argc = tkffi.new("int*")
@@ -310,7 +315,7 @@
         if res == tklib.TCL_ERROR:
             self.raiseTclError()
 
-        result = tuple(tkffi.string(argv[0][i])
+        result = tuple(tkffi.string(argv[0][i]).decode('utf-8')
                        for i in range(argc[0]))
         tklib.Tcl_Free(argv[0])
         return result
@@ -326,7 +331,7 @@
             for elem, newelem in zip(arg, newelems):
                 if elem is not newelem:
                     return newelems
-        elif isinstance(arg, str):
+        elif isinstance(arg, bytes):
             argc = tkffi.new("int*")
             argv = tkffi.new("char***")
             res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
@@ -345,7 +350,7 @@
             # Not a list.
             # Could be a quoted string containing funnies, e.g. {"}.
             # Return the string itself.
-            return arg
+            return arg.decode('utf-8')
 
         try:
             if argc[0] == 0:
@@ -361,6 +366,7 @@
     def getboolean(self, s):
         if isinstance(s, int):
             return s
+        s = s.encode('utf-8')
         v = tkffi.new("int*")
         res = tklib.Tcl_GetBoolean(self.interp, s, v)
         if res == tklib.TCL_ERROR:
@@ -383,7 +389,7 @@
         self.quitMainLoop = False
         if self.errorInCmd:
             self.errorInCmd = False
-            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+            raise self.exc_info[1].with_traceback(self.exc_info[2])
 
     def quit(self):
         self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -4,13 +4,13 @@
 
 class TypeCache(object):
     def __init__(self):
-        self.BooleanType = tklib.Tcl_GetObjType("boolean")
-        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
-        self.DoubleType = tklib.Tcl_GetObjType("double")
-        self.IntType = tklib.Tcl_GetObjType("int")
-        self.ListType = tklib.Tcl_GetObjType("list")
-        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
-        self.StringType = tklib.Tcl_GetObjType("string")
+        self.BooleanType = tklib.Tcl_GetObjType(b"boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType(b"bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType(b"double")
+        self.IntType = tklib.Tcl_GetObjType(b"int")
+        self.ListType = tklib.Tcl_GetObjType(b"list")
+        self.ProcBodyType = tklib.Tcl_GetObjType(b"procbody")
+        self.StringType = tklib.Tcl_GetObjType(b"string")
         
 
 def FromObj(app, value):
@@ -18,13 +18,7 @@
     typeCache = app._typeCache
     if not value.typePtr:
         buf = tkffi.buffer(value.bytes, value.length)
-        result = buf[:]
-        # If the result contains any bytes with the top bit set, it's
-        # UTF-8 and we should decode it to Unicode.
-        try:
-            result.decode('ascii')
-        except UnicodeDecodeError:
-            result = result.decode('utf8')
+        result = buf[:].decode('utf8')
         return result
 
     elif value.typePtr == typeCache.BooleanType:
@@ -60,12 +54,15 @@
     return TclObject(value)
 
 def AsObj(value):
-    if isinstance(value, str):
+    if isinstance(value, bytes):
         return tklib.Tcl_NewStringObj(value, len(value))
     elif isinstance(value, bool):
         return tklib.Tcl_NewBooleanObj(value)
     elif isinstance(value, int):
-        return tklib.Tcl_NewLongObj(value)
+        try:
+            return tklib.Tcl_NewLongObj(value)
+        except OverflowError:
+            pass  # and fall through to default object handling.
     elif isinstance(value, float):
         return tklib.Tcl_NewDoubleObj(value)
     elif isinstance(value, tuple):
@@ -73,16 +70,16 @@
         for i in range(len(value)):
             argv[i] = AsObj(value[i])
         return tklib.Tcl_NewListObj(len(value), argv)
-    elif isinstance(value, unicode):
+    elif isinstance(value, str):
         encoded = value.encode('utf-16')[2:]
         buf = tkffi.new("char[]", encoded)
         inbuf = tkffi.cast("Tcl_UniChar*", buf)
-        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)//2)
     elif isinstance(value, TclObject):
         tklib.Tcl_IncrRefCount(value._value)
         return value._value
-    else:
-        return AsObj(str(value))
+
+    return AsObj(str(value))
 
 class TclObject(object):
     def __new__(cls, value):
@@ -98,17 +95,15 @@
     def __str__(self):
         if self._string and isinstance(self._string, str):
             return self._string
-        return tkffi.string(tklib.Tcl_GetString(self._value))
+        return tkffi.string(tklib.Tcl_GetString(self._value)).decode('utf-8')
 
     @property
     def string(self):
+        "the string representation of this object, either as str or bytes"
         if self._string is None:
             length = tkffi.new("int*")
             s = tklib.Tcl_GetStringFromObj(self._value, length)
             value = tkffi.buffer(s, length[0])[:]
-            try:
-                value.decode('ascii')
-            except UnicodeDecodeError:
-                value = value.decode('utf8')
+            value = value.decode('utf8')
             self._string = value
         return self._string

From noreply at buildbot.pypy.org  Sat Jun 29 10:35:56 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat, 29 Jun 2013 10:35:56 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: one more corner case
Message-ID: <20130629083556.832711C1556@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65087:7c8873f9eeb2
Date: 2013-06-28 18:58 +0200
http://bitbucket.org/pypy/pypy/changeset/7c8873f9eeb2/

Log:	one more corner case

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -186,6 +186,8 @@
         if self.ll_chars[i] == 'e' or self.ll_chars[i] == 'E':
             is_float = True
             i += 1
+            if self.ll_chars[i] == '+' or self.ll_chars[i] == '-':
+                i += 1
             while self.ll_chars[i].isdigit():
                 i += 1
         #
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -126,6 +126,8 @@
         check(x, float(x))
         x = str(sys.maxint+1) + 'E1'
         check(x, float(x))
+        x = str(sys.maxint+1) + 'E-1'
+        check(x, float(x))
         #
         check('1E400', float('inf'))
 

From noreply at buildbot.pypy.org  Sat Jun 29 10:35:57 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat, 29 Jun 2013 10:35:57 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: completely change the strategy for
 parsing floats: we now rely on rlib.rdtoa, which is slower but accurate
Message-ID: <20130629083557.C26761C2FB2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65088:f5cbf0738f31
Date: 2013-06-29 10:32 +0200
http://bitbucket.org/pypy/pypy/changeset/f5cbf0738f31/

Log:	completely change the strategy for parsing floats: we now rely on
	rlib.rdtoa, which is slower but accurate

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -3,6 +3,7 @@
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import rfloat
+from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter import unicodehelper
@@ -50,12 +51,19 @@
     def __init__(self, space, s):
         self.space = space
         self.s = s
-        # we put a sentinel at the end so that we never have to check for the
-        # "end of string" condition
-        self.ll_chars = llstr(s+'\0').chars
+        # we put our string in a raw buffer so:
+        # 1) we automatically get the '\0' sentinel at the end of the string,
+        #    which means that we never have to check for the "end of string"
+        # 2) we can pass the buffer directly to strtod
+        self.ll_chars = rffi.str2charp(s)
+        self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
 
+    def close(self):
+        rffi.free_charp(self.ll_chars)
+        lltype.free(self.end_ptr, flavor='raw')
+
     def getslice(self, start, end):
         assert start >= 0
         assert end >= 0
@@ -128,43 +136,57 @@
         return w_res
 
     def decode_numeric_fast(self, i):
+        start = i
         i, ovf_maybe, intval = self.parse_integer(i, allow_leading_0=False)
-        if ovf_maybe:
+        #
+        # check for the optional fractional part
+        ch = self.ll_chars[i]
+        if ch == '.':
+            if not self.ll_chars[i+1].isdigit():
+                self._raise("Expected digit at char %d", i+1)
+            return self.parse_float(start)
+        elif ch == 'e' or ch == 'E':
+            return self.parse_float(start)
+        elif ovf_maybe:
             # apparently we get a ~30% slowdown on my microbenchmark if we
             # return None instead of w_None, probably because the annotation
             # of the results geta can_be_None=True. We need to check if this
             # is still true also for the full pypy
             return self.space.w_None
-        #
-        is_float = False
-        exp = 0
-        frcval = 0.0
-        frccount = 0
-        #
-        # check for the optional fractional part
-        ch = self.ll_chars[i]
-        if ch == '.':
-            is_float = True
-            i, frcval, frccount = self.parse_digits(i+1)
-            frcval = neg_pow_10(frcval, frccount)
-            ch = self.ll_chars[i]
-        # check for the optional exponent part
-        if ch == 'E' or ch == 'e':
-            is_float = True
-            i, ovf_maybe, exp = self.parse_integer(i+1, allow_leading_0=True)
-        #
+
         self.pos = i
-        if is_float:
-            # build the float
-            floatval = intval + frcval
-            if exp != 0:
-                try:
-                    floatval = floatval * math.pow(10, exp)
-                except OverflowError:
-                    floatval = rfloat.INFINITY
-            return self.space.wrap(floatval)
-        else:
-            return self.space.wrap(intval)
+        return self.space.wrap(intval)
+
+        ## if ch == '.':
+        ##     is_float = True
+        ##     i, frcval, frccount = self.parse_digits(i+1)
+        ##     frcval = neg_pow_10(frcval, frccount)
+        ##     ch = self.ll_chars[i]
+        ## # check for the optional exponent part
+        ## if ch == 'E' or ch == 'e':
+        ##     is_float = True
+        ##     i, ovf_maybe, exp = self.parse_integer(i+1, allow_leading_0=True)
+        ## #
+        ## self.pos = i
+        ## if is_float:
+        ##     # build the float
+        ##     floatval = intval + frcval
+        ##     if exp != 0:
+        ##         try:
+        ##             floatval = floatval * math.pow(10, exp)
+        ##         except OverflowError:
+        ##             floatval = rfloat.INFINITY
+        ##     return self.space.wrap(floatval)
+        ## else:
+        ##     return self.space.wrap(intval)
+
+    def parse_float(self, i):
+        from rpython.rlib import rdtoa
+        start = rffi.ptradd(self.ll_chars, i)
+        floatval = rdtoa.dg_strtod(start, self.end_ptr)
+        diff = rffi.cast(rffi.LONG, self.end_ptr[0]) - rffi.cast(rffi.LONG, start)
+        self.pos = i + diff
+        return self.space.wrap(floatval)
 
     def decode_numeric_slow(self, i):
         is_float = False
@@ -407,10 +429,13 @@
                              space.wrap("Expected utf8-encoded str, got unicode"))
     s = space.str_w(w_s)
     decoder = JSONDecoder(space, s)
-    w_res = decoder.decode_any(0)
-    i = decoder.skip_whitespace(decoder.pos)
-    if i < len(s):
-        start = i
-        end = len(s) - 1
-        raise operationerrfmt(space.w_ValueError, "Extra data: char %d - %d", start, end)
-    return w_res
+    try:
+        w_res = decoder.decode_any(0)
+        i = decoder.skip_whitespace(decoder.pos)
+        if i < len(s):
+            start = i
+            end = len(s) - 1
+            raise operationerrfmt(space.w_ValueError, "Extra data: char %d - %d", start, end)
+        return w_res
+    finally:
+        decoder.close()

From noreply at buildbot.pypy.org  Sat Jun 29 11:55:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 11:55:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add a test for the
 gcptrlist_insert(&d->old_objects_to_trace) after stmgcpage_malloc()
Message-ID: <20130629095525.AA1111C1556@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r317:95b19b2a2ee3
Date: 2013-06-29 11:06 +0200
http://bitbucket.org/pypy/stmgc/changeset/95b19b2a2ee3/

Log:	Add a test for the gcptrlist_insert(&d->old_objects_to_trace) after
	stmgcpage_malloc()

diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -298,9 +298,19 @@
     assert lib.getlong(p2, 0) == 389719
 
 def test_nalloc_large_object():
-    for words in range(80):
-        p1 = nalloc(HDR + words * WORD)
-    # assert did not crash
+    for words in range(40, 81):
+        p1 = nalloc_refs(words)
+        lib.rawsetptr(p1, 0, p1)
+    p2 = nalloc(HDR)   # this alone should not collect
+    check_not_free(p1)
+    lib.rawsetptr(p1, 79, p2)
+    lib.stm_push_root(p1)
+    minor_collect()
+    p1b = lib.stm_pop_root()
+    assert p1b == p1
+    check_not_free(p1)
+    assert lib.rawgetptr(p1, 0) == p1
+    check_not_free(lib.rawgetptr(p1, 79))
 
 def test_collect_soon():
     lib.stmgc_minor_collect_soon()

From noreply at buildbot.pypy.org  Sat Jun 29 11:55:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 11:55:26 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tests, asserts,
 and fixes for stm_allocate() returning correctly
Message-ID: <20130629095526.C73611C1556@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r318:c856b6a0d157
Date: 2013-06-29 11:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/c856b6a0d157/

Log:	Tests, asserts, and fixes for stm_allocate() returning correctly
	cleared memory.

diff --git a/c4/dbgmem.c b/c4/dbgmem.c
--- a/c4/dbgmem.c
+++ b/c4/dbgmem.c
@@ -64,6 +64,7 @@
 
     dprintf(("stm_malloc(%zu): %p\n", sz, result));
     assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
+    memset(result, 0xBB, sz);
     return result;
 }
 
@@ -89,6 +90,13 @@
     return accessible_pages[base] == 42;
 }
 
+void assert_cleared(char *p, size_t size)
+{
+    size_t i;
+    for (i = 0; i < size; i++)
+        assert(p[i] == 0);
+}
+
 /************************************************************/
 #endif
 
@@ -97,6 +105,7 @@
                                   size_t already_cleared)
 {
     char *baseaddr = base;
+    assert(already_cleared <= size);
 
     if (size > 2 * PAGE_SIZE) {
         int lowbits = ((intptr_t)baseaddr) & (PAGE_SIZE-1);
@@ -127,4 +136,5 @@
     if (size > already_cleared) { /* clear the final misaligned part, if any */
         memset(baseaddr, 0, size - already_cleared);
     }
+    assert_cleared(base, size);
 }
diff --git a/c4/dbgmem.h b/c4/dbgmem.h
--- a/c4/dbgmem.h
+++ b/c4/dbgmem.h
@@ -7,11 +7,13 @@
 void *stm_malloc(size_t);
 void stm_free(void *, size_t);
 int _stm_can_access_memory(char *);
+void assert_cleared(char *, size_t);
 
 #else
 
 #define stm_malloc(sz)    malloc(sz)
 #define stm_free(p,sz)    free(p)
+#define assert_cleared(p,sz)     do { } while(0)
 
 #endif
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -72,6 +72,15 @@
         P = (gcptr)cur;
         P->h_tid = tid;
     }
+#ifdef _GC_DEBUG
+    if (P != NULL) {
+        assert(P->h_tid != 0);
+        assert_cleared(((char *)P) + sizeof(revision_t),
+                       size - sizeof(revision_t));
+    }
+    else
+        assert(tid == -1);
+#endif
     return P;
 }
 
@@ -538,18 +547,6 @@
     */
     teardown_minor_collect(d);
 
-    /* if in debugging mode, we allocate a different nursery and make
-       the old one inaccessible
-    */
-#if defined(_GC_DEBUG) && _GC_DEBUG >= 2
-    stm_free(d->nursery_base, GC_NURSERY);
-    d->nursery_base = stm_malloc(GC_NURSERY);
-    d->nursery_end = d->nursery_base + GC_NURSERY;
-    d->nursery_nextlimit = d->nursery_base;
-    dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
-             d->nursery_end));
-#endif
-
     /* When doing minor collections with the nursery "mostly empty",
        as occurs when other threads force major collections but this
        thread didn't do much at all, then we clear the nursery using
@@ -572,6 +569,20 @@
         d->nursery_cleared = NC_REGULAR;
     }
 
+    /* if in debugging mode, we allocate a different nursery and make
+       the old one inaccessible
+    */
+#if defined(_GC_DEBUG) && _GC_DEBUG >= 2
+    if (d->nursery_cleared == NC_ALREADY_CLEARED)
+        assert_cleared(d->nursery_base, GC_NURSERY);
+    stm_free(d->nursery_base, GC_NURSERY);
+    d->nursery_base = stm_malloc(GC_NURSERY);
+    d->nursery_end = d->nursery_base + GC_NURSERY;
+    dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
+             d->nursery_end));
+    if (d->nursery_cleared == NC_ALREADY_CLEARED)
+        memset(d->nursery_base, 0, GC_NURSERY);
+#endif
     d->nursery_current = d->nursery_base;
     d->nursery_nextlimit = d->nursery_base;
 
@@ -640,6 +651,7 @@
 
         /* Allocate it externally, and make it old */
         gcptr P = stmgcpage_malloc(allocate_size);
+        memset(P, 0, allocate_size);
         P->h_tid = tid | GCFLAG_OLD;
         gcptrlist_insert(&d->old_objects_to_trace, P);
         return P;
@@ -672,5 +684,7 @@
     assert(d->nursery_current <= d->nursery_nextlimit);
 
     P->h_tid = tid;
+    assert_cleared(((char *)P) + sizeof(revision_t),
+                   allocate_size - sizeof(revision_t));
     return P;
 }
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -465,6 +465,7 @@
 def oalloc(size):
     "Allocate an 'old' protected object, outside any nursery"
     p = lib.stmgcpage_malloc(size)
+    lib.memset(p, 0, size)
     p.h_tid = GCFLAG_OLD | GCFLAG_WRITE_BARRIER
     p.h_revision = -sys.maxint
     lib.settid(p, 42 + size)
@@ -473,12 +474,12 @@
 def oalloc_refs(nrefs):
     """Allocate an 'old' protected object, outside any nursery,
     with nrefs pointers"""
-    p = lib.stmgcpage_malloc(HDR + WORD * nrefs)
+    size = HDR + WORD * nrefs
+    p = lib.stmgcpage_malloc(size)
+    lib.memset(p, 0, size)
     p.h_tid = GCFLAG_OLD | GCFLAG_WRITE_BARRIER
     p.h_revision = -sys.maxint
     lib.settid(p, 42142 + nrefs)
-    for i in range(nrefs):
-        rawsetptr(p, i, ffi.NULL)
     return p
 
 ofree = lib.stmgcpage_free
diff --git a/c4/test/test_nursery.py b/c4/test/test_nursery.py
--- a/c4/test/test_nursery.py
+++ b/c4/test/test_nursery.py
@@ -298,19 +298,23 @@
     assert lib.getlong(p2, 0) == 389719
 
 def test_nalloc_large_object():
-    for words in range(40, 81):
-        p1 = nalloc_refs(words)
-        lib.rawsetptr(p1, 0, p1)
-    p2 = nalloc(HDR)   # this alone should not collect
-    check_not_free(p1)
-    lib.rawsetptr(p1, 79, p2)
-    lib.stm_push_root(p1)
-    minor_collect()
-    p1b = lib.stm_pop_root()
-    assert p1b == p1
-    check_not_free(p1)
-    assert lib.rawgetptr(p1, 0) == p1
-    check_not_free(lib.rawgetptr(p1, 79))
+    for repeat in range(3):
+        for words in range(40, 81):
+            p1 = nalloc_refs(words)
+            for c in range(words):
+                assert lib.rawgetlong(p1, c) == 0     # null-initialized
+                lib.rawsetptr(p1, c, p1)
+        p2 = nalloc(HDR)   # this alone should not collect
+        check_not_free(p1)
+        lib.rawsetptr(p1, 79, p2)
+        lib.stm_push_root(p1)
+        minor_collect()
+        p1b = lib.stm_pop_root()
+        assert p1b == p1
+        check_not_free(p1)
+        assert lib.rawgetptr(p1, 0) == p1
+        check_not_free(lib.rawgetptr(p1, 79))
+        major_collect()
 
 def test_collect_soon():
     lib.stmgc_minor_collect_soon()

From noreply at buildbot.pypy.org  Sat Jun 29 11:57:03 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat, 29 Jun 2013 11:57:03 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add an option to enable fast but less
 precise parsing of floats
Message-ID: <20130629095703.DAAAB1C1556@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65089:55ed11b7b3be
Date: 2013-06-29 11:37 +0200
http://bitbucket.org/pypy/pypy/changeset/55ed11b7b3be/

Log:	add an option to enable fast but less precise parsing of floats

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -48,7 +48,7 @@
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 class JSONDecoder(object):
-    def __init__(self, space, s):
+    def __init__(self, space, s, precise_float=True):
         self.space = space
         self.s = s
         # we put our string in a raw buffer so:
@@ -59,6 +59,7 @@
         self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
+        self.precise_float = precise_float
 
     def close(self):
         rffi.free_charp(self.ll_chars)
@@ -144,9 +145,9 @@
         if ch == '.':
             if not self.ll_chars[i+1].isdigit():
                 self._raise("Expected digit at char %d", i+1)
-            return self.parse_float(start)
+            return self.parse_float(start, i, intval, ovf_maybe)
         elif ch == 'e' or ch == 'E':
-            return self.parse_float(start)
+            return self.parse_float(start, i, intval, ovf_maybe)
         elif ovf_maybe:
             # apparently we get a ~30% slowdown on my microbenchmark if we
             # return None instead of w_None, probably because the annotation
@@ -157,30 +158,12 @@
         self.pos = i
         return self.space.wrap(intval)
 
-        ## if ch == '.':
-        ##     is_float = True
-        ##     i, frcval, frccount = self.parse_digits(i+1)
-        ##     frcval = neg_pow_10(frcval, frccount)
-        ##     ch = self.ll_chars[i]
-        ## # check for the optional exponent part
-        ## if ch == 'E' or ch == 'e':
-        ##     is_float = True
-        ##     i, ovf_maybe, exp = self.parse_integer(i+1, allow_leading_0=True)
-        ## #
-        ## self.pos = i
-        ## if is_float:
-        ##     # build the float
-        ##     floatval = intval + frcval
-        ##     if exp != 0:
-        ##         try:
-        ##             floatval = floatval * math.pow(10, exp)
-        ##         except OverflowError:
-        ##             floatval = rfloat.INFINITY
-        ##     return self.space.wrap(floatval)
-        ## else:
-        ##     return self.space.wrap(intval)
+    def parse_float(self, start, i, intval, ovf_maybe):
+        if self.precise_float or ovf_maybe:
+            return self.parse_float_precise(start)
+        return self.parse_float_fast(i, intval)
 
-    def parse_float(self, i):
+    def parse_float_precise(self, i):
         from rpython.rlib import rdtoa
         start = rffi.ptradd(self.ll_chars, i)
         floatval = rdtoa.dg_strtod(start, self.end_ptr)
@@ -188,6 +171,27 @@
         self.pos = i + diff
         return self.space.wrap(floatval)
 
+    def parse_float_fast(self, i, intval):
+        exp = 0
+        frcval = 0
+        ch = self.ll_chars[i]
+        if ch == '.':
+            i, frcval, frccount = self.parse_digits(i+1)
+            frcval = neg_pow_10(frcval, frccount)
+            ch = self.ll_chars[i]
+        # check for the optional exponent part
+        if ch == 'E' or ch == 'e':
+            i, ovf_maybe, exp = self.parse_integer(i+1, allow_leading_0=True)
+        #
+        floatval = float(intval) + frcval
+        if exp != 0:
+            try:
+                floatval = floatval * math.pow(10, exp)
+            except OverflowError:
+                floatval = rfloat.INFINITY
+        self.pos = i
+        return self.space.wrap(floatval)
+
     def decode_numeric_slow(self, i):
         is_float = False
         start = i
@@ -422,13 +426,13 @@
         lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
         return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
 
-
-def loads(space, w_s):
+ at unwrap_spec(precise_float=bool)
+def loads(space, w_s, precise_float=True):
     if space.isinstance_w(w_s, space.w_unicode):
         raise OperationError(space.w_TypeError,
                              space.wrap("Expected utf8-encoded str, got unicode"))
     s = space.str_w(w_s)
-    decoder = JSONDecoder(space, s)
+    decoder = JSONDecoder(space, s, precise_float)
     try:
         w_res = decoder.decode_any(0)
         i = decoder.skip_whitespace(decoder.pos)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -105,6 +105,10 @@
             res = _fastjson.loads(s)
             assert type(res) is type(val)
             assert res == val
+            #
+            res = _fastjson.loads(s, precise_float=False)
+            assert type(res) is type(val)
+            assert res == val
         #
         check('42', 42)
         check('-42', -42)
@@ -135,6 +139,7 @@
         import _fastjson
         def error(s):
             raises(ValueError, _fastjson.loads, s)
+            raises(ValueError, _fastjson.loads, s, False)
         #
         error('  42   abc')
         error('.123')

From noreply at buildbot.pypy.org  Sat Jun 29 11:57:05 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat, 29 Jun 2013 11:57:05 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: do the clenaup
Message-ID: <20130629095705.3EA321C1556@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65090:ea82529ba24a
Date: 2013-06-29 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/ea82529ba24a/

Log:	do the clenaup

diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -9,6 +9,7 @@
     assert dec.skip_whitespace(0) == 3
     assert dec.skip_whitespace(3) == 3
     assert dec.skip_whitespace(8) == len(s)
+    dec.close()
 
     
 

From noreply at buildbot.pypy.org  Sat Jun 29 11:57:06 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat, 29 Jun 2013 11:57:06 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: decode_numeric_slow now it's used only
 for integers which possibly overflow, remove all the part for floats
Message-ID: <20130629095706.6D8D91C1556@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65091:6d13a4e94ad7
Date: 2013-06-29 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/6d13a4e94ad7/

Log:	decode_numeric_slow now it's used only for integers which possibly
	overflow, remove all the part for floats

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -130,13 +130,6 @@
         self._raise("Error when decoding false at char %d", i)
 
     def decode_numeric(self, i):
-        w_res = self.decode_numeric_fast(i)
-        if w_res is self.space.w_None:
-            # possible overflow, reparse it
-            return self.decode_numeric_slow(i)
-        return w_res
-
-    def decode_numeric_fast(self, i):
         start = i
         i, ovf_maybe, intval = self.parse_integer(i, allow_leading_0=False)
         #
@@ -149,11 +142,7 @@
         elif ch == 'e' or ch == 'E':
             return self.parse_float(start, i, intval, ovf_maybe)
         elif ovf_maybe:
-            # apparently we get a ~30% slowdown on my microbenchmark if we
-            # return None instead of w_None, probably because the annotation
-            # of the results geta can_be_None=True. We need to check if this
-            # is still true also for the full pypy
-            return self.space.w_None
+            return self.decode_int_slow(start)
 
         self.pos = i
         return self.space.wrap(intval)
@@ -192,40 +181,15 @@
         self.pos = i
         return self.space.wrap(floatval)
 
-    def decode_numeric_slow(self, i):
-        is_float = False
+    def decode_int_slow(self, i):
         start = i
         if self.ll_chars[i] == '-':
             i += 1
-        # skip the integral part
         while self.ll_chars[i].isdigit():
             i += 1
-        #
-        # skip the fractional part, if any
-        if self.ll_chars[i] == '.':
-            is_float = True
-            i += 1
-            while self.ll_chars[i].isdigit():
-                i += 1
-        #
-        # skip the exponent part, if any
-        if self.ll_chars[i] == 'e' or self.ll_chars[i] == 'E':
-            is_float = True
-            i += 1
-            if self.ll_chars[i] == '+' or self.ll_chars[i] == '-':
-                i += 1
-            while self.ll_chars[i].isdigit():
-                i += 1
-        #
+        s = self.getslice(start, i)
         self.pos = i
-        s = self.getslice(start, i)
-        if is_float:
-            w_func = self.space.w_float
-        else:
-            w_func = self.space.w_int
-        w_res = self.space.call_function(w_func, self.space.wrap(s))
-        #assert w_res is not None # XXX check if this brings any speedup in pypy-c
-        return w_res
+        return self.space.call_function(self.space.w_int, self.space.wrap(s))
 
     def parse_integer(self, i, allow_leading_0=False):
         "Parse a decimal number with an optional minus sign"

From noreply at buildbot.pypy.org  Sat Jun 29 12:17:57 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 29 Jun 2013 12:17:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue1392: Implement
	IOBase._checkClosed().
Message-ID: <20130629101757.952CE1C0255@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r65092:e83efe56cd20
Date: 2013-06-29 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/e83efe56cd20/

Log:	Issue1392: Implement IOBase._checkClosed().

	Patch by limeburst.

diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -101,6 +101,9 @@
             raise OperationError(
                 space.w_ValueError, space.wrap(message))
 
+    def check_closed_w(self, space):
+        self._check_closed(space)
+
     def closed_get_w(self, space):
         return space.newbool(self.__IOBase_closed)
 
@@ -277,6 +280,7 @@
     _checkReadable = interp2app(check_readable_w),
     _checkWritable = interp2app(check_writable_w),
     _checkSeekable = interp2app(check_seekable_w),
+    _checkClosed = interp2app(W_IOBase.check_closed_w),
     closed = GetSetProperty(W_IOBase.closed_get_w),
     __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, cls=W_IOBase),
     __weakref__ = make_weakref_descr(W_IOBase),
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -22,7 +22,9 @@
         import io
         with io.BufferedIOBase() as f:
             assert not f.closed
+            f._checkClosed()
         assert f.closed
+        raises(ValueError, f._checkClosed)
 
     def test_iter(self):
         import io

From noreply at buildbot.pypy.org  Sat Jun 29 14:45:46 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 29 Jun 2013 14:45:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix
	module.test_lib_pypy.pyrepl.test_wishes.
Message-ID: <20130629124546.9A0091C0255@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65093:c8a3a92d0dd6
Date: 2013-06-29 14:24 +0200
http://bitbucket.org/pypy/pypy/changeset/c8a3a92d0dd6/

Log:	Fix module.test_lib_pypy.pyrepl.test_wishes.

diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -55,6 +55,11 @@
         else:
             return c
 
+if 'a'[0] == 'a':
+    # When running tests with python2, bytes characters are bytes.
+    def _my_unctrl(c, uc=_my_unctrl):
+        return uc(ord(c))
+
 
 def disp_str(buffer, join=''.join, uc=_my_unctrl):
     """ disp_str(buffer:string) -> (string, [int])

From noreply at buildbot.pypy.org  Sat Jun 29 14:45:48 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 29 Jun 2013 14:45:48 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix
 module.test_lib_pypy.pyrepl.test_readline when run by python2
Message-ID: <20130629124548.457971C0255@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65094:cc7ed4b513e0
Date: 2013-06-29 14:45 +0200
http://bitbucket.org/pypy/pypy/changeset/cc7ed4b513e0/

Log:	Fix module.test_lib_pypy.pyrepl.test_readline when run by python2

	(in this file unicode is also defined for py3k, it's an alias to
	str)

diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -524,7 +524,7 @@
 
     def do_cmd(self, cmd):
         #print cmd
-        if isinstance(cmd[0], str):
+        if isinstance(cmd[0], (str, unicode)):
             cmd = self.commands.get(cmd[0],
                                     commands.invalid_command)(self, *cmd)
         elif isinstance(cmd[0], type):

From noreply at buildbot.pypy.org  Sat Jun 29 16:06:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 16:06:25 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: add the source revision
Message-ID: <20130629140625.E61691C1450@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65095:b8c98426a9cc
Date: 2013-06-29 10:14 +0200
http://bitbucket.org/pypy/pypy/changeset/b8c98426a9cc/

Log:	add the source revision

diff --git a/rpython/translator/stm/import_stmgc.py b/rpython/translator/stm/import_stmgc.py
--- a/rpython/translator/stm/import_stmgc.py
+++ b/rpython/translator/stm/import_stmgc.py
@@ -7,15 +7,21 @@
 The working copy comes from:  hg clone https://bitbucket.org/pypy/stmgc
 """
 
-import sys, py
+import sys, py, subprocess
 
-def mangle(lines):
-    yield "/* Imported by rpython/translator/stm/import_stmgc.py */\n"
+def mangle(lines, rev):
+    yield ("/* Imported by rpython/translator/stm/import_stmgc.py: %s */\n"
+           % rev)
     for line in lines:
         yield line
 
 def main(stmgc_dir):
     stmgc_dir = py.path.local(stmgc_dir).join('c4')
+    popen = subprocess.Popen(['hg', 'id', '-i'], cwd=str(stmgc_dir),
+                             stdout=subprocess.PIPE)
+    rev = popen.stdout.read().strip()
+    popen.wait()
+    #
     stmgc_dest = py.path.local(__file__).join('..', 'src_stm')
     plist = stmgc_dir.visit(rec=lambda p: False)
     for p in sorted(plist):
@@ -28,7 +34,7 @@
         path.join('..').ensure(dir=1)
         if path.check():
             path.remove()
-        path.write(''.join(mangle(p.readlines())))
+        path.write(''.join(mangle(p.readlines(), rev)))
         path.chmod(0444)
 
 if __name__ == '__main__':
diff --git a/rpython/translator/stm/src_stm/atomic_ops.h b/rpython/translator/stm/src_stm/atomic_ops.h
--- a/rpython/translator/stm/src_stm/atomic_ops.h
+++ b/rpython/translator/stm/src_stm/atomic_ops.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_ATOMIC_OPS_
 #define _SRCSTM_ATOMIC_OPS_
 
diff --git a/rpython/translator/stm/src_stm/dbgmem.c b/rpython/translator/stm/src_stm/dbgmem.c
--- a/rpython/translator/stm/src_stm/dbgmem.c
+++ b/rpython/translator/stm/src_stm/dbgmem.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include "stmimpl.h"
 #include <sys/mman.h>
 
diff --git a/rpython/translator/stm/src_stm/dbgmem.h b/rpython/translator/stm/src_stm/dbgmem.h
--- a/rpython/translator/stm/src_stm/dbgmem.h
+++ b/rpython/translator/stm/src_stm/dbgmem.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_DBGMEM_H
 #define _SRCSTM_DBGMEM_H
 
diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
--- a/rpython/translator/stm/src_stm/et.c
+++ b/rpython/translator/stm/src_stm/et.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 /* -*- c-basic-offset: 2 -*- */
 
 /* XXX assumes that time never wraps around (in a 'long'), which may be
diff --git a/rpython/translator/stm/src_stm/et.h b/rpython/translator/stm/src_stm/et.h
--- a/rpython/translator/stm/src_stm/et.h
+++ b/rpython/translator/stm/src_stm/et.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 /*** Extendable Timestamps
  *
  * Documentation:
diff --git a/rpython/translator/stm/src_stm/fprintcolor.c b/rpython/translator/stm/src_stm/fprintcolor.c
--- a/rpython/translator/stm/src_stm/fprintcolor.c
+++ b/rpython/translator/stm/src_stm/fprintcolor.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/fprintcolor.h b/rpython/translator/stm/src_stm/fprintcolor.h
--- a/rpython/translator/stm/src_stm/fprintcolor.h
+++ b/rpython/translator/stm/src_stm/fprintcolor.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include <stdarg.h>
 #include "stmimpl.h"
 
diff --git a/rpython/translator/stm/src_stm/gcpage.c b/rpython/translator/stm/src_stm/gcpage.c
--- a/rpython/translator/stm/src_stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/gcpage.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/gcpage.h b/rpython/translator/stm/src_stm/gcpage.h
--- a/rpython/translator/stm/src_stm/gcpage.h
+++ b/rpython/translator/stm/src_stm/gcpage.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_GCPAGE_H
 #define _SRCSTM_GCPAGE_H
 
diff --git a/rpython/translator/stm/src_stm/lists.c b/rpython/translator/stm/src_stm/lists.c
--- a/rpython/translator/stm/src_stm/lists.c
+++ b/rpython/translator/stm/src_stm/lists.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include "stmimpl.h"
 
 /************************************************************/
diff --git a/rpython/translator/stm/src_stm/lists.h b/rpython/translator/stm/src_stm/lists.h
--- a/rpython/translator/stm/src_stm/lists.h
+++ b/rpython/translator/stm/src_stm/lists.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_LISTS_H
 #define _SRCSTM_LISTS_H
 
diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c
--- a/rpython/translator/stm/src_stm/nursery.c
+++ b/rpython/translator/stm/src_stm/nursery.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/nursery.h b/rpython/translator/stm/src_stm/nursery.h
--- a/rpython/translator/stm/src_stm/nursery.h
+++ b/rpython/translator/stm/src_stm/nursery.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_NURSERY_H
 #define _SRCSTM_NURSERY_H
 
diff --git a/rpython/translator/stm/src_stm/steal.c b/rpython/translator/stm/src_stm/steal.c
--- a/rpython/translator/stm/src_stm/steal.c
+++ b/rpython/translator/stm/src_stm/steal.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/steal.h b/rpython/translator/stm/src_stm/steal.h
--- a/rpython/translator/stm/src_stm/steal.h
+++ b/rpython/translator/stm/src_stm/steal.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_STEAL_H
 #define _SRCSTM_STEAL_H
 
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 /* This file is optional: it can be used to compile all .c files
  * into one single big compilation unit.  You must compile either
  * this, or individually the other .c files in this directory.
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _STMGC_H
 #define _STMGC_H
 
diff --git a/rpython/translator/stm/src_stm/stmimpl.h b/rpython/translator/stm/src_stm/stmimpl.h
--- a/rpython/translator/stm/src_stm/stmimpl.h
+++ b/rpython/translator/stm/src_stm/stmimpl.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_IMPL_H
 #define _SRCSTM_IMPL_H
 
diff --git a/rpython/translator/stm/src_stm/stmsync.c b/rpython/translator/stm/src_stm/stmsync.c
--- a/rpython/translator/stm/src_stm/stmsync.c
+++ b/rpython/translator/stm/src_stm/stmsync.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/stmsync.h b/rpython/translator/stm/src_stm/stmsync.h
--- a/rpython/translator/stm/src_stm/stmsync.h
+++ b/rpython/translator/stm/src_stm/stmsync.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
+/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
 #ifndef _SRCSTM_STMSYNC_H
 #define _SRCSTM_STMSYNC_H
 

From noreply at buildbot.pypy.org  Sat Jun 29 16:06:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 16:06:27 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Reimport
Message-ID: <20130629140627.1D7F41C1450@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65096:e51b8ee36edd
Date: 2013-06-29 14:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e51b8ee36edd/

Log:	Reimport

diff --git a/rpython/translator/stm/import_stmgc.py b/rpython/translator/stm/import_stmgc.py
--- a/rpython/translator/stm/import_stmgc.py
+++ b/rpython/translator/stm/import_stmgc.py
@@ -9,9 +9,8 @@
 
 import sys, py, subprocess
 
-def mangle(lines, rev):
-    yield ("/* Imported by rpython/translator/stm/import_stmgc.py: %s */\n"
-           % rev)
+def mangle(lines):
+    yield "/* Imported by rpython/translator/stm/import_stmgc.py */\n"
     for line in lines:
         yield line
 
@@ -34,8 +33,10 @@
         path.join('..').ensure(dir=1)
         if path.check():
             path.remove()
-        path.write(''.join(mangle(p.readlines(), rev)))
+        path.write(''.join(mangle(p.readlines())))
         path.chmod(0444)
+    #
+    stmgc_dest.join('revision').write('%s\n' % rev)
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
diff --git a/rpython/translator/stm/src_stm/atomic_ops.h b/rpython/translator/stm/src_stm/atomic_ops.h
--- a/rpython/translator/stm/src_stm/atomic_ops.h
+++ b/rpython/translator/stm/src_stm/atomic_ops.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_ATOMIC_OPS_
 #define _SRCSTM_ATOMIC_OPS_
 
diff --git a/rpython/translator/stm/src_stm/dbgmem.c b/rpython/translator/stm/src_stm/dbgmem.c
--- a/rpython/translator/stm/src_stm/dbgmem.c
+++ b/rpython/translator/stm/src_stm/dbgmem.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 #include <sys/mman.h>
 
@@ -65,6 +65,7 @@
 
     dprintf(("stm_malloc(%zu): %p\n", sz, result));
     assert(((intptr_t)(result + sz) & (PAGE_SIZE-1)) == 0);
+    memset(result, 0xBB, sz);
     return result;
 }
 
@@ -90,6 +91,13 @@
     return accessible_pages[base] == 42;
 }
 
+void assert_cleared(char *p, size_t size)
+{
+    size_t i;
+    for (i = 0; i < size; i++)
+        assert(p[i] == 0);
+}
+
 /************************************************************/
 #endif
 
@@ -98,6 +106,7 @@
                                   size_t already_cleared)
 {
     char *baseaddr = base;
+    assert(already_cleared <= size);
 
     if (size > 2 * PAGE_SIZE) {
         int lowbits = ((intptr_t)baseaddr) & (PAGE_SIZE-1);
@@ -128,4 +137,5 @@
     if (size > already_cleared) { /* clear the final misaligned part, if any */
         memset(baseaddr, 0, size - already_cleared);
     }
+    assert_cleared(base, size);
 }
diff --git a/rpython/translator/stm/src_stm/dbgmem.h b/rpython/translator/stm/src_stm/dbgmem.h
--- a/rpython/translator/stm/src_stm/dbgmem.h
+++ b/rpython/translator/stm/src_stm/dbgmem.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_DBGMEM_H
 #define _SRCSTM_DBGMEM_H
 
@@ -8,11 +8,13 @@
 void *stm_malloc(size_t);
 void stm_free(void *, size_t);
 int _stm_can_access_memory(char *);
+void assert_cleared(char *, size_t);
 
 #else
 
 #define stm_malloc(sz)    malloc(sz)
 #define stm_free(p,sz)    free(p)
+#define assert_cleared(p,sz)     do { } while(0)
 
 #endif
 
diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
--- a/rpython/translator/stm/src_stm/et.c
+++ b/rpython/translator/stm/src_stm/et.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 /* -*- c-basic-offset: 2 -*- */
 
 /* XXX assumes that time never wraps around (in a 'long'), which may be
diff --git a/rpython/translator/stm/src_stm/et.h b/rpython/translator/stm/src_stm/et.h
--- a/rpython/translator/stm/src_stm/et.h
+++ b/rpython/translator/stm/src_stm/et.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 /*** Extendable Timestamps
  *
  * Documentation:
diff --git a/rpython/translator/stm/src_stm/fprintcolor.c b/rpython/translator/stm/src_stm/fprintcolor.c
--- a/rpython/translator/stm/src_stm/fprintcolor.c
+++ b/rpython/translator/stm/src_stm/fprintcolor.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/fprintcolor.h b/rpython/translator/stm/src_stm/fprintcolor.h
--- a/rpython/translator/stm/src_stm/fprintcolor.h
+++ b/rpython/translator/stm/src_stm/fprintcolor.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include <stdarg.h>
 #include "stmimpl.h"
 
diff --git a/rpython/translator/stm/src_stm/gcpage.c b/rpython/translator/stm/src_stm/gcpage.c
--- a/rpython/translator/stm/src_stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/gcpage.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/gcpage.h b/rpython/translator/stm/src_stm/gcpage.h
--- a/rpython/translator/stm/src_stm/gcpage.h
+++ b/rpython/translator/stm/src_stm/gcpage.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_GCPAGE_H
 #define _SRCSTM_GCPAGE_H
 
diff --git a/rpython/translator/stm/src_stm/lists.c b/rpython/translator/stm/src_stm/lists.c
--- a/rpython/translator/stm/src_stm/lists.c
+++ b/rpython/translator/stm/src_stm/lists.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 
 /************************************************************/
diff --git a/rpython/translator/stm/src_stm/lists.h b/rpython/translator/stm/src_stm/lists.h
--- a/rpython/translator/stm/src_stm/lists.h
+++ b/rpython/translator/stm/src_stm/lists.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_LISTS_H
 #define _SRCSTM_LISTS_H
 
diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c
--- a/rpython/translator/stm/src_stm/nursery.c
+++ b/rpython/translator/stm/src_stm/nursery.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 
 
@@ -73,6 +73,15 @@
         P = (gcptr)cur;
         P->h_tid = tid;
     }
+#ifdef _GC_DEBUG
+    if (P != NULL) {
+        assert(P->h_tid != 0);
+        assert_cleared(((char *)P) + sizeof(revision_t),
+                       size - sizeof(revision_t));
+    }
+    else
+        assert(tid == -1);
+#endif
     return P;
 }
 
@@ -539,18 +548,6 @@
     */
     teardown_minor_collect(d);
 
-    /* if in debugging mode, we allocate a different nursery and make
-       the old one inaccessible
-    */
-#if defined(_GC_DEBUG) && _GC_DEBUG >= 2
-    stm_free(d->nursery_base, GC_NURSERY);
-    d->nursery_base = stm_malloc(GC_NURSERY);
-    d->nursery_end = d->nursery_base + GC_NURSERY;
-    d->nursery_nextlimit = d->nursery_base;
-    dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
-             d->nursery_end));
-#endif
-
     /* When doing minor collections with the nursery "mostly empty",
        as occurs when other threads force major collections but this
        thread didn't do much at all, then we clear the nursery using
@@ -573,6 +570,20 @@
         d->nursery_cleared = NC_REGULAR;
     }
 
+    /* if in debugging mode, we allocate a different nursery and make
+       the old one inaccessible
+    */
+#if defined(_GC_DEBUG) && _GC_DEBUG >= 2
+    if (d->nursery_cleared == NC_ALREADY_CLEARED)
+        assert_cleared(d->nursery_base, GC_NURSERY);
+    stm_free(d->nursery_base, GC_NURSERY);
+    d->nursery_base = stm_malloc(GC_NURSERY);
+    d->nursery_end = d->nursery_base + GC_NURSERY;
+    dprintf(("minor: nursery moved to [%p to %p]\n", d->nursery_base,
+             d->nursery_end));
+    if (d->nursery_cleared == NC_ALREADY_CLEARED)
+        memset(d->nursery_base, 0, GC_NURSERY);
+#endif
     d->nursery_current = d->nursery_base;
     d->nursery_nextlimit = d->nursery_base;
 
@@ -641,6 +652,7 @@
 
         /* Allocate it externally, and make it old */
         gcptr P = stmgcpage_malloc(allocate_size);
+        memset(P, 0, allocate_size);
         P->h_tid = tid | GCFLAG_OLD;
         gcptrlist_insert(&d->old_objects_to_trace, P);
         return P;
@@ -673,5 +685,7 @@
     assert(d->nursery_current <= d->nursery_nextlimit);
 
     P->h_tid = tid;
+    assert_cleared(((char *)P) + sizeof(revision_t),
+                   allocate_size - sizeof(revision_t));
     return P;
 }
diff --git a/rpython/translator/stm/src_stm/nursery.h b/rpython/translator/stm/src_stm/nursery.h
--- a/rpython/translator/stm/src_stm/nursery.h
+++ b/rpython/translator/stm/src_stm/nursery.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_NURSERY_H
 #define _SRCSTM_NURSERY_H
 
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/revision
@@ -0,0 +1,1 @@
+c856b6a0d157
diff --git a/rpython/translator/stm/src_stm/steal.c b/rpython/translator/stm/src_stm/steal.c
--- a/rpython/translator/stm/src_stm/steal.c
+++ b/rpython/translator/stm/src_stm/steal.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/steal.h b/rpython/translator/stm/src_stm/steal.h
--- a/rpython/translator/stm/src_stm/steal.h
+++ b/rpython/translator/stm/src_stm/steal.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_STEAL_H
 #define _SRCSTM_STEAL_H
 
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 /* This file is optional: it can be used to compile all .c files
  * into one single big compilation unit.  You must compile either
  * this, or individually the other .c files in this directory.
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _STMGC_H
 #define _STMGC_H
 
diff --git a/rpython/translator/stm/src_stm/stmimpl.h b/rpython/translator/stm/src_stm/stmimpl.h
--- a/rpython/translator/stm/src_stm/stmimpl.h
+++ b/rpython/translator/stm/src_stm/stmimpl.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_IMPL_H
 #define _SRCSTM_IMPL_H
 
diff --git a/rpython/translator/stm/src_stm/stmsync.c b/rpython/translator/stm/src_stm/stmsync.c
--- a/rpython/translator/stm/src_stm/stmsync.c
+++ b/rpython/translator/stm/src_stm/stmsync.c
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 
 
diff --git a/rpython/translator/stm/src_stm/stmsync.h b/rpython/translator/stm/src_stm/stmsync.h
--- a/rpython/translator/stm/src_stm/stmsync.h
+++ b/rpython/translator/stm/src_stm/stmsync.h
@@ -1,4 +1,4 @@
-/* Imported by rpython/translator/stm/import_stmgc.py: 45380d4cb89c */
+/* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_STMSYNC_H
 #define _SRCSTM_STMSYNC_H
 

From noreply at buildbot.pypy.org  Sat Jun 29 16:06:28 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 16:06:28 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: in-progress
Message-ID: <20130629140628.4C5E31C1450@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65097:039f1f916424
Date: 2013-06-29 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/039f1f916424/

Log:	in-progress

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/stmgc.py
@@ -0,0 +1,73 @@
+"""
+Minimal GC interface for STM.  The actual GC is implemented in C code,
+from rpython/translator/stm/src_stm/.
+"""
+
+from rpython.memory.gc.base import MovingGCBase
+from rpython.rtyper.lltypesystem import lltype, llmemory, llgroup, rffi
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rlib.debug import ll_assert
+
+
+class PrebuiltStmGCHeader(llmemory.Symbolic):
+    def __init__(self, typeid16):
+        self.typeid16 = typeid16
+    def annotation(self):
+        from rpython.annotator import model
+        return model.SomePtr(self.lltype())
+    def lltype(self):
+        return lltype.Ptr(StmGC.HDR)
+
+
+class StmGC(MovingGCBase):
+    _alloc_flavor_ = "raw"
+    inline_simple_malloc = True
+    inline_simple_malloc_varsize = True
+    needs_write_barrier = True
+    prebuilt_gc_objects_are_static_roots = False
+    malloc_zero_filled = True
+    #gcflag_extra = GCFLAG_EXTRA
+
+    HDR = rffi.COpaque('struct stm_object_s')
+    typeid_is_in_field = None
+
+    TRANSLATION_PARAMS = {}
+
+    def get_type_id(self, obj):
+        return llop.stm_get_tid(llgroup.HALFWORD, obj)
+
+    def init_gc_object_immortal(self, addr, typeid16, flags=0):
+        assert flags == 0
+        return PrebuiltStmGCHeader(typeid16)
+
+    def malloc_fixedsize_clear(self, typeid, size,
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
+        ll_assert(not needs_finalizer, 'XXX')
+        ll_assert(not is_finalizer_light, 'XXX')
+        ll_assert(not contains_weakptr, 'XXX')
+        return llop.stm_allocate(llmemory.GCREF, size, typeid)
+
+    def malloc_varsize_clear(self, typeid, length, size, itemsize,
+                             offset_to_length):
+        ll_assert(False, 'XXX')
+        return llop.stm_allocate(llmemory.GCREF)
+
+    def collect(self, gen=1):
+        """Do a minor (gen=0) or major (gen>0) collection."""
+        if gen > 0:
+            llop.stm_major_collect(lltype.Void)
+        else:
+            llop.stm_minor_collect(lltype.Void)
+
+    def writebarrier_before_copy(self, source_addr, dest_addr,
+                                 source_start, dest_start, length):
+        ll_assert(False, 'XXX')
+        return False
+
+    def id(self, gcobj):
+        return llop.stm_id(lltype.Signed, gcobj)
+
+    def identityhash(self, gcobj):
+        return llop.stm_hash(lltype.Signed, gcobj)
diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -1,205 +1,5 @@
 from rpython.memory.gctransform import shadowstack
-from rpython.memory.gctransform.framework import BaseRootWalker
-from rpython.memory.gctransform.framework import sizeofaddr
-from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rtyper import rmodel
-from rpython.annotator import model as annmodel
-from rpython.rlib.debug import fatalerror_notb
-from rpython.rlib.nonconst import NonConstant
-from rpython.rlib.objectmodel import specialize
-
-
-END_MARKER = -8      # keep in sync with src_stm/rpyintf.c
 
 
 class StmFrameworkGCTransformer(shadowstack.ShadowStackFrameworkGCTransformer):
-
-    def _declare_functions(self, GCClass, getfn, s_gc, *args):
-        super(StmFrameworkGCTransformer, self)._declare_functions(
-            GCClass, getfn, s_gc, *args)
-        self.stm_start_ptr = getfn(
-            self.gcdata.gc.start_transaction.im_func,
-            [s_gc], annmodel.s_None)
-        self.stm_stop_ptr = getfn(
-            self.gcdata.gc.stop_transaction.im_func,
-            [s_gc], annmodel.s_None)
-##        self.stm_writebarrier_ptr = getfn(
-##            self.gcdata.gc.stm_writebarrier,
-##            [annmodel.SomeAddress()], annmodel.SomeAddress())
-##        self.stm_normalize_global_ptr = getfn(
-##            self.gcdata.gc.stm_normalize_global,
-##            [annmodel.SomeAddress()], annmodel.SomeAddress())
-##        self.write_barrier_failing_case_ptr = getfn(
-##            self.gcdata.gc._stm_write_barrier_global,
-##            [annmodel.SomeAddress()], annmodel.SomeAddress())
-
-    def build_root_walker(self):
-        return StmShadowStackRootWalker(self)
-
-    def _gc_adr_of_gcdata_attr(self, hop, attrname):
-        x = self.root_walker.stackgcdata
-        c_const_stackgcdata = rmodel.inputconst(lltype.typeOf(x), x)
-        op = hop.spaceop
-        ofs = llmemory.offsetof(c_const_stackgcdata.concretetype.TO,
-                                attrname)
-        c_ofs = rmodel.inputconst(lltype.Signed, ofs)
-        v_gcdata_adr = hop.genop('cast_ptr_to_adr', [c_const_stackgcdata],
-                                 resulttype=llmemory.Address)
-        hop.genop('adr_add', [v_gcdata_adr, c_ofs], resultvar=op.result)
-
-##    def gct_stm_writebarrier(self, hop):
-##        op = hop.spaceop
-##        v_adr = hop.genop('cast_ptr_to_adr',
-##                          [op.args[0]], resulttype=llmemory.Address)
-##        v_localadr = hop.genop("direct_call",
-##                               [self.stm_writebarrier_ptr, v_adr],
-##                               resulttype=llmemory.Address)
-##        hop.genop('cast_adr_to_ptr', [v_localadr], resultvar=op.result)
-
-##    def gct_stm_normalize_global(self, hop):
-##        op = hop.spaceop
-##        v_adr = hop.genop('cast_ptr_to_adr',
-##                          [op.args[0]], resulttype=llmemory.Address)
-##        v_globaladr = hop.genop("direct_call",
-##                                [self.stm_normalize_global_ptr, v_adr],
-##                                resulttype=llmemory.Address)
-##        hop.genop('cast_adr_to_ptr', [v_globaladr], resultvar=op.result)
-
-    def gct_stm_start_transaction(self, hop):
-        livevars = self.push_roots(hop)
-        hop.genop("direct_call", [self.stm_start_ptr, self.c_const_gc])
-        self.pop_roots(hop, livevars)
-
-    def gct_stm_stop_transaction(self, hop):
-        livevars = self.push_roots(hop)
-        hop.genop("direct_call", [self.stm_stop_ptr, self.c_const_gc])
-        self.pop_roots(hop, livevars)
-
-##    def gct_gc_store(self, hop):
-##        hop.rename('stm_gc_store')
-
-
-class StmShadowStackRootWalker(BaseRootWalker):
-    need_root_stack = True
-    root_stack_depth = 163840
-
-    def __init__(self, gctransformer):
-        from rpython.memory.gctransform import shadowstack
-        #
-        BaseRootWalker.__init__(self, gctransformer)
-        # we use the thread-local self.stackgcdata to store state;
-        # 'self' is frozen.
-        STACKGCDATA = lltype.Struct('STACKGCDATA',
-            ('root_stack_top',  llmemory.Address),
-            ('root_stack_base', llmemory.Address),
-            hints={'stm_thread_local': True})
-        stackgcdata = lltype.malloc(STACKGCDATA, immortal=True)
-        self.stackgcdata = stackgcdata
-
-        def incr_stack(n):
-            top = stackgcdata.root_stack_top
-            stackgcdata.root_stack_top = top + n*sizeofaddr
-            return top
-        self.incr_stack = incr_stack
-
-        def decr_stack(n):
-            top = stackgcdata.root_stack_top - n*sizeofaddr
-            stackgcdata.root_stack_top = top
-            return top
-        self.decr_stack = decr_stack
-
-        root_iterator = shadowstack.get_root_iterator(gctransformer)
-        @specialize.argtype(1)
-        def walk_stack_root(callback, arg, end):
-            root_iterator.setcontext(NonConstant(llmemory.NULL))
-            gc = self.gc
-            addr = end
-            while True:
-                addr = root_iterator.nextleft(gc, llmemory.NULL, addr)
-                if addr.signed[0] == END_MARKER:
-                    break
-                callback(arg, addr)
-            return addr
-        self.rootstackhook = walk_stack_root
-
-        rsd = gctransformer.root_stack_depth
-        if rsd is not None:
-            self.root_stack_depth = rsd
-
-    def need_thread_support(self, gctransformer, getfn):
-        gc = gctransformer.gcdata.gc
-        #
-        def gc_thread_start():
-            self.allocate_shadow_stack()
-            gc.setup_thread()
-        #
-        def gc_thread_die():
-            gc.teardown_thread()
-            self.free_shadow_stack()
-        #
-        self.thread_start_ptr = getfn(
-            gc_thread_start,
-            [], annmodel.s_None)
-        self.thread_die_ptr = getfn(
-            gc_thread_die,
-            [], annmodel.s_None)
-
-    def setup_root_walker(self):
-        self.allocate_shadow_stack()
-        self.gcdata.main_thread_stack_base = self.stackgcdata.root_stack_base
-        BaseRootWalker.setup_root_walker(self)
-
-    def allocate_shadow_stack(self):
-        root_stack_size = sizeofaddr * self.root_stack_depth
-        base = llmemory.raw_malloc(root_stack_size)
-        if base == llmemory.NULL:
-            raise MemoryError
-        base.signed[0] = END_MARKER
-        shift = llmemory.sizeof(llmemory.Address)
-        self.stackgcdata.root_stack_base = base
-        self.stackgcdata.root_stack_top  = base + shift
-
-    def free_shadow_stack(self):
-        base = self.stackgcdata.root_stack_base
-        llmemory.raw_free(base)
-
-    def walk_roots(self, c1, c2, c3):
-        raise NotImplementedError
-
-    def walk_stack_roots(self, *args):
-        "NOT_RPYTHON"
-        raise NotImplementedError
-
-    @specialize.argtype(2)
-    def walk_current_stack_roots(self, collect_stack_root, arg):
-        stackgcdata = self.stackgcdata
-        self.rootstackhook(collect_stack_root, arg,
-                           stackgcdata.root_stack_top)
-
-    @specialize.argtype(2)
-    def walk_current_nongc_roots(self, collect_nongc_root, arg):
-        gcdata = self.gcdata
-        gc = self.gc
-        addr = gcdata.static_root_start
-        end = gcdata.static_root_nongcend
-        while addr != end:
-            result = addr.address[0]
-            if gc.points_to_valid_gc_object(result):
-                collect_nongc_root(arg, result)
-            addr += sizeofaddr
-
-    @specialize.argtype(2)
-    def walk_all_stack_roots(self, collect_stack_root, arg):
-        # assume that we have the ll_global_lock here
-        stmtls = self.gc.linked_list_stmtls
-        while stmtls is not None:
-            # for every stmtls:
-            root_stack_base = stmtls.adr_of_stack_base.address[0]
-            root_stack_top  = stmtls.adr_of_stack_top.address[0]
-            # we walk all segments of the shadow stack, each of them
-            # with an END_MARKER at the bottom, until we reach the
-            # marker that is at root_stack_base.
-            while root_stack_top != root_stack_base:
-                root_stack_top = self.rootstackhook(collect_stack_root, arg,
-                                                    root_stack_top)
-            stmtls = stmtls.linked_list_next
+    pass
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -422,21 +422,13 @@
     # to keep them as operations until the genc stage)
 
     'stm_barrier':            LLOp(sideeffects=False),
-    'stm_read_barrier':       LLOp(),    # explicit read barrier, special case
-    'stm_write_barrier':      LLOp(),    # explicit write barrier, special case
+    'stm_allocate':           LLOp(sideeffects=False),
     'stm_become_inevitable':  LLOp(),
-    'stm_ptr_eq':             LLOp(sideeffects=False),
-    'stm_start_transaction':  LLOp(canrun=True, canmallocgc=True),
-    'stm_stop_transaction':   LLOp(canrun=True, canmallocgc=True),
-    #'stm_jit_invoke_code':    LLOp(canmallocgc=True),
-    'stm_threadlocalref_get': LLOp(sideeffects=False),
-    'stm_threadlocalref_set': LLOp(),
-    'stm_threadlocalref_llset':   LLOp(),
-    'stm_threadlocalref_llcount': LLOp(sideeffects=False),
-    'stm_threadlocalref_lladdr':  LLOp(sideeffects=False),
-    'stm_abort_info_push':    LLOp(),
-    'stm_extraref_llcount':   LLOp(sideeffects=False),
-    'stm_extraref_lladdr':    LLOp(sideeffects=False),
+    'stm_minor_collect':      LLOp(),
+    'stm_major_collect':      LLOp(),
+    'stm_get_tid':            LLOp(canfold=True),
+    'stm_id':                 LLOp(sideeffects=False),
+    'stm_hash':               LLOp(sideeffects=False),
 
     # __________ address operations __________
 
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -19,13 +19,13 @@
     assert type(category_change) is str and len(category_change) == 3   # "x2y"
     arg = funcgen.expr(op.args[1])
     result = funcgen.expr(op.result)
-    return '%s = STM_BARRIER_%s(%s);' % (result, category_change, arg)
+    return '%s = stm_barrier_%s(%s);' % (result, category_change, arg)
 
 def stm_ptr_eq(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
     result = funcgen.expr(op.result)
-    return '%s = STM_PTR_EQ(%s, %s);' % (result, arg0, arg1)
+    return '%s = stm_pointer_equal(%s, %s);' % (result, arg0, arg1)
 
 def stm_become_inevitable(funcgen, op):
     try:
@@ -33,7 +33,7 @@
     except IndexError:
         info = "rstm.become_inevitable"    # cannot insert it in 'llop'
     string_literal = c_string_constant(info)
-    return 'BecomeInevitable(%s);' % (string_literal,)
+    return 'stm_become_inevitable(%s);' % (string_literal,)
 
 ##def stm_jit_invoke_code(funcgen, op):
 ##    return funcgen.OP_DIRECT_CALL(op)
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -9,7 +9,7 @@
 
 eci = ExternalCompilationInfo(
     include_dirs = [cdir, cdir2],
-    includes = ['src_stm/et.h'],
+    includes = ['src_stm/stmgc.h'],
     pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT,
                         '#define RPY_STM 1'],
 )
diff --git a/rpython/translator/stm/test/support.py b/rpython/translator/stm/test/support.py
--- a/rpython/translator/stm/test/support.py
+++ b/rpython/translator/stm/test/support.py
@@ -4,12 +4,11 @@
 
 
 class CompiledSTMTests(StandaloneTests):
-    gc = "stmgc"
 
     def compile(self, entry_point, **kwds):
         from pypy.config.pypyoption import get_pypy_config
         self.config = get_pypy_config(translating=True)
-        self.config.translation.gc = self.gc
+        self.config.translation.gc = "stmgc"
         self.config.translation.stm = True
         #
         # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown
@@ -22,7 +21,3 @@
         finally:
             RaiseAnalyzer.fail_on_unknown_operation = False
         return res
-
-
-class NoGcCompiledSTMTests(CompiledSTMTests):
-    gc = "none"
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -3,19 +3,25 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr
-from rpython.translator.stm.test.support import NoGcCompiledSTMTests
 from rpython.translator.stm.test.support import CompiledSTMTests
 from rpython.translator.stm.test import targetdemo2
 
 
-class TestNoGcSTMTranslated(NoGcCompiledSTMTests):
-    def test_nogc_targetdemo(self):
-        t, cbuilder = self.compile(targetdemo2.entry_point)
-        data, dataerr = cbuilder.cmdexec('4 100', err=True)
-        assert 'check ok!' in data
+class TestSTMTranslated(CompiledSTMTests):
 
-
-class TestSTMTranslated(CompiledSTMTests):
+    def test_malloc(self):
+        class Foo:
+            pass
+        def entry_point(argv):
+            lst = []
+            for i in range(int(argv[1])):
+                lst.append(Foo())
+            print '<', len(lst), '>'
+            return 0
+        #
+        t, cbuilder = self.compile(entry_point, backendopt=True)
+        data = cbuilder.cmdexec('42')
+        assert '< 42 >' in data, "got: %r" % (data,)
 
     def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo2.entry_point)

From noreply at buildbot.pypy.org  Sat Jun 29 17:14:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 17:14:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the tests
Message-ID: <20130629151447.A1EE41C13FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r65098:839b05ae6bb9
Date: 2013-06-29 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/839b05ae6bb9/

Log:	Fix the tests

diff --git a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
--- a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
+++ b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
@@ -32,14 +32,6 @@
     return d
 
 
-def test_syslog():
-    try:
-        import lib_pypy.syslog
-    except ImportError:
-        py.test.skip('no syslog on this platform')
-    d = run('syslog.ctc.py', '_syslog_cache.py')
-    assert 'LOG_NOTICE' in d
-
 def test_resource():
     try:
         import lib_pypy.resource
diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -8,7 +8,7 @@
 def test_basic():
     g = grp.getgrnam("root")
     assert g.gr_gid == 0
-    assert g.gr_mem == ['root']
+    assert g.gr_mem == ['root'] or g.gr_mem == []
     assert g.gr_name == 'root'
     assert isinstance(g.gr_passwd, str)    # usually just 'x', don't hope :-)
 
diff --git a/pypy/module/test_lib_pypy/test_syslog.py b/pypy/module/test_lib_pypy/test_syslog.py
--- a/pypy/module/test_lib_pypy/test_syslog.py
+++ b/pypy/module/test_lib_pypy/test_syslog.py
@@ -1,9 +1,13 @@
 from __future__ import absolute_import
-import py
+import sys, py
 try:
     from lib_pypy import syslog
 except ImportError:
     py.test.skip('no syslog on this platform')
+except AssertionError:
+    if '__pypy__' in sys.builtin_module_names:
+        raise
+    py.test.skip('AssertionError during import (wrong cffi version?)')
 
 # XXX very minimal test
 

From noreply at buildbot.pypy.org  Sat Jun 29 18:29:52 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 29 Jun 2013 18:29:52 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Port p3k diffs in defunct strutil.py to
 the rpython/ directory.
Message-ID: <20130629162952.530571C2FB2@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65100:2ab06d170d6f
Date: 2013-06-29 18:29 +0200
http://bitbucket.org/pypy/pypy/changeset/2ab06d170d6f/

Log:	Port p3k diffs in defunct strutil.py to the rpython/ directory.

	Yes, I know this is wrong, but:

	- it's a branch!

	- this is the fastest way to have a working py3k after the branch
	merge, so that other people can work on other parts.

	- I will remove those differences in rpython/, at least we start
	from a stable interpreter.

	- some pieces of code that were moved to rpython/ are specific to
	python2, like the 'L' suffix, and the 'long' type in error
	messages.

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -673,6 +673,7 @@
 # String parsing support
 # ---------------------------
 
+ at objectmodel.enforceargs(unicode, None)
 def string_to_int(s, base=10):
     """Utility to converts a string to an integer.
     If base is 0, the proper base is guessed based on the leading
@@ -683,7 +684,7 @@
         ParseStringOverflowError, \
         ParseStringError, strip_spaces
     s = literal = strip_spaces(s)
-    p = NumberStringParser(s, literal, base, 'int')
+    p = NumberStringParser(s, literal, base, u'int')
     base = p.base
     result = 0
     while True:
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -4,7 +4,7 @@
 from rpython.rlib.rfloat import isinf, isnan
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.debug import make_sure_not_resized, check_regular_int
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, enforceargs
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper import extregistry
@@ -254,16 +254,13 @@
 
     @staticmethod
     @jit.elidable
+    @enforceargs(unicode, None)
     def fromstr(s, base=0):
-        """As string_to_int(), but ignores an optional 'l' or 'L' suffix
-        and returns an rbigint."""
+        """As string_to_int(), but returns an rbigint."""
         from rpython.rlib.rstring import NumberStringParser, \
             strip_spaces
         s = literal = strip_spaces(s)
-        if (s.endswith('l') or s.endswith('L')) and base < 22:
-            # in base 22 and above, 'L' is a valid digit!  try: long('L',22)
-            s = s[:-1]
-        parser = NumberStringParser(s, literal, base, 'long')
+        parser = NumberStringParser(s, literal, base, u'int')
         return rbigint._from_numberstring_parser(parser)
 
     @staticmethod
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -27,6 +27,7 @@
 
 globals().update(rffi_platform.configure(CConfig))
 
+ at objectmodel.enforceargs(unicode)
 def string_to_float(s):
     """
     Conversion of string to float.
@@ -40,10 +41,25 @@
     s = strip_spaces(s)
 
     if not s:
-        raise ParseStringError("empty string for float()")
+        raise ParseStringError(u"empty string for float()")
 
 
-    low = s.lower()
+    try:
+        ascii_s = s.encode('ascii')
+    except UnicodeEncodeError:
+        # if s is not ASCII, it certainly is not a float literal (because the
+        # unicode-decimal to ascii-decimal conversion already happened
+        # earlier). We just set ascii_s to something which will fail when
+        # passed to rstring_to_float, to keep the code as similar as possible
+        # to the one we have on default.
+        #
+        # Note that CPython does something different and it encodes the string
+        # to UTF-8 before trying to parse it. We cannot since .encode('utf-8')
+        # is not RPython. However, it doesn't change anything since the UTF-8
+        # encoded string would make rstring_to_float to fail anyway.
+        ascii_s = "not a float"
+
+    low = ascii_s.lower()
     if low == "-inf" or low == "-infinity":
         return -INFINITY
     elif low == "inf" or low == "+inf":
@@ -56,9 +72,11 @@
         return -NAN
 
     try:
-        return rstring_to_float(s)
+        return rstring_to_float(ascii_s)
     except ValueError:
-        raise ParseStringError("invalid literal for float(): '%s'" % s)
+        # note that we still put the original unicode string in the error
+        # message, not ascii_s
+        raise ParseStringError(u"invalid literal for float(): '%s'" % s)
 
 def rstring_to_float(s):
     return rstring_to_float_impl(s)
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -4,10 +4,11 @@
 
 from rpython.annotator.model import (SomeObject, SomeString, s_None, SomeChar,
     SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePtr, SomePBC)
-from rpython.rlib.objectmodel import newlist_hint, specialize
+from rpython.rlib.objectmodel import newlist_hint, specialize, enforceargs
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pairtype
+from rpython.tool.sourcetools import with_unicode_literals
 from rpython.rlib import jit
 
 
@@ -188,6 +189,8 @@
 
 # -------------- numeric parsing support --------------------
 
+ at enforceargs(unicode)
+ at with_unicode_literals
 def strip_spaces(s):
     # XXX this is not locale-dependent
     p = 0
@@ -200,6 +203,7 @@
     return s[p:q]
 
 class ParseStringError(Exception):
+    @enforceargs(None, unicode)
     def __init__(self, msg):
         self.msg = msg
 
@@ -211,9 +215,11 @@
 class NumberStringParser:
 
     def error(self):
-        raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
+        raise ParseStringError(u"invalid literal for %s() with base %d: '%s'" %
                                (self.fname, self.original_base, self.literal))
 
+    @enforceargs(None, unicode, unicode, int, unicode)
+    @with_unicode_literals
     def __init__(self, s, literal, base, fname):
         self.literal = literal
         self.fname = fname
@@ -236,7 +242,7 @@
             else:
                 base = 10
         elif base < 2 or base > 36:
-            raise ParseStringError, "%s() base must be >= 2 and <= 36" % (fname,)
+            raise ParseStringError, u"%s() base must be >= 2 and <= 36" % (fname,)
         self.base = base
 
         if base == 16 and (s.startswith('0x') or s.startswith('0X')):
@@ -254,6 +260,7 @@
     def rewind(self):
         self.i = 0
 
+    @with_unicode_literals
     def next_digit(self): # -1 => exhausted
         if self.i < self.n:
             c = self.s[self.i]

From noreply at buildbot.pypy.org  Sat Jun 29 18:29:50 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 29 Jun 2013 18:29:50 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20130629162950.DA9781C1556@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r65099:1b957582573c
Date: 2013-06-29 16:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1b957582573c/

Log:	hg merge default

diff too long, truncating to 2000 out of 2591 lines

diff --git a/lib_pypy/ctypes_config_cache/syslog.ctc.py b/lib_pypy/ctypes_config_cache/syslog.ctc.py
deleted file mode 100644
--- a/lib_pypy/ctypes_config_cache/syslog.ctc.py
+++ /dev/null
@@ -1,75 +0,0 @@
-"""
-'ctypes_configure' source for syslog.py.
-Run this to rebuild _syslog_cache.py.
-"""
-
-from ctypes_configure.configure import (configure,
-    ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger)
-import dumpcache
-
-
-_CONSTANTS = (
-    'LOG_EMERG',
-    'LOG_ALERT',
-    'LOG_CRIT',
-    'LOG_ERR',
-    'LOG_WARNING',
-    'LOG_NOTICE',
-    'LOG_INFO',
-    'LOG_DEBUG',
-
-    'LOG_PID',
-    'LOG_CONS',
-    'LOG_NDELAY',
-
-    'LOG_KERN',
-    'LOG_USER',
-    'LOG_MAIL',
-    'LOG_DAEMON',
-    'LOG_AUTH',
-    'LOG_LPR',
-    'LOG_LOCAL0',
-    'LOG_LOCAL1',
-    'LOG_LOCAL2',
-    'LOG_LOCAL3',
-    'LOG_LOCAL4',
-    'LOG_LOCAL5',
-    'LOG_LOCAL6',
-    'LOG_LOCAL7',
-)
-_OPTIONAL_CONSTANTS = (
-    'LOG_NOWAIT',
-    'LOG_PERROR',
-
-    'LOG_SYSLOG',
-    'LOG_CRON',
-    'LOG_UUCP',
-    'LOG_NEWS',
-)
-
-# Constant aliases if there are not defined
-_ALIAS = (
-    ('LOG_SYSLOG', 'LOG_DAEMON'),
-    ('LOG_CRON', 'LOG_DAEMON'),
-    ('LOG_NEWS', 'LOG_MAIL'),
-    ('LOG_UUCP', 'LOG_MAIL'),
-)
-
-class SyslogConfigure:
-    _compilation_info_ = ExternalCompilationInfo(includes=['sys/syslog.h'])
-for key in _CONSTANTS:
-    setattr(SyslogConfigure, key, ConstantInteger(key))
-for key in _OPTIONAL_CONSTANTS:
-    setattr(SyslogConfigure, key, DefinedConstantInteger(key))
-
-config = configure(SyslogConfigure)
-for key in _OPTIONAL_CONSTANTS:
-    if config[key] is None:
-        del config[key]
-for alias, key in _ALIAS:
-    config.setdefault(alias, config[key])
-
-all_constants = config.keys()
-all_constants.sort()
-config['ALL_CONSTANTS'] = tuple(all_constants)
-dumpcache.dumpcache2('syslog', config)
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -46,16 +46,16 @@
         if parent is not None:
             self.parent = parent
 
-    def switch(self, *args):
+    def switch(self, *args, **kwds):
         "Switch execution to this greenlet, optionally passing the values "
         "given as argument(s).  Returns the value passed when switching back."
-        return self.__switch('switch', args)
+        return self.__switch('switch', (args, kwds))
 
     def throw(self, typ=GreenletExit, val=None, tb=None):
         "raise exception in greenlet, return value passed when switching back"
         return self.__switch('throw', typ, val, tb)
 
-    def __switch(target, methodname, *args):
+    def __switch(target, methodname, *baseargs):
         current = getcurrent()
         #
         while not (target.__main or _continulet.is_pending(target)):
@@ -65,9 +65,9 @@
                     greenlet_func = _greenlet_start
                 else:
                     greenlet_func = _greenlet_throw
-                _continulet.__init__(target, greenlet_func, *args)
+                _continulet.__init__(target, greenlet_func, *baseargs)
                 methodname = 'switch'
-                args = ()
+                baseargs = ()
                 target.__started = True
                 break
             # already done, go to the parent instead
@@ -78,11 +78,15 @@
         #
         try:
             unbound_method = getattr(_continulet, methodname)
-            args = unbound_method(current, *args, to=target)
+            args, kwds = unbound_method(current, *baseargs, to=target)
         finally:
             _tls.current = current
         #
-        if len(args) == 1:
+        if kwds:
+            if args:
+                return args, kwds
+            return kwds
+        elif len(args) == 1:
             return args[0]
         else:
             return args
@@ -128,15 +132,15 @@
     _tls.main = gmain
     _tls.current = gmain
 
-def _greenlet_start(greenlet, args):
+def _greenlet_start(greenlet, (args, kwds)):
     _tls.current = greenlet
     try:
-        res = greenlet.run(*args)
+        res = greenlet.run(*args, **kwds)
     except GreenletExit as e:
         res = e
     finally:
         _continuation.permute(greenlet, greenlet.parent)
-    return (res,)
+    return ((res,), None)
 
 def _greenlet_throw(greenlet, exc, value, tb):
     _tls.current = greenlet
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -9,6 +9,7 @@
 
 from ctypes import Structure, c_char_p, c_int, POINTER
 from ctypes_support import standard_c_lib as libc
+import _structseq
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -24,32 +25,13 @@
         ('gr_mem', POINTER(c_char_p)),
         )
 
-class Group(object):
-    def __init__(self, gr_name, gr_passwd, gr_gid, gr_mem):
-        self.gr_name = gr_name
-        self.gr_passwd = gr_passwd
-        self.gr_gid = gr_gid
-        self.gr_mem = gr_mem
+class struct_group:
+    __metaclass__ = _structseq.structseqtype
 
-    def __getitem__(self, item):
-        if item == 0:
-            return self.gr_name
-        elif item == 1:
-            return self.gr_passwd
-        elif item == 2:
-            return self.gr_gid
-        elif item == 3:
-            return self.gr_mem
-        else:
-            raise IndexError(item)
-
-    def __len__(self):
-        return 4
-
-    def __repr__(self):
-        return str((self.gr_name, self.gr_passwd, self.gr_gid, self.gr_mem))
-
-    # whatever else...
+    gr_name   = _structseq.structseqfield(0)
+    gr_passwd = _structseq.structseqfield(1)
+    gr_gid    = _structseq.structseqfield(2)
+    gr_mem    = _structseq.structseqfield(3)
 
 libc.getgrgid.argtypes = [gid_t]
 libc.getgrgid.restype = POINTER(GroupStruct)
@@ -72,10 +54,10 @@
     while res.contents.gr_mem[i]:
         mem.append(res.contents.gr_mem[i])
         i += 1
-    return Group(os.fsdecode(res.contents.gr_name),
-                 os.fsdecode(res.contents.gr_passwd),
-                 res.contents.gr_gid,
-                 mem)
+    return struct_group((os.fsdecode(res.contents.gr_name),
+                         os.fsdecode(res.contents.gr_passwd),
+                         res.contents.gr_gid,
+                         mem))
 
 @builtinify
 def getgrgid(gid):
diff --git a/lib_pypy/pyrepl/curses.py b/lib_pypy/pyrepl/curses.py
--- a/lib_pypy/pyrepl/curses.py
+++ b/lib_pypy/pyrepl/curses.py
@@ -19,11 +19,15 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-# avoid importing the whole curses, if possible
-try:
+# If we are running on top of pypy, we import only _minimal_curses.
+# Don't try to fall back to _curses, because that's going to use cffi
+# and fall again more loudly.
+import sys
+if '__pypy__' in sys.builtin_module_names:
     # pypy case
     import _minimal_curses as _curses
-except ImportError:
+else:
+    # cpython case
     try:
         import _curses
     except ImportError:
diff --git a/lib_pypy/syslog.py b/lib_pypy/syslog.py
--- a/lib_pypy/syslog.py
+++ b/lib_pypy/syslog.py
@@ -1,3 +1,4 @@
+# this cffi version was rewritten based on the
 # ctypes implementation: Victor Stinner, 2008-05-08
 """
 This module provides an interface to the Unix syslog library routines.
@@ -9,34 +10,84 @@
 if sys.platform == 'win32':
     raise ImportError("No syslog on Windows")
 
-# load the platform-specific cache made by running syslog.ctc.py
-from ctypes_config_cache._syslog_cache import *
-
-from ctypes_support import standard_c_lib as libc
-from ctypes import c_int, c_char_p
+from cffi import FFI
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
 
+ffi = FFI()
 
-# Real prototype is:
-# void syslog(int priority, const char *format, ...);
-# But we also need format ("%s") and one format argument (message)
-_syslog = libc.syslog
-_syslog.argtypes = (c_int, c_char_p, c_char_p)
-_syslog.restype = None
+ffi.cdef("""
+/* mandatory constants */
+#define LOG_EMERG ...
+#define LOG_ALERT ...
+#define LOG_CRIT ...
+#define LOG_ERR ...
+#define LOG_WARNING ...
+#define LOG_NOTICE ...
+#define LOG_INFO ...
+#define LOG_DEBUG ...
 
-_openlog = libc.openlog
-_openlog.argtypes = (c_char_p, c_int, c_int)
-_openlog.restype = None
+#define LOG_PID ...
+#define LOG_CONS ...
+#define LOG_NDELAY ...
 
-_closelog = libc.closelog
-_closelog.argtypes = None
-_closelog.restype = None
+#define LOG_KERN ...
+#define LOG_USER ...
+#define LOG_MAIL ...
+#define LOG_DAEMON ...
+#define LOG_AUTH ...
+#define LOG_LPR ...
+#define LOG_LOCAL0 ...
+#define LOG_LOCAL1 ...
+#define LOG_LOCAL2 ...
+#define LOG_LOCAL3 ...
+#define LOG_LOCAL4 ...
+#define LOG_LOCAL5 ...
+#define LOG_LOCAL6 ...
+#define LOG_LOCAL7 ...
 
-_setlogmask = libc.setlogmask
-_setlogmask.argtypes = (c_int,)
-_setlogmask.restype = c_int
+/* optional constants, gets defined to -919919 if missing */
+#define LOG_NOWAIT ...
+#define LOG_PERROR ...
+
+/* aliased constants, gets defined as some other constant if missing */
+#define LOG_SYSLOG ...
+#define LOG_CRON ...
+#define LOG_UUCP ...
+#define LOG_NEWS ...
+
+/* functions */
+void openlog(const char *ident, int option, int facility);
+void syslog(int priority, const char *format, const char *string);
+// NB. the signature of syslog() is specialized to the only case we use
+void closelog(void);
+int setlogmask(int mask);
+""")
+
+lib = ffi.verify("""
+#include <syslog.h>
+
+#ifndef LOG_NOWAIT
+#define LOG_NOWAIT -919919
+#endif
+#ifndef LOG_PERROR
+#define LOG_PERROR -919919
+#endif
+#ifndef LOG_SYSLOG
+#define LOG_SYSLOG LOG_DAEMON
+#endif
+#ifndef LOG_CRON
+#define LOG_CRON LOG_DAEMON
+#endif
+#ifndef LOG_UUCP
+#define LOG_UUCP LOG_MAIL
+#endif
+#ifndef LOG_NEWS
+#define LOG_NEWS LOG_MAIL
+#endif
+""")
+
 
 _S_log_open = False
 _S_ident_o = None
@@ -52,17 +103,19 @@
     return None
 
 @builtinify
-def openlog(ident=None, logoption=0, facility=LOG_USER):
+def openlog(ident=None, logoption=0, facility=lib.LOG_USER):
     global _S_ident_o, _S_log_open
     if ident is None:
         ident = _get_argv()
-    if ident is not None:
+    if ident is None:
+        _S_ident_o = ffi.NULL
+    else:
         if not isinstance(ident, str):
             msg = "openlog() argument 1 must be a str, not {!r}"
             raise TypeError(msg.format(type(ident).__name__))
         ident = ident.encode(sys.getdefaultencoding())
-    _S_ident_o = c_char_p(ident)    # keepalive
-    _openlog(_S_ident_o, logoption, facility)
+        _S_ident_o = ffi.new("char[]", ident)  # keepalive
+    lib.openlog(_S_ident_o, logoption, facility)
     _S_log_open = True
 
 @builtinify
@@ -78,19 +131,19 @@
         raise TypeError("syslog() message must be a str, not {!r}".format(
                 type(message).__name__))
     message = message.encode(sys.getdefaultencoding())
-    _syslog(priority, b"%s", message)
+    lib.syslog(priority, b"%s", message)
 
 @builtinify
 def closelog():
     global _S_log_open, S_ident_o
     if _S_log_open:
-        _closelog()
+        lib.closelog()
         _S_log_open = False
         _S_ident_o = None
 
 @builtinify
 def setlogmask(mask):
-    return _setlogmask(mask)
+    return lib.setlogmask(mask)
 
 @builtinify
 def LOG_MASK(pri):
@@ -100,8 +153,15 @@
 def LOG_UPTO(pri):
     return (1 << (pri + 1)) - 1
 
-__all__ = ALL_CONSTANTS + (
+__all__ = []
+
+for name in sorted(lib.__dict__):
+    if name.startswith('LOG_'):
+        value = getattr(lib, name)
+        if value != -919919:
+            globals()[name] = value
+            __all__.append(name)
+
+__all__ = tuple(__all__) + (
     'openlog', 'syslog', 'closelog', 'setlogmask',
     'LOG_MASK', 'LOG_UPTO')
-
-del ALL_CONSTANTS
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -147,7 +147,7 @@
                    requires=module_dependencies.get(modname, []),
                    suggests=module_suggests.get(modname, []),
                    negation=modname not in essential_modules,
-                   validator=get_module_validator(modname))
+                   ) #validator=get_module_validator(modname))
         for modname in all_modules]),
 
     BoolOption("allworkingmodules", "use as many working modules as possible",
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -61,3 +61,7 @@
 
 .. branch: faster-str-of-bigint
 Improve performance of str(long).
+
+.. branch: ndarray-view
+Add view to ndarray and zeroD arrays, not on dtype scalars yet
+
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -41,8 +41,8 @@
     def save_field(self, field_builder):
         field = field_builder.build()
         if self.numeric_field:
-            from pypy.objspace.std.strutil import ParseStringError
-            from pypy.objspace.std.strutil import string_to_float
+            from rpython.rlib.rstring import ParseStringError
+            from rpython.rlib.rfloat import string_to_float
             self.numeric_field = False
             try:
                 ff = string_to_float(field)
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -293,9 +293,9 @@
     _checkReadable = interp2app(check_readable_w),
     _checkWritable = interp2app(check_writable_w),
     _checkSeekable = interp2app(check_seekable_w),
+    _checkClosed = interp2app(W_IOBase.check_closed_w),
     closed = GetSetProperty(W_IOBase.closed_get_w,
                             doc="True if the file is closed"),
-    _checkClosed = interp2app(W_IOBase.check_closed_w),
     __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, cls=W_IOBase),
     __weakref__ = make_weakref_descr(W_IOBase),
 
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -71,6 +71,7 @@
         'complex_': 'interp_boxes.W_Complex128Box',
         'complex128': 'interp_boxes.W_Complex128Box',
         'complex64': 'interp_boxes.W_Complex64Box',
+        'cfloat': 'interp_boxes.W_Complex64Box',
     }
     if ENABLED_LONG_DOUBLE:
         long_double_dtypes = [
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -75,6 +75,12 @@
         else:
             return None
 
+    def get_view(self, orig_array, dtype, new_shape):
+        strides, backstrides = support.calc_strides(new_shape, dtype,
+                                                    self.order)
+        return SliceArray(self.start, strides, backstrides, new_shape,
+                          self, orig_array, dtype=dtype)
+
     def get_real(self, orig_array):
         strides = self.get_strides()
         backstrides = self.get_backstrides()
diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -63,6 +63,11 @@
     def transpose(self, _):
         return self
 
+    def get_view(self, orig_array, dtype, new_shape):
+        scalar = Scalar(dtype)
+        scalar.value = self.value.convert_to(dtype)
+        return scalar
+
     def get_real(self, orig_array):
         if self.dtype.is_complex_type():
             scalar = Scalar(self.dtype.float_type)
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -51,6 +51,7 @@
     w_IndexError = W_TypeObject("IndexError")
     w_OverflowError = W_TypeObject("OverflowError")
     w_NotImplementedError = W_TypeObject("NotImplementedError")
+    w_AttributeError = W_TypeObject("AttributeError")
     w_None = None
 
     w_bool = W_TypeObject("bool")
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -240,6 +240,16 @@
         v = self.convert_to(self.get_dtype(space))
         return self.get_dtype(space).itemtype.round(v, decimals)
 
+    def descr_view(self, space, w_dtype):
+        from pypy.module.micronumpy.interp_dtype import W_Dtype
+        dtype = space.interp_w(W_Dtype,
+            space.call_function(space.gettypefor(W_Dtype), w_dtype))
+        if dtype.get_size() != self.get_dtype(space).get_size():
+            raise OperationError(space.w_ValueError, space.wrap(
+                "new type not compatible with array."))
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "view not implelemnted yet"))
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -507,6 +517,7 @@
     all = interp2app(W_GenericBox.descr_all),
     ravel = interp2app(W_GenericBox.descr_ravel),
     round = interp2app(W_GenericBox.descr_round),
+    view = interp2app(W_GenericBox.descr_view),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -618,8 +618,40 @@
             "trace not implemented yet"))
 
     def descr_view(self, space, w_dtype=None, w_type=None) :
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "view not implemented yet"))
+        if w_type is not None:
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "view(... type=<class>) not implemented yet"))
+        if w_dtype:
+            dtype = space.interp_w(interp_dtype.W_Dtype,
+                space.call_function(space.gettypefor(interp_dtype.W_Dtype),
+                                                                   w_dtype))
+        else:
+            dtype = self.get_dtype()
+        old_itemsize = self.get_dtype().get_size()
+        new_itemsize = dtype.get_size()
+        impl = self.implementation
+        new_shape = self.get_shape()
+        dims = len(new_shape)
+        if dims == 0:
+            # Cannot resize scalars
+            if old_itemsize != new_itemsize:
+                raise OperationError(space.w_ValueError, space.wrap(
+                    "new type not compatible with array shape"))
+        else:
+            if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]:
+                # Column-major, resize first dimension
+                if new_shape[0] * old_itemsize % new_itemsize != 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                new_shape[0] = new_shape[0] * old_itemsize / new_itemsize
+            else:
+                # Row-major, resize last dimension
+                if new_shape[-1] * old_itemsize % new_itemsize != 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                new_shape[-1] = new_shape[-1] * old_itemsize / new_itemsize
+        return W_NDimArray(impl.get_view(self, dtype, new_shape))
+
 
     # --------------------- operations ----------------------------
 
@@ -996,6 +1028,7 @@
     round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
+    view = interp2app(W_NDimArray.descr_view),
 
     ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented
     __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface),
@@ -1028,15 +1061,21 @@
                                   order)
 
     dtype = interp_dtype.decode_w_dtype(space, w_dtype)
-    if isinstance(w_object, W_NDimArray):
-        if (not space.is_none(w_dtype) and
-            w_object.get_dtype() is not dtype):
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                                  "copying over different dtypes unsupported"))
+    if isinstance(w_object, W_NDimArray) and \
+        (space.is_none(w_dtype) or w_object.get_dtype() is dtype):
+        shape = w_object.get_shape()
         if copy:
-            return w_object.descr_copy(space)
-        return w_object
-
+            w_ret = w_object.descr_copy(space)
+        else:
+            if ndmin<= len(shape):
+                return w_object
+            new_impl = w_object.implementation.set_shape(space, w_object, shape)
+            w_ret = W_NDimArray(new_impl)
+        if ndmin > len(shape):
+            shape = [1] * (ndmin - len(shape)) + shape
+            w_ret.implementation = w_ret.implementation.set_shape(space,
+                                            w_ret, shape)
+        return w_ret
     shape, elems_w = find_shape_and_elems(space, w_object, dtype)
     if dtype is None or (
                  dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1):
diff --git a/pypy/module/micronumpy/interp_support.py b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.micronumpy import interp_dtype, loop
-from pypy.objspace.std.strutil import strip_spaces
+from rpython.rlib.rstring import strip_spaces
 from rpython.rlib.rarithmetic import maxint
 from pypy.module.micronumpy.base import W_NDimArray
 
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -584,6 +584,7 @@
         import numpypy as numpy
 
         assert numpy.complex_ is numpy.complex128
+        assert numpy.cfloat is numpy.complex64
         assert numpy.complex64.__mro__ == (numpy.complex64,
             numpy.complexfloating, numpy.inexact, numpy.number, numpy.generic,
             object)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -276,6 +276,24 @@
         arr = array([1], ndmin=3)
         assert arr.shape == (1, 1, 1)
 
+    def test_array_copy(self):
+        from numpypy import array
+        a = array(range(12)).reshape(3,4)
+        b = array(a, ndmin=4)
+        assert b.shape == (1, 1, 3, 4)
+        b = array(a, copy=False)
+        b[0, 0] = 100
+        assert a[0, 0] == 100
+        b = array(a, copy=True, ndmin=2)
+        b[0, 0] = 0
+        assert a[0, 0] == 100
+        b = array(a, dtype=float)
+        assert (b[0] == [100, 1, 2, 3]).all()
+        assert b.dtype.kind == 'f'
+        b = array(a, copy=False, ndmin=4)
+        b[0,0,0,0] = 0
+        assert a[0, 0] == 0
+
     def test_type(self):
         from numpypy import array
         ar = array(range(5))
@@ -325,13 +343,16 @@
         assert a[1] == 1.0
 
     def test_ones(self):
-        from numpypy import ones
+        from numpypy import ones, dtype
         a = ones(3)
         assert len(a) == 3
         assert a[0] == 1
         raises(IndexError, "a[3]")
         a[2] = 4
         assert a[2] == 4
+        b = ones(3, complex)
+        assert b[0] == 1+0j
+        assert b.dtype is dtype(complex)
 
     def test_copy(self):
         from numpypy import arange, array
@@ -1390,6 +1411,44 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
+    def test_ndarray_view(self):
+        from numpypy import array, int8, int16, dtype
+        x = array([(1, 2)], dtype=[('a', int8), ('b', int8)])
+        y = x.view(dtype=int16)
+        print y,y.shape
+        assert y[0] == 513
+        assert y.dtype == dtype('int16')
+        y[0] = 670
+        assert x['a'] == -98
+        assert x['b'] == 2
+        f = array([1000, -1234], dtype='i4')
+        nnp = self.non_native_prefix
+        d = f.view(dtype=nnp + 'i4')
+        assert (d == [-402456576,  788267007]).all()
+        x = array(range(15), dtype='i2').reshape(3,5)
+        exc = raises(ValueError, x.view, dtype='i4')
+        assert exc.value[0] == "new type not compatible with array."
+        assert x.view('int8').shape == (3, 10)
+        x = array(range(15), dtype='int16').reshape(3,5).T
+        assert x.view('int8').shape == (10, 3)
+
+    def test_ndarray_view_empty(self):
+        from numpypy import array, int8, int16, dtype
+        x = array([], dtype=[('a', int8), ('b', int8)])
+        y = x.view(dtype=int16)
+
+    def test_scalar_view(self):
+        from numpypy import int64, array
+        a = array(0, dtype='int32')
+        b = a.view(dtype='float32')
+        assert b.shape == ()
+        assert b == 0
+        s = int64(12)
+        exc = raises(ValueError, s.view, 'int8')
+        assert exc.value[0] == "new type not compatible with array."
+        skip('not implemented yet')
+        assert s.view('double') < 7e-323
+
     def test_tolist_scalar(self):
         from numpypy import int32, bool_
         x = int32(23)
@@ -2176,6 +2235,10 @@
         d.fill(100)
         assert d == 100
 
+        e = array(10, dtype=complex)
+        e.fill(1.5-3j)
+        assert e == 1.5-3j
+
     def test_array_indexing_bool(self):
         from numpypy import arange
         a = arange(10)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1100,6 +1100,13 @@
         op = '+' if imag >= 0 else ''
         return ''.join(['(', real_str, op, imag_str, ')'])
 
+    def fill(self, storage, width, box, start, stop, offset):
+        real, imag = self.unbox(box)
+        for i in xrange(start, stop, width):
+            raw_storage_setitem(storage, i+offset, real)
+            raw_storage_setitem(storage,
+                    i+offset+rffi.sizeof(self.T), imag)
+
     @staticmethod
     def for_computation(v):
         return float(v[0]), float(v[1])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -80,39 +80,11 @@
             i23 = strgetitem(p10, i19)
             p25 = newstr(1)
             strsetitem(p25, 0, i23)
-            p28 = call(ConstClass(strip_spaces), p25, descr=<Callr . r EF=4>)
+            p93 = call(ConstClass(fromstr), p25, 16, ConstPtr(ptr70), descr=<Callr . rir EF=3>)
             guard_no_exception(descr=...)
-            i29 = strlen(p28)
-            i30 = int_is_true(i29)
-            guard_true(i30, descr=...)
-            i32 = int_sub(i29, 1)
-            i33 = strgetitem(p28, i32)
-            i35 = int_eq(i33, 108)
-            guard_false(i35, descr=...)
-            i37 = int_eq(i33, 76)
-            guard_false(i37, descr=...)
-            i39 = strgetitem(p28, 0)
-            i41 = int_eq(i39, 45)
-            guard_false(i41, descr=...)
-            i43 = int_eq(i39, 43)
-            guard_false(i43, descr=...)
-            i43 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr42), descr=<Calli 1 rr EF=0>)
-            guard_false(i43, descr=...)
-            i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<Calli 1 rr EF=0>)
-            guard_false(i46, descr=...)
-            p51 = new_with_vtable(...)
-            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            setfield_gc(p51, _, descr=...)
-            p55 = call(ConstClass(parse_digit_string), p51, descr=<Callr . r EF=4>)
+            i94 = call(ConstClass(rbigint.toint), p93, descr=<Calli . r EF=3>)
             guard_no_exception(descr=...)
-            i57 = call(ConstClass(rbigint.toint), p55, descr=<Calli . r EF=3>)
-            guard_no_exception(descr=...)
-            i58 = int_add_ovf(i6, i57)
+            i95 = int_add_ovf(i6, i94)
             guard_no_overflow(descr=...)
             --TICK--
             jump(..., descr=...)
diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/test_lib_pypy/test_greenlet.py
--- a/pypy/module/test_lib_pypy/test_greenlet.py
+++ b/pypy/module/test_lib_pypy/test_greenlet.py
@@ -319,3 +319,25 @@
         g = G(lambda: 42)
         x = g.switch()
         assert x == 42
+
+    def test_kwargs_to_f(self):
+        import greenlet
+        seen = []
+        def f(*args, **kwds):
+            seen.append([args, kwds])
+        g = greenlet.greenlet(f)
+        g.switch(1, 2, x=3, y=4)
+        assert seen == [[(1, 2), {'x': 3, 'y': 4}]]
+
+    def test_kwargs_to_switch(self):
+        import greenlet
+        main = greenlet.getcurrent()
+        assert main.switch() == ()
+        assert main.switch(5) == 5
+        assert main.switch(5, 6) == (5, 6)
+        #
+        assert main.switch(x=5) == {'x': 5}
+        assert main.switch(x=5, y=6) == {'x': 5, 'y': 6}
+        assert main.switch(3, x=5) == ((3,), {'x': 5})
+        assert main.switch(3, x=5, y=6) == ((3,), {'x': 5, 'y': 6})
+        assert main.switch(2, 3, x=6) == ((2, 3), {'x': 6})
diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -4,6 +4,22 @@
 except ImportError:
     py.test.skip("No grp module on this platform")
 
+def test_basic():
+    g = grp.getgrnam("root")
+    assert g.gr_gid == 0
+    assert g.gr_mem == ['root']
+    assert g.gr_name == 'root'
+    assert isinstance(g.gr_passwd, str)    # usually just 'x', don't hope :-)
+
 def test_extra():
     py.test.raises(TypeError, grp.getgrnam, False)
     py.test.raises(TypeError, grp.getgrnam, None)
+
+def test_struct_group():
+    g = grp.struct_group((10, 20, 30, 40))
+    assert len(g) == 4
+    assert list(g) == [10, 20, 30, 40]
+    assert g.gr_name == 10
+    assert g.gr_passwd == 20
+    assert g.gr_gid == 30
+    assert g.gr_mem == 40
diff --git a/pypy/module/test_lib_pypy/test_syslog.py b/pypy/module/test_lib_pypy/test_syslog.py
--- a/pypy/module/test_lib_pypy/test_syslog.py
+++ b/pypy/module/test_lib_pypy/test_syslog.py
@@ -6,9 +6,5 @@
 
 # XXX very minimal test
 
-from lib_pypy.ctypes_config_cache import rebuild
-rebuild.rebuild_one('syslog.ctc.py')
-
-
 def test_syslog():
     assert hasattr(syslog, 'LOG_ALERT')
diff --git a/pypy/objspace/std/complextype.py b/pypy/objspace/std/complextype.py
--- a/pypy/objspace/std/complextype.py
+++ b/pypy/objspace/std/complextype.py
@@ -2,11 +2,12 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std.register_all import register_all
-from pypy.objspace.std.strutil import string_to_float, ParseStringError
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef
 from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod
 from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
+from rpython.rlib.rfloat import string_to_float
+from rpython.rlib.rstring import ParseStringError
 
 # ERRORCODES
 
diff --git a/pypy/objspace/std/floattype.py b/pypy/objspace/std/floattype.py
--- a/pypy/objspace/std/floattype.py
+++ b/pypy/objspace/std/floattype.py
@@ -8,10 +8,9 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
-from pypy.objspace.std.strutil import ParseStringError
-from pypy.objspace.std.strutil import string_to_float
 from pypy.objspace.std.model import W_Object
 from rpython.rlib.rbigint import rbigint
+from rpython.rlib.rstring import ParseStringError
 
 
 float_as_integer_ratio = SMM("as_integer_ratio", 1)
@@ -41,7 +40,7 @@
           space.isinstance_w(w_value, space.w_bytearray)):
         strvalue = space.bufferstr_w(w_value)
         try:
-            value = string_to_float(strvalue.decode('latin-1'))
+            value = rfloat.string_to_float(strvalue.decode('latin-1'))
         except ParseStringError, e:
             raise OperationError(space.w_ValueError,
                                  space.wrap(e.msg))
@@ -49,7 +48,7 @@
         from unicodeobject import unicode_to_decimal_w
         strvalue = unicode_to_decimal_w(space, w_value)
         try:
-            value = string_to_float(strvalue)
+            value = rfloat.string_to_float(strvalue)
         except ParseStringError, e:
             raise OperationError(space.w_ValueError,
                                  space.wrap(e.msg))
diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py
--- a/pypy/objspace/std/inttype.py
+++ b/pypy/objspace/std/inttype.py
@@ -5,13 +5,12 @@
 from pypy.interpreter.buffer import Buffer
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
-from pypy.objspace.std.strutil import (string_to_int, string_to_bigint,
-                                       ParseStringError,
-                                       ParseStringOverflowError)
 from pypy.objspace.std.model import W_Object
-from rpython.rlib.rarithmetic import r_uint
+from rpython.rlib.rarithmetic import r_uint, string_to_int
 from rpython.rlib.objectmodel import instantiate
 from rpython.rlib.rbigint import rbigint
+from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
+from rpython.rlib import jit
 
 # ____________________________________________________________
 
@@ -70,6 +69,7 @@
 
 # ____________________________________________________________
 
+## @jit.elidable
 ## def string_to_int_or_long(space, string, base=10):
 ##     w_longval = None
 ##     value = 0
@@ -82,15 +82,14 @@
 ##         w_longval = retry_to_w_long(space, e.parser)
 ##     return value, w_longval
 
-## def retry_to_w_long(space, parser, base=0):
+## def retry_to_w_long(space, parser):
 ##     parser.rewind()
 ##     try:
-##         bigint = string_to_bigint(None, base=base, parser=parser)
+##         bigint = rbigint._from_numberstring_parser(parser)
 ##     except ParseStringError, e:
 ##         raise OperationError(space.w_ValueError,
 ##                              space.wrap(e.msg))
-##     from pypy.objspace.std.longobject import newlong
-##     return newlong(space, bigint)
+##     return space.newlong_from_rbigint(bigint)
 
 ## @unwrap_spec(w_x = WrappedDefault(0))
 ## def descr__new__(space, w_inttype, w_x, w_base=None):
diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py
--- a/pypy/objspace/std/longtype.py
+++ b/pypy/objspace/std/longtype.py
@@ -3,8 +3,8 @@
 from pypy.interpreter.gateway import (
     WrappedDefault, applevel, interp2app, interpindirect2app, unwrap_spec)
 from pypy.objspace.std.model import W_Object
-from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
-from pypy.objspace.std.strutil import string_to_bigint, ParseStringError
+from pypy.objspace.std.stdtypedef import StdTypeDef
+from rpython.rlib.rstring import ParseStringError
 from rpython.rlib.rbigint import rbigint, InvalidEndiannessError, InvalidSignednessError
 
 def descr_conjugate(space, w_int):
@@ -71,7 +71,7 @@
 
 def string_to_w_long(space, w_longtype, s, base=10):
     try:
-        bigint = string_to_bigint(s, base)
+        bigint = rbigint.fromstr(s, base)
     except ParseStringError, e:
         raise OperationError(space.w_ValueError,
                              space.wrap(e.msg))
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
deleted file mode 100644
--- a/pypy/objspace/std/strutil.py
+++ /dev/null
@@ -1,212 +0,0 @@
-"""
-Pure Python implementation of string utilities.
-"""
-
-from rpython.tool.sourcetools import with_unicode_literals
-from rpython.rlib.objectmodel import enforceargs
-from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rfloat import rstring_to_float, INFINITY, NAN
-from rpython.rlib.rbigint import rbigint, parse_digit_string
-from pypy.interpreter.error import OperationError
-import math
-
-# XXX factor more functions out of stringobject.py.
-# This module is independent from PyPy.
-
- at enforceargs(unicode)
- at with_unicode_literals
-def strip_spaces(s):
-    # XXX this is not locale-dependent
-    p = 0
-    q = len(s)
-    while p < q and s[p] in ' \f\n\r\t\v':
-        p += 1
-    while p < q and s[q-1] in ' \f\n\r\t\v':
-        q -= 1
-    assert q >= p     # annotator hint, don't remove
-    return s[p:q]
-
-class ParseStringError(Exception):
-    @enforceargs(None, unicode)
-    def __init__(self, msg):
-        self.msg = msg
-
-class ParseStringOverflowError(Exception):
-    def __init__(self, parser):
-        self.parser = parser
-
-# iterator-like class
-class NumberStringParser:
-
-    def error(self):
-        raise ParseStringError(u"invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.original_base, self.literal))
-
-    @enforceargs(None, unicode, unicode, int, unicode)
-    @with_unicode_literals
-    def __init__(self, s, literal, base, fname):
-        self.literal = literal
-        self.fname = fname
-        sign = 1
-        if s.startswith('-'):
-            sign = -1
-            s = strip_spaces(s[1:])
-        elif s.startswith('+'):
-            s = strip_spaces(s[1:])
-        self.sign = sign
-        self.original_base = base
-
-        if base == 0:
-            if s.startswith('0x') or s.startswith('0X'):
-                base = 16
-            elif s.startswith('0b') or s.startswith('0B'):
-                base = 2
-            elif s.startswith('0'): # also covers the '0o' case
-                base = 8
-            else:
-                base = 10
-        elif base < 2 or base > 36:
-            raise ParseStringError, u"%s() base must be >= 2 and <= 36" % (fname,)
-        self.base = base
-
-        if base == 16 and (s.startswith('0x') or s.startswith('0X')):
-            s = s[2:]
-        if base == 8 and (s.startswith('0o') or s.startswith('0O')):
-            s = s[2:]
-        if base == 2 and (s.startswith('0b') or s.startswith('0B')):
-            s = s[2:]
-        if not s:
-            self.error()
-        self.s = s
-        self.n = len(s)
-        self.i = 0
-
-    def rewind(self):
-        self.i = 0
-
-    @with_unicode_literals
-    def next_digit(self): # -1 => exhausted
-        if self.i < self.n:
-            c = self.s[self.i]
-            digit = ord(c)
-            if '0' <= c <= '9':
-                digit -= ord('0')
-            elif 'A' <= c <= 'Z':
-                digit = (digit - ord('A')) + 10
-            elif 'a' <= c <= 'z':
-                digit = (digit - ord('a')) + 10
-            else:
-                self.error()
-            if digit >= self.base:
-                self.error()
-            self.i += 1
-            return digit
-        else:
-            return -1
-
- at enforceargs(unicode, None)
-def string_to_int(s, base=10):
-    """Utility to converts a string to an integer.
-    If base is 0, the proper base is guessed based on the leading
-    characters of 's'.  Raises ParseStringError in case of error.
-    Raises ParseStringOverflowError in case the result does not fit.
-    """
-    s = literal = strip_spaces(s)
-    p = NumberStringParser(s, literal, base, u'int')
-    base = p.base
-    result = 0
-    while True:
-        digit = p.next_digit()
-        if digit == -1:
-            return result
-
-        if p.sign == -1:
-            digit = -digit
-
-        try:
-            result = ovfcheck(result * base)
-            result = ovfcheck(result + digit)
-        except OverflowError:
-            raise ParseStringOverflowError(p)
-
- at enforceargs(unicode, None, None)
-def string_to_bigint(s, base=10, parser=None):
-    """As string_to_int() but returns an rbigint."""
-    if parser is None:
-        s = literal = strip_spaces(s)
-        p = NumberStringParser(s, literal, base, u'int')
-    else:
-        p = parser
-    return parse_digit_string(p)
-
-# Tim's comment:
-# 57 bits are more than needed in any case.
-# to allow for some rounding, we take one
-# digit more.
-
-# In the PyPy case, we can compute everything at compile time:
-# XXX move this stuff to some central place, it is now also
-# in _float_formatting.
-
-def calc_mantissa_bits():
-    bits = 1 # I know it is almost always 53, but let it compute...
-    while 1:
-        pattern = (1L << bits) - 1
-        comp = long(float(pattern))
-        if comp != pattern:
-            return bits - 1
-        bits += 1
-
-MANTISSA_BITS = calc_mantissa_bits()
-del calc_mantissa_bits
-MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1
-
- at enforceargs(unicode)
-def string_to_float(s):
-    """
-    Conversion of string to float.
-    This version tries to only raise on invalid literals.
-    Overflows should be converted to infinity whenever possible.
-
-    Expects an unwrapped string and return an unwrapped float.
-    """
-
-    s = strip_spaces(s)
-
-    if not s:
-        raise ParseStringError(u"empty string for float()")
-
-
-    try:
-        ascii_s = s.encode('ascii')
-    except UnicodeEncodeError:
-        # if s is not ASCII, it certainly is not a float literal (because the
-        # unicode-decimal to ascii-decimal conversion already happened
-        # earlier). We just set ascii_s to something which will fail when
-        # passed to rstring_to_float, to keep the code as similar as possible
-        # to the one we have on default.
-        #
-        # Note that CPython does something different and it encodes the string
-        # to UTF-8 before trying to parse it. We cannot since .encode('utf-8')
-        # is not RPython. However, it doesn't change anything since the UTF-8
-        # encoded string would make rstring_to_float to fail anyway.
-        ascii_s = "not a float"
-    else:
-        low = ascii_s.lower()
-        if low == "-inf" or low == "-infinity":
-            return -INFINITY
-        elif low == "inf" or low == "+inf":
-            return INFINITY
-        elif low == "infinity" or low == "+infinity":
-            return INFINITY
-        elif low == "nan" or low == "+nan":
-            return NAN
-        elif low == "-nan":
-            return -NAN
-
-    try:
-        return rstring_to_float(ascii_s)
-    except ValueError:
-        # note that we still put the original unicode string in the error
-        # message, not ascii_s
-        raise ParseStringError(u"invalid literal for float(): '%s'" % s)
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_strutil.py
+++ /dev/null
@@ -1,193 +0,0 @@
-# in default string_to_int accepts str, in py3k it accepts unicode. We use
-# __future__.unicode_literals here to minimize the diff
-from __future__ import unicode_literals
-
-import py, random
-from pypy.objspace.std.strutil import *
-from pypy.interpreter.error import OperationError
-
-
-class TestStrUtil:
-
-    def test_string_to_int(self):
-        space = self.space
-        cases = [('0', 0),
-                 ('1', 1),
-                 ('9', 9),
-                 ('10', 10),
-                 ('09', 9),
-                 ('0000101', 101),    # not octal unless base 0 or 8
-                 ('5123', 5123),
-                 (' 0', 0),
-                 ('0  ', 0),
-                 (' \t \n   32313  \f  \v   \r  \n\r    ', 32313),
-                 ('+12', 12),
-                 ('-5', -5),
-                 ('- 5', -5),
-                 ('+ 5', 5),
-                 ('  -123456789 ', -123456789),
-                 ]
-        for s, expected in cases:
-            assert string_to_int(s) == expected
-            assert string_to_bigint(s).tolong() == expected
-
-    def test_string_to_int_base(self):
-        space = self.space        
-        cases = [('111', 2, 7),
-                 ('010', 2, 2),
-                 ('102', 3, 11),
-                 ('103', 4, 19),
-                 ('107', 8, 71),
-                 ('109', 10, 109),
-                 ('10A', 11, 131),
-                 ('10a', 11, 131),
-                 ('10f', 16, 271),
-                 ('10F', 16, 271),
-                 ('0x10f', 16, 271),
-                 ('0x10F', 16, 271),
-                 ('10z', 36, 1331),
-                 ('10Z', 36, 1331),
-                 ('12',   0, 12),
-                 ('015',  0, 13),
-                 ('0x10', 0, 16),
-                 ('0XE',  0, 14),
-                 ('0',    0, 0),
-                 ('0b11', 2, 3),
-                 ('0B10', 2, 2),
-                 ('0o77', 8, 63),
-                 ]
-        for s, base, expected in cases:
-            assert string_to_int(s, base) == expected
-            assert string_to_int('+'+s, base) == expected
-            assert string_to_int('-'+s, base) == -expected
-            assert string_to_int(s+'\n', base) == expected
-            assert string_to_int('  +'+s, base) == expected
-            assert string_to_int('-'+s+'  ', base) == -expected
-
-    def test_string_to_int_error(self):
-        space = self.space
-        cases = ['0x123',    # must use base 0 or 16
-                 ' 0X12 ',
-                 '0b01',
-                 '0o01',
-                 '',
-                 '++12',
-                 '+-12',
-                 '-+12',
-                 '--12',
-                 '12a6',
-                 '12A6',
-                 'f',
-                 'Z',
-                 '.',
-                 '@',
-                 ]
-        for s in cases:
-            raises(ParseStringError, string_to_int, s)
-            raises(ParseStringError, string_to_int, '  '+s)
-            raises(ParseStringError, string_to_int, s+'  ')
-            raises(ParseStringError, string_to_int, '+'+s)
-            raises(ParseStringError, string_to_int, '-'+s)
-        raises(ParseStringError, string_to_int, '0x', 16)
-        raises(ParseStringError, string_to_int, '-0x', 16)
-
-        exc = raises(ParseStringError, string_to_int, '')
-        assert exc.value.msg == "invalid literal for int() with base 10: ''"
-        exc = raises(ParseStringError, string_to_int, '', 0)
-        assert exc.value.msg == "invalid literal for int() with base 0: ''"
-
-    def test_string_to_int_overflow(self):
-        import sys
-        space = self.space
-        raises(ParseStringOverflowError, string_to_int,
-               unicode(sys.maxint*17))
-
-    def test_string_to_int_not_overflow(self):
-        import sys
-        for x in [-sys.maxint-1, sys.maxint]:
-            y = string_to_int(unicode(x))
-            assert y == x
-
-    def test_string_to_int_base_error(self):
-        space = self.space
-        cases = [('1', 1),
-                 ('1', 37),
-                 ('a', 0),
-                 ('9', 9),
-                 ('0x123', 7),
-                 ('145cdf', 15),
-                 ('12', 37),
-                 ('12', 98172),
-                 ('12', -1),
-                 ('12', -908),
-                 ('12.3', 10),
-                 ('12.3', 13),
-                 ('12.3', 16),
-                 ]
-        for s, base in cases:
-            raises(ParseStringError, string_to_int, s, base)
-            raises(ParseStringError, string_to_int, '  '+s, base)
-            raises(ParseStringError, string_to_int, s+'  ', base)
-            raises(ParseStringError, string_to_int, '+'+s, base)
-            raises(ParseStringError, string_to_int, '-'+s, base)
-
-    def test_string_to_bigint(self):
-        assert string_to_bigint('123').tolong() == 123
-        assert string_to_bigint('123  ').tolong() == 123
-        raises(ParseStringError, string_to_bigint, 'L')
-        raises(ParseStringError, string_to_bigint, 'L  ')
-        assert string_to_bigint('123', 4).tolong() == 27
-        assert string_to_bigint('123L', 30).tolong() == 27000 + 1800 + 90 + 21
-        assert string_to_bigint('123L', 22).tolong() == 10648 + 968 + 66 + 21
-        assert string_to_bigint('123', 21).tolong() == 441 + 42 + 3
-        assert string_to_bigint('1891234174197319').tolong() == 1891234174197319
-
-    def test_string_to_float(self):
-        assert string_to_float('0') == 0.0
-        assert string_to_float('1') == 1.0
-        assert string_to_float('-1.5') == -1.5
-        assert string_to_float('1.5E2') == 150.0
-        assert string_to_float('2.5E-1') == 0.25
-        assert string_to_float('1e1111111111111') == float('1e1111111111111')
-        assert string_to_float('1e-1111111111111') == float('1e-1111111111111')
-        assert string_to_float('-1e1111111111111') == float('-1e1111111111111')
-        assert string_to_float('-1e-1111111111111') == float('-1e-1111111111111')
-        assert string_to_float('1e111111111111111111111') == float('1e111111111111111111111')
-        assert string_to_float('1e-111111111111111111111') == float('1e-111111111111111111111')
-        assert string_to_float('-1e111111111111111111111') == float('-1e111111111111111111111')
-        assert string_to_float('-1e-111111111111111111111') == float('-1e-111111111111111111111')
-
-        valid_parts = [['', '  ', ' \f\n\r\t\v'],
-                       ['', '+', '-'],
-                       ['00', '90', '.5', '2.4', '3.', '0.07',
-                        '12.3489749871982471987198371293717398256187563298638726'
-                        '2187362820947193247129871083561249818451804287437824015'
-                        '013816418758104762348932657836583048761487632840726386'],
-                       ['', 'e0', 'E+1', 'E-01', 'E42'],
-                       ['', '  ', ' \f\n\r\t\v'],
-                       ]
-        invalid_parts = [['#'],
-                         ['++', '+-', '-+', '--'],
-                         ['', '1.2.3', '.', '5..6'],
-                         ['E+', 'E-', 'e', 'e++', 'E++2'],
-                         ['#'],
-                         ]
-        for part0 in valid_parts[0]:
-            for part1 in valid_parts[1]:
-                for part2 in valid_parts[2]:
-                    for part3 in valid_parts[3]:
-                        for part4 in valid_parts[4]:
-                            s = part0+part1+part2+part3+part4
-                            assert (abs(string_to_float(s) - float(s)) <=
-                                    1E-13 * abs(float(s)))
-
-        for j in range(len(invalid_parts)):
-            for invalid in invalid_parts[j]:
-                for i in range(20):
-                    parts = [random.choice(lst) for lst in valid_parts]
-                    parts[j] = invalid
-                    s = ''.join(parts)
-                    print repr(s)
-                    if s.strip(): # empty s raises OperationError directly
-                        py.test.raises(ParseStringError, string_to_float, s)
-        py.test.raises(ParseStringError, string_to_float, "")
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -69,6 +69,7 @@
     subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
     if not sys.platform == 'win32':
         subprocess.check_call([str(pypy_c), '-c', 'import _curses'])
+        subprocess.check_call([str(pypy_c), '-c', 'import syslog'])
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
         rename_pypy_c += '.exe'
     binaries = [(pypy_c, rename_pypy_c)]
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -3,35 +3,34 @@
 """
 
 import os
-from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
-                                            ConstFloat, BoxInt,
-                                            BoxFloat, INT, REF, FLOAT,
-                                            TargetToken)
-from rpython.jit.backend.x86.regloc import *
-from rpython.rtyper.lltypesystem import lltype, rffi, rstr
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rgc
 from rpython.jit.backend.llsupport import symbolic
+from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
+    unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
+from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc,
+     RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op)
+from rpython.jit.backend.x86 import rx86
+from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
+    IS_X86_64)
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
+from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
+    FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
+    ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
+    xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
+    X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG)
 from rpython.jit.codewriter import longlong
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
+    ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken)
 from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.backend.llsupport.descr import ArrayDescr
-from rpython.jit.backend.llsupport.descr import CallDescr
-from rpython.jit.backend.llsupport.descr import unpack_arraydescr
-from rpython.jit.backend.llsupport.descr import unpack_fielddescr
-from rpython.jit.backend.llsupport.descr import unpack_interiorfielddescr
-from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
-from rpython.jit.backend.llsupport.regalloc import FrameManager, BaseRegalloc,\
-     RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op
-from rpython.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE
-from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64
-from rpython.jit.backend.x86 import rx86
+from rpython.rlib import rgc
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
+
 
 class X86RegisterManager(RegisterManager):
-
     box_types = [INT, REF]
     all_regs = [ecx, eax, edx, ebx, esi, edi]
     no_lower_byte_regs = [esi, edi]
@@ -886,7 +885,6 @@
                 gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8))
         return gcmap
 
-
     def consider_setfield_gc(self, op):
         ofs, size, _ = unpack_fielddescr(op.getdescr())
         ofs_loc = imm(ofs)
@@ -950,7 +948,7 @@
     def consider_setarrayitem_gc(self, op):
         itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
         args = op.getarglist()
-        base_loc  = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
         if itemsize == 1:
             need_lower_byte = True
         else:
@@ -1311,7 +1309,6 @@
         #if jump_op is not None and jump_op.getdescr() is descr:
         #    self._compute_hint_frame_locations_from_descr(descr)
 
-
     def consider_keepalive(self, op):
         pass
 
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -668,3 +668,36 @@
     if T == lltype.Float:
         return longlong2float(rffi.cast(rffi.LONGLONG, res))
     return rffi.cast(T, res)
+
+
+# String parsing support
+# ---------------------------
+
+def string_to_int(s, base=10):
+    """Utility to converts a string to an integer.
+    If base is 0, the proper base is guessed based on the leading
+    characters of 's'.  Raises ParseStringError in case of error.
+    Raises ParseStringOverflowError in case the result does not fit.
+    """
+    from rpython.rlib.rstring import NumberStringParser, \
+        ParseStringOverflowError, \
+        ParseStringError, strip_spaces
+    s = literal = strip_spaces(s)
+    p = NumberStringParser(s, literal, base, 'int')
+    base = p.base
+    result = 0
+    while True:
+        digit = p.next_digit()
+        if digit == -1:
+            return result
+
+        if p.sign == -1:
+            digit = -digit
+
+        try:
+            result = ovfcheck(result * base)
+            result = ovfcheck(result + digit)
+        except OverflowError:
+            raise ParseStringOverflowError(p)
+
+
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -61,7 +61,7 @@
     KARATSUBA_CUTOFF = 19
 else:
     KARATSUBA_CUTOFF = 38
-    
+
 KARATSUBA_SQUARE_CUTOFF = 2 * KARATSUBA_CUTOFF
 
 # For exponentiation, use the binary left-to-right algorithm
@@ -85,38 +85,41 @@
 
 def _load_unsigned_digit(x):
     return rffi.cast(UNSIGNED_TYPE, x)
-        
+
 _load_unsigned_digit._always_inline_ = True
 
 NULLDIGIT = _store_digit(0)
-ONEDIGIT  = _store_digit(1)
+ONEDIGIT = _store_digit(1)
 
 def _check_digits(l):
     for x in l:
         assert type(x) is type(NULLDIGIT)
         assert UDIGIT_MASK(x) & MASK == UDIGIT_MASK(x)
-            
+
 class InvalidEndiannessError(Exception):
     pass
 
 class InvalidSignednessError(Exception):
     pass
 
+
 class Entry(extregistry.ExtRegistryEntry):
     _about_ = _check_digits
+
     def compute_result_annotation(self, s_list):
         from rpython.annotator import model as annmodel
         assert isinstance(s_list, annmodel.SomeList)
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
+
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
 
+
 class rbigint(object):
     """This is a reimplementation of longs using a list of digits."""
     _immutable_ = True
     _immutable_fields_ = ["_digits"]
-    
 
     def __init__(self, digits=[NULLDIGIT], sign=0, size=0):
         if not we_are_translated():
@@ -164,7 +167,7 @@
     def numdigits(self):
         return self.size
     numdigits._always_inline_ = True
-    
+
     @staticmethod
     @jit.elidable
     def fromint(intval):
@@ -180,15 +183,14 @@
             ival = r_uint(intval)
         else:
             return NULLRBIGINT
-        
+
         carry = ival >> SHIFT
         if carry:
             return rbigint([_store_digit(ival & MASK),
                 _store_digit(carry)], sign, 2)
         else:
             return rbigint([_store_digit(ival & MASK)], sign, 1)
-            
-        
+
     @staticmethod
     @jit.elidable
     def frombool(b):
@@ -246,12 +248,30 @@
     @staticmethod
     @jit.elidable
     def fromdecimalstr(s):
-        # This function is marked as pure, so you must not call it and
+        # This function is marked as elidable, so you must not call it and
         # then modify the result.
         return _decimalstr_to_bigint(s)
 
     @staticmethod
     @jit.elidable
+    def fromstr(s, base=0):
+        """As string_to_int(), but ignores an optional 'l' or 'L' suffix
+        and returns an rbigint."""
+        from rpython.rlib.rstring import NumberStringParser, \
+            strip_spaces
+        s = literal = strip_spaces(s)
+        if (s.endswith('l') or s.endswith('L')) and base < 22:
+            # in base 22 and above, 'L' is a valid digit!  try: long('L',22)
+            s = s[:-1]
+        parser = NumberStringParser(s, literal, base, 'long')
+        return rbigint._from_numberstring_parser(parser)
+
+    @staticmethod
+    def _from_numberstring_parser(parser):
+        return parse_digit_string(parser)
+
+    @staticmethod
+    @jit.elidable
     def frombytes(s, byteorder, signed):
         if byteorder not in ('big', 'little'):
             raise InvalidEndiannessError()
@@ -319,7 +339,7 @@
                 # Avoid bogus 0's
                 s = d ^ MASK if self.sign == -1 else d
                 while s:
-                    s >>=1
+                    s >>= 1
                     accumbits += 1
             else:
                 accumbits += SHIFT
@@ -340,7 +360,7 @@
 
             if self.sign == -1:
                 # Add a sign bit
-                accum |= (~_widen_digit(0)) << accumbits;
+                accum |= (~_widen_digit(0)) << accumbits
 
             result.append(chr(accum & 0xFF))
 
@@ -539,15 +559,15 @@
     def mul(self, b):
         asize = self.numdigits()
         bsize = b.numdigits()
-        
+
         a = self
-        
+
         if asize > bsize:
             a, b, asize, bsize = b, a, bsize, asize
 
         if a.sign == 0 or b.sign == 0:
             return NULLRBIGINT
-        
+
         if asize == 1:
             if a._digits[0] == NULLDIGIT:
                 return NULLRBIGINT
@@ -560,14 +580,14 @@
                     return rbigint([_store_digit(res & MASK), _store_digit(carry)], a.sign * b.sign, 2)
                 else:
                     return rbigint([_store_digit(res & MASK)], a.sign * b.sign, 1)
-                
-            result =  _x_mul(a, b, a.digit(0))
+
+            result = _x_mul(a, b, a.digit(0))
         elif USE_KARATSUBA:
             if a is b:
                 i = KARATSUBA_SQUARE_CUTOFF
             else:
                 i = KARATSUBA_CUTOFF
-                
+
             if asize <= i:
                 result = _x_mul(a, b)
                 """elif 2 * asize <= bsize:
@@ -593,13 +613,13 @@
                 return rbigint(self._digits[:self.size], 1, self.size)
             elif digit and digit & (digit - 1) == 0:
                 return self.rshift(ptwotable[digit])
-            
+
         div, mod = _divrem(self, other)
         if mod.sign * other.sign == -1:
             if div.sign == 0:
                 return ONENEGATIVERBIGINT
             div = div.sub(ONERBIGINT)
-            
+
         return div
 
     @jit.look_inside
@@ -610,7 +630,7 @@
     def mod(self, other):
         if self.sign == 0:
             return NULLRBIGINT
-        
+
         if other.sign != 0 and other.numdigits() == 1:
             digit = other.digit(0)
             if digit == 1:
@@ -633,7 +653,7 @@
                         size -= 1
                 else:
                     rem = self.digit(0) % digit
-                    
+
                 if rem == 0:
                     return NULLRBIGINT
                 mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1)
@@ -684,9 +704,9 @@
                     "cannot be negative when 3rd argument specified")
             # XXX failed to implement
             raise ValueError("bigint pow() too negative")
-        
+
         size_b = b.numdigits()
-        
+
         if c is not None:
             if c.sign == 0:
                 raise ValueError("pow() 3rd argument cannot be 0")
@@ -702,13 +722,13 @@
             #     return 0
             if c.numdigits() == 1 and c._digits[0] == ONEDIGIT:
                 return NULLRBIGINT
-   
+
             # if base < 0:
             #     base = base % modulus
             # Having the base positive just makes things easier.
             if a.sign < 0:
                 a = a.mod(c)
-            
+
         elif b.sign == 0:
             return ONERBIGINT
         elif a.sign == 0:
@@ -730,12 +750,12 @@
                     if a.sign == -1 and not digit % 2:
                         ret.sign = 1
                     return ret
-                
+
         # At this point a, b, and c are guaranteed non-negative UNLESS
         # c is NULL, in which case a may be negative. */
 
         z = rbigint([ONEDIGIT], 1, 1)
-        
+
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
         if size_b <= FIVEARY_CUTOFF:
@@ -751,7 +771,7 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 size_b -= 1
-                
+
         else:
             # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
             # This is only useful in the case where c != None.
@@ -784,7 +804,7 @@
                     # must get the next digit from 'b' in order to complete
                     if size_b == 0:
                         break # Done
-                        
+
                     size_b -= 1
                     assert size_b >= 0
                     bi = b.udigit(size_b)
@@ -798,7 +818,7 @@
                     z = _help_mult(z, table[index], c)
             #
             assert j == -5
-        
+
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
         return z
@@ -817,12 +837,12 @@
     def invert(self): #Implement ~x as -(x + 1)
         if self.sign == 0:
             return ONENEGATIVERBIGINT
-        
+
         ret = self.add(ONERBIGINT)
         ret.sign = -ret.sign
         return ret
-        
-    @jit.elidable    
+
+    @jit.elidable
     def lshift(self, int_other):
         if int_other < 0:
             raise ValueError("negative shift count")
@@ -831,14 +851,14 @@
 
         # wordshift, remshift = divmod(int_other, SHIFT)
         wordshift = int_other // SHIFT
-        remshift  = int_other - wordshift * SHIFT
+        remshift = int_other - wordshift * SHIFT
 
         if not remshift:
             # So we can avoid problems with eq, AND avoid the need for normalize.
             if self.sign == 0:
                 return self
             return rbigint([NULLDIGIT] * wordshift + self._digits, self.sign, self.size + wordshift)
-        
+
         oldsize = self.numdigits()
         newsize = oldsize + wordshift + 1
         z = rbigint([NULLDIGIT] * newsize, self.sign, newsize)
@@ -850,7 +870,7 @@
             accum >>= SHIFT
             wordshift += 1
             j += 1
-        
+
         newsize -= 1
         assert newsize >= 0
         z.setdigit(newsize, accum)
@@ -858,7 +878,7 @@
         z._normalize()
         return z
     lshift._always_inline_ = True # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def lqshift(self, int_other):
         " A quicker one with much less checks, int_other is valid and for the most part constant."
@@ -878,7 +898,7 @@
         z._normalize()
         return z
     lqshift._always_inline_ = True # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def rshift(self, int_other, dont_invert=False):
         if int_other < 0:
@@ -910,7 +930,7 @@
         z._normalize()
         return z
     rshift._always_inline_ = 'try' # It's so fast that it's always benefitial.
-    
+
     @jit.elidable
     def and_(self, other):
         return _bitwise(self, '&', other)
@@ -968,7 +988,7 @@
             self._digits = [NULLDIGIT]
 
     _normalize._always_inline_ = True
-    
+
     @jit.elidable
     def bit_length(self):
         i = self.numdigits()
@@ -1018,7 +1038,7 @@
     # is NULL.
     if c is not None:
         res = res.mod(c)
-        
+
     return res
 
 def digits_from_nonneg_long(l):
@@ -1103,7 +1123,7 @@
 
 def _x_sub(a, b):
     """ Subtract the absolute values of two integers. """
-    
+
     size_a = a.numdigits()
     size_b = b.numdigits()
     sign = 1
@@ -1124,7 +1144,7 @@
             sign = -1
             a, b = b, a
         size_a = size_b = i+1
-        
+
     z = rbigint([NULLDIGIT] * size_a, sign, size_a)
     borrow = UDIGIT_TYPE(0)
     i = _load_unsigned_digit(0)
@@ -1142,7 +1162,7 @@
         borrow >>= SHIFT
         #borrow &= 1
         i += 1
-        
+
     assert borrow == 0
     z._normalize()
     return z
@@ -1152,7 +1172,7 @@
 for x in range(SHIFT-1):
     ptwotable[r_longlong(2 << x)] = x+1
     ptwotable[r_longlong(-2 << x)] = x+1
-    
+
 def _x_mul(a, b, digit=0):
     """
     Grade school multiplication, ignoring the signs.
@@ -1201,14 +1221,14 @@
             i += 1
         z._normalize()
         return z
-    
+
     elif digit:
         if digit & (digit - 1) == 0:
             return b.lqshift(ptwotable[digit])
-        
+
         # Even if it's not power of two it can still be useful.
         return _muladd1(b, digit)
-        
+
     z = rbigint([NULLDIGIT] * (size_a + size_b), 1)
     # gradeschool long mult
     i = UDIGIT_TYPE(0)
@@ -1258,7 +1278,7 @@
     """
     asize = a.numdigits()
     bsize = b.numdigits()
-    
+
     # (ah*X+al)(bh*X+bl) = ah*bh*X*X + (ah*bl + al*bh)*X + al*bl
     # Let k = (ah+al)*(bh+bl) = ah*bl + al*bh  + ah*bh + al*bl
     # Then the original product is
@@ -1341,7 +1361,7 @@
         t2 = _x_add(bh, bl)
 
     t3 = t1.mul(t2)
-    assert t3.sign >=0
+    assert t3.sign >= 0
 
     # Add t3.  It's not obvious why we can't run out of room here.
     # See the (*) comment after this function.
@@ -1421,9 +1441,9 @@
     #bslice = rbigint([0] * asize, 1)
     # XXX we cannot pre-allocate, see comments below!
     # XXX prevent one list from being created.
-    bslice = rbigint(sign = 1)

From noreply at buildbot.pypy.org  Sat Jun 29 19:49:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 19:49:47 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: in-progress
Message-ID: <20130629174947.9F6911C1450@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65101:a163fe15803e
Date: 2013-06-29 19:49 +0200
http://bitbucket.org/pypy/pypy/changeset/a163fe15803e/

Log:	in-progress

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -9,16 +9,6 @@
 from rpython.rlib.debug import ll_assert
 
 
-class PrebuiltStmGCHeader(llmemory.Symbolic):
-    def __init__(self, typeid16):
-        self.typeid16 = typeid16
-    def annotation(self):
-        from rpython.annotator import model
-        return model.SomePtr(self.lltype())
-    def lltype(self):
-        return lltype.Ptr(StmGC.HDR)
-
-
 class StmGC(MovingGCBase):
     _alloc_flavor_ = "raw"
     inline_simple_malloc = True
@@ -38,7 +28,15 @@
 
     def init_gc_object_immortal(self, addr, typeid16, flags=0):
         assert flags == 0
-        return PrebuiltStmGCHeader(typeid16)
+        assert isinstance(typeid16, llgroup.GroupMemberOffset)
+        ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
+        prebuilt_hash = lltype.identityhash_nocache(ptr)
+        assert prebuilt_hash != 0     # xxx probably good enough
+        #
+        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        hdr._obj._name = typeid16.index   # debug only
+        hdr._obj.typeid16 = typeid16
+        hdr._obj.prebuilt_hash = prebuilt_hash
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -1,5 +1,31 @@
-from rpython.memory.gctransform import shadowstack
+from rpython.memory.gctransform.framework import (
+     BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr)
 
 
-class StmFrameworkGCTransformer(shadowstack.ShadowStackFrameworkGCTransformer):
-    pass
+class StmFrameworkGCTransformer(BaseFrameworkGCTransformer):
+
+    def build_root_walker(self):
+        return StmRootWalker(self)
+
+    def push_roots(self, hop, keep_current_args=False):
+        livevars = self.get_livevars_for_roots(hop, keep_current_args)
+        self.num_pushs += len(livevars)
+        for var in livevars:
+            hop.genop("stm_push_root", [var])
+        return livevars
+
+    def pop_roots(self, hop, livevars):
+        for var in reversed(livevars):
+            hop.genop("stm_pop_root_into", [var])
+
+    def gc_header_for(self, obj, needs_hash=False):
+        return self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
+
+
+class StmRootWalker(BaseRootWalker):
+
+    def need_thread_support(self, gctransform, getfn):
+        pass
+
+    def walk_stack_roots(self, collect_stack_root):
+        raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -429,6 +429,8 @@
     'stm_get_tid':            LLOp(canfold=True),
     'stm_id':                 LLOp(sideeffects=False),
     'stm_hash':               LLOp(sideeffects=False),
+    'stm_push_root':          LLOp(),
+    'stm_pop_root_into':      LLOp(),
 
     # __________ address operations __________
 
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -458,11 +458,14 @@
         return 'pypy_asm_stack_bottom();'
 
 class StmFrameworkGcPolicy(BasicFrameworkGcPolicy):
-    
+
     def gettransformer(self):
         from rpython.memory.gctransform import stmframework
         return stmframework.StmFrameworkGCTransformer(self.db.translator)
 
+    def get_prebuilt_hash(self, obj):
+        return None       # done differently with the stmgc
+
 
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -1002,6 +1002,9 @@
         return db.gcpolicy.rtti_node_factory()(db, T, obj)
     if T.hints.get("render_structure", False):
         return ExtType_OpaqueNode(db, T, obj)
+    if T.__name__ == 'struct stm_object_s':
+        from rpython.translator.stm.funcgen import StmHeader_OpaqueNode
+        return StmHeader_OpaqueNode(db, T, obj)
     raise Exception("don't know about %r" % (T,))
 
 
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -1,20 +1,44 @@
 from rpython.translator.c.support import c_string_constant
+from rpython.translator.c.node import ContainerNode
+from rpython.translator.c.primitive import name_small_integer
 from rpython.translator.stm.stmgcintf import StmOperations
 
 
+class StmHeader_OpaqueNode(ContainerNode):
+    nodekind = 'stmhdr'
+    globalcontainer = True
+    typename = 'struct stm_object_s @'
+    implementationtypename = typename
+    _funccodegen_owner = None
+
+    def __init__(self, db, T, obj):
+        assert isinstance(obj._name, int)
+        self.db = db
+        self.T = T
+        self.obj = obj
+
+    def initializationexpr(self, decoration=''):
+        yield '{ %s | PREBUILT_FLAGS, PREBUILT_REVISION, %dL }' % (
+            name_small_integer(self.obj.typeid16, self.db),
+            self.obj.prebuilt_hash)
+
+
 def stm_start_transaction(funcgen, op):
+    xxx
     # only for testing.  With stmgc, this operation should have been handled
     # already by gctransform.
     assert funcgen.db.translator.config.translation.gc == 'none'
     return 'stm_nogc_start_transaction();'
 
 def stm_stop_transaction(funcgen, op):
+    xxx
     # only for testing.  With stmgc, this operation should have been handled
     # already by gctransform.
     assert funcgen.db.translator.config.translation.gc == 'none'
     return 'stm_nogc_stop_transaction();'
 
 def stm_barrier(funcgen, op):
+    xxx
     category_change = op.args[0].value
     assert type(category_change) is str and len(category_change) == 3   # "x2y"
     arg = funcgen.expr(op.args[1])
@@ -22,12 +46,14 @@
     return '%s = stm_barrier_%s(%s);' % (result, category_change, arg)
 
 def stm_ptr_eq(funcgen, op):
+    xxx
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
     result = funcgen.expr(op.result)
     return '%s = stm_pointer_equal(%s, %s);' % (result, arg0, arg1)
 
 def stm_become_inevitable(funcgen, op):
+    xxx
     try:
         info = op.args[0].value
     except IndexError:
@@ -39,21 +65,25 @@
 ##    return funcgen.OP_DIRECT_CALL(op)
 
 def stm_abort_info_push(funcgen, op):
+    xxx
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
     return 'stm_abort_info_push(%s, %s);' % (arg0, arg1)
 
 def stm_extraref_llcount(funcgen, op):
+    xxx
     result = funcgen.expr(op.result)
     return '%s = stm_extraref_llcount();' % (result,)
 
 def stm_extraref_lladdr(funcgen, op):
+    xxx
     arg0 = funcgen.expr(op.args[0])
     result = funcgen.expr(op.result)
     return '%s = (char *)stm_extraref_lladdr(%s);' % (result, arg0)
 
 def _stm_nogc_init_function():
     """Called at process start-up when running with no GC."""
+    xxx
     StmOperations.descriptor_init()
     StmOperations.begin_inevitable_transaction()
 

From noreply at buildbot.pypy.org  Sat Jun 29 20:04:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 20:04:15 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: still in-progress
Message-ID: <20130629180415.345F31C1450@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65102:e41f1d5ce8a2
Date: 2013-06-29 20:03 +0200
http://bitbucket.org/pypy/pypy/changeset/e41f1d5ce8a2/

Log:	still in-progress

diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -37,13 +37,22 @@
     assert funcgen.db.translator.config.translation.gc == 'none'
     return 'stm_nogc_stop_transaction();'
 
+_STM_BARRIER_FUNCS = {   # XXX try to see if some combinations can be shorter
+    'P2R': 'stm_read_barrier',
+    'G2R': 'stm_read_barrier',
+    'O2R': 'stm_read_barrier',
+    'P2W': 'stm_write_barrier',
+    'G2W': 'stm_write_barrier',
+    'O2W': 'stm_write_barrier',
+    'R2W': 'stm_write_barrier',
+    }
+
 def stm_barrier(funcgen, op):
-    xxx
     category_change = op.args[0].value
-    assert type(category_change) is str and len(category_change) == 3   # "x2y"
+    funcname = _STM_BARRIER_FUNCS[category_change]
     arg = funcgen.expr(op.args[1])
     result = funcgen.expr(op.result)
-    return '%s = stm_barrier_%s(%s);' % (result, category_change, arg)
+    return '%s = %s(%s);' % (result, funcname, arg)
 
 def stm_ptr_eq(funcgen, op):
     xxx
@@ -53,7 +62,6 @@
     return '%s = stm_pointer_equal(%s, %s);' % (result, arg0, arg1)
 
 def stm_become_inevitable(funcgen, op):
-    xxx
     try:
         info = op.args[0].value
     except IndexError:
diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -84,7 +84,12 @@
         funcptr = get_funcobj(op.args[0].value)
         if not hasattr(funcptr, "external"):
             return False
-        return not getattr(funcptr, "transactionsafe", False)
+        if getattr(funcptr, "transactionsafe", False):
+            return False
+        try:
+            return funcptr._name + '()'
+        except AttributeError:
+            return True
 
     if op.opname == 'indirect_call':
         tographs = op.args[-1].value
@@ -118,6 +123,9 @@
     for block in graph.iterblocks():
         for i in range(len(block.operations)-1, -1, -1):
             op = block.operations[i]
-            if should_turn_inevitable(op, block, fresh_mallocs):
-                inev_op = turn_inevitable_op(op.opname)
+            inev = should_turn_inevitable(op, block, fresh_mallocs)
+            if inev:
+                if not isinstance(inev, str):
+                    inev = op.opname
+                inev_op = turn_inevitable_op(inev)
                 block.operations.insert(i, inev_op)

From noreply at buildbot.pypy.org  Sat Jun 29 20:23:34 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 20:23:34 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: in-progress
Message-ID: <20130629182334.2E2211C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65103:f2dc70f514d0
Date: 2013-06-29 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/f2dc70f514d0/

Log:	in-progress

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -1,5 +1,4 @@
 import thread, weakref
-from rpython.translator.stm import stmgcintf
 from rpython.rlib.debug import ll_assert, fatalerror
 from rpython.rlib.objectmodel import keepalive_until_here, specialize
 from rpython.rlib.objectmodel import we_are_translated
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -592,6 +592,8 @@
     OP_STM_ABORT_INFO_PUSH = _OP_STM
     OP_STM_EXTRAREF_LLCOUNT = _OP_STM
     OP_STM_EXTRAREF_LLADDR = _OP_STM
+    OP_STM_PUSH_ROOT = _OP_STM
+    OP_STM_POP_ROOT_INTO = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -1,7 +1,6 @@
-from rpython.translator.c.support import c_string_constant
+from rpython.translator.c.support import c_string_constant, cdecl
 from rpython.translator.c.node import ContainerNode
 from rpython.translator.c.primitive import name_small_integer
-from rpython.translator.stm.stmgcintf import StmOperations
 
 
 class StmHeader_OpaqueNode(ContainerNode):
@@ -50,9 +49,12 @@
 def stm_barrier(funcgen, op):
     category_change = op.args[0].value
     funcname = _STM_BARRIER_FUNCS[category_change]
+    assert op.args[1].concretetype == op.result.concretetype
     arg = funcgen.expr(op.args[1])
     result = funcgen.expr(op.result)
-    return '%s = %s(%s);' % (result, funcname, arg)
+    return '%s = (%s)%s((gcptr)%s);' % (
+        result, cdecl(funcgen.lltypename(op.result), ''),
+        funcname, arg)
 
 def stm_ptr_eq(funcgen, op):
     xxx
@@ -95,6 +97,15 @@
     StmOperations.descriptor_init()
     StmOperations.begin_inevitable_transaction()
 
+def stm_push_root(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    return 'stm_push_root((gcptr)%s);' % (arg0,)
+
+def stm_pop_root_into(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    return '%s = (%s)stm_pop_root();' % (
+        arg0, cdecl(funcgen.lltypename(op.args[0]), ''))
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/jitdriver.py b/rpython/translator/stm/jitdriver.py
--- a/rpython/translator/stm/jitdriver.py
+++ b/rpython/translator/stm/jitdriver.py
@@ -2,7 +2,6 @@
 from rpython.flowspace.model import checkgraph, copygraph
 from rpython.flowspace.model import Block, Link, SpaceOperation, Constant
 from rpython.translator.unsimplify import split_block, varoftype
-from rpython.translator.stm.stmgcintf import StmOperations
 from rpython.annotator.model import lltype_to_annotation, s_Int
 from rpython.rtyper.annlowlevel import (MixLevelHelperAnnotator,
                                       cast_base_ptr_to_instance)
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -1,7 +1,5 @@
-import py, os
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+import os
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.conftest import cdir as cdir2
 
 
@@ -10,73 +8,4 @@
 eci = ExternalCompilationInfo(
     include_dirs = [cdir, cdir2],
     includes = ['src_stm/stmgc.h'],
-    pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT,
-                        '#define RPY_STM 1'],
 )
-
-def _llexternal(name, args, result, **kwds):
-    return rffi.llexternal(name, args, result, compilation_info=eci,
-                           _nowrapper=True, transactionsafe=True,
-                           **kwds)
-
-def smexternal(name, args, result):
-    return staticmethod(_llexternal(name, args, result))
-
-# ____________________________________________________________
-
-
-class StmOperations(object):
-
-    CALLBACK_TX     = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed],
-                                                 lltype.Signed))
-    DUPLICATE       = lltype.Ptr(lltype.FuncType([llmemory.Address],
-                                                 llmemory.Address))
-    CALLBACK_ENUM   = lltype.Ptr(lltype.FuncType([llmemory.Address]*2,
-                                                 lltype.Void))
-
-    def _freeze_(self):
-        return True
-
-    # C part of the implementation of the pypy.rlib.rstm module
-    in_transaction = smexternal('stm_in_transaction', [], lltype.Signed)
-    is_inevitable = smexternal('stm_is_inevitable', [], lltype.Signed)
-    should_break_transaction = smexternal('stm_should_break_transaction',
-                                          [], lltype.Signed)
-    add_atomic = smexternal('stm_add_atomic', [lltype.Signed], lltype.Void)
-    get_atomic = smexternal('stm_get_atomic', [], lltype.Signed)
-    descriptor_init = smexternal('DescriptorInit', [], lltype.Signed)
-    descriptor_done = smexternal('DescriptorDone', [], lltype.Void)
-    begin_inevitable_transaction = smexternal(
-        'BeginInevitableTransaction', [], lltype.Void)
-    commit_transaction = smexternal(
-        'CommitTransaction', [], lltype.Void)
-    perform_transaction = smexternal('stm_perform_transaction',
-                                     [CALLBACK_TX, rffi.VOIDP, llmemory.Address],
-                                     lltype.Void)
-
-    # for the GC: store and read a thread-local-storage field
-    set_tls = smexternal('stm_set_tls', [llmemory.Address], llmemory.Address)
-    get_tls = smexternal('stm_get_tls', [], llmemory.Address)
-    del_tls = smexternal('stm_del_tls', [], lltype.Void)
-
-    # calls FindRootsForLocalCollect() and invokes for each such root
-    # the callback set in CALLBACK_ENUM.
-    tldict_enum = smexternal('stm_tldict_enum', [], lltype.Void)
-    tldict_enum_external = smexternal('stm_tldict_enum_external',
-                                      [llmemory.Address], lltype.Void)
-
-    # sets the transaction length, after which should_break_transaction()
-    # returns True
-    set_transaction_length = smexternal('stm_set_transaction_length',
-                                        [lltype.Signed], lltype.Void)
-
-    abort_info_pop = smexternal('stm_abort_info_pop',
-                                [lltype.Signed], lltype.Void)
-    inspect_abort_info = smexternal('stm_inspect_abort_info',
-                                    [], rffi.CCHARP)
-
-    start_single_thread = smexternal('stm_start_single_thread',[], lltype.Void)
-    stop_single_thread  = smexternal('stm_stop_single_thread', [], lltype.Void)
-
-    # for testing
-    abort_and_retry = smexternal('stm_abort_and_retry', [], lltype.Void)

From noreply at buildbot.pypy.org  Sat Jun 29 20:23:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 20:23:35 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: always in-progress
Message-ID: <20130629182335.612CD1C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65104:a35b41057209
Date: 2013-06-29 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/a35b41057209/

Log:	always in-progress

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -38,19 +38,23 @@
         hdr._obj.typeid16 = typeid16
         hdr._obj.prebuilt_hash = prebuilt_hash
 
-    def malloc_fixedsize_clear(self, typeid, size,
+    def malloc_fixedsize_clear(self, typeid16, size,
                                needs_finalizer=False,
                                is_finalizer_light=False,
                                contains_weakptr=False):
         ll_assert(not needs_finalizer, 'XXX')
         ll_assert(not is_finalizer_light, 'XXX')
         ll_assert(not contains_weakptr, 'XXX')
-        return llop.stm_allocate(llmemory.GCREF, size, typeid)
+        # XXX call optimized versions, e.g. if size < GC_NURSERY_SECTION
+        return llop.stm_allocate(llmemory.GCREF, size, typeid16)
 
-    def malloc_varsize_clear(self, typeid, length, size, itemsize,
+    def malloc_varsize_clear(self, typeid16, length, size, itemsize,
                              offset_to_length):
-        ll_assert(False, 'XXX')
-        return llop.stm_allocate(llmemory.GCREF)
+        # XXX be careful about overflows, and call optimized versions
+        totalsize = size + itemsize * length
+        obj = llop.stm_allocate(llmemory.Address, typeid16, totalsize)
+        (obj + offset_to_length).signed[0] = length
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def collect(self, gen=1):
         """Do a minor (gen=0) or major (gen>0) collection."""
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -584,16 +584,12 @@
             from rpython.translator.stm.funcgen import op_stm
             self.__class__.op_stm = op_stm
         return self.op_stm(op)
-    OP_STM_START_TRANSACTION = _OP_STM
-    OP_STM_STOP_TRANSACTION = _OP_STM
     OP_STM_BECOME_INEVITABLE = _OP_STM
     OP_STM_BARRIER = _OP_STM
     OP_STM_PTR_EQ = _OP_STM
-    OP_STM_ABORT_INFO_PUSH = _OP_STM
-    OP_STM_EXTRAREF_LLCOUNT = _OP_STM
-    OP_STM_EXTRAREF_LLADDR = _OP_STM
     OP_STM_PUSH_ROOT = _OP_STM
     OP_STM_POP_ROOT_INTO = _OP_STM
+    OP_STM_ALLOCATE = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c
--- a/rpython/translator/c/src/debug_print.c
+++ b/rpython/translator/c/src/debug_print.c
@@ -135,10 +135,10 @@
           debug_stop_colors);
 }
 
-typedef Unsigned revision_t;
 #ifdef RPY_STM
 # include <src_stm/atomic_ops.h>
 #else
+  typedef long revision_t;
 # define bool_cas(vp, o, n) (*(vp)=(n), 1)
 #endif
 static volatile revision_t threadcounter = 0;
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -57,7 +57,6 @@
         funcname, arg)
 
 def stm_ptr_eq(funcgen, op):
-    xxx
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
     result = funcgen.expr(op.result)
@@ -71,32 +70,6 @@
     string_literal = c_string_constant(info)
     return 'stm_become_inevitable(%s);' % (string_literal,)
 
-##def stm_jit_invoke_code(funcgen, op):
-##    return funcgen.OP_DIRECT_CALL(op)
-
-def stm_abort_info_push(funcgen, op):
-    xxx
-    arg0 = funcgen.expr(op.args[0])
-    arg1 = funcgen.expr(op.args[1])
-    return 'stm_abort_info_push(%s, %s);' % (arg0, arg1)
-
-def stm_extraref_llcount(funcgen, op):
-    xxx
-    result = funcgen.expr(op.result)
-    return '%s = stm_extraref_llcount();' % (result,)
-
-def stm_extraref_lladdr(funcgen, op):
-    xxx
-    arg0 = funcgen.expr(op.args[0])
-    result = funcgen.expr(op.result)
-    return '%s = (char *)stm_extraref_lladdr(%s);' % (result, arg0)
-
-def _stm_nogc_init_function():
-    """Called at process start-up when running with no GC."""
-    xxx
-    StmOperations.descriptor_init()
-    StmOperations.begin_inevitable_transaction()
-
 def stm_push_root(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
     return 'stm_push_root((gcptr)%s);' % (arg0,)
@@ -106,6 +79,12 @@
     return '%s = (%s)stm_pop_root();' % (
         arg0, cdecl(funcgen.lltypename(op.args[0]), ''))
 
+def stm_allocate(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    result = funcgen.expr(op.result)
+    return '%s = stm_allocate(%s, %s);' % (result, arg0, arg1)
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -8,4 +8,5 @@
 eci = ExternalCompilationInfo(
     include_dirs = [cdir, cdir2],
     includes = ['src_stm/stmgc.h'],
+    pre_include_bits = ['#define RPY_STM 1'],
 )

From noreply at buildbot.pypy.org  Sat Jun 29 20:44:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 20:44:03 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: in-progress
Message-ID: <20130629184403.646461C06B1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65105:dadb9e0f175c
Date: 2013-06-29 20:43 +0200
http://bitbucket.org/pypy/pypy/changeset/dadb9e0f175c/

Log:	in-progress

diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c
--- a/rpython/translator/c/src/debug_print.c
+++ b/rpython/translator/c/src/debug_print.c
@@ -137,11 +137,13 @@
 
 #ifdef RPY_STM
 # include <src_stm/atomic_ops.h>
+# include <src_stm/fprintcolor.h>
 #else
   typedef long revision_t;
 # define bool_cas(vp, o, n) (*(vp)=(n), 1)
+# define dprintfcolor() 0
 #endif
-static volatile revision_t threadcounter = 0;
+static revision_t threadcounter = 0;
 
 static void _prepare_display_colors(void)
 {
@@ -160,7 +162,14 @@
     }
     else {
         /* tty output */
-        int color = 31 + (int)(counter % 7);
+        /* If we have STM and it is compiled in debug mode, share
+           the color to use.  It avoids endless confusion caused by
+           each thread using two independent colors for the two
+           debugging outputs. */
+        int color = dprintfcolor();
+        if (color == 0) {
+            color = 31 + (int)(counter % 7);
+        }
         sprintf(debug_start_colors_1, "\033[%dm%d# \033[1m",
                 color, (int)counter);
         sprintf(debug_start_colors_2, "\033[%dm%d# ",
diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c
--- a/rpython/translator/c/src/entrypoint.c
+++ b/rpython/translator/c/src/entrypoint.c
@@ -14,11 +14,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-#ifdef RPY_STM
-#include <src/debug_print.h>
-#include "src_stm/et.c"
-#endif
-
 #ifdef __GNUC__
 /* Hack to prevent this function from being inlined.  Helps asmgcc
    because the main() function has often a different prologue/epilogue. */
diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c
--- a/rpython/translator/c/src/mem.c
+++ b/rpython/translator/c/src/mem.c
@@ -17,10 +17,10 @@
 
 #ifdef RPY_STM
 # include "src_stm/atomic_ops.h"
-static volatile Unsigned pypy_debug_alloc_lock = 0;
+static revision_t pypy_debug_alloc_lock = 0;
 #else
-# define stm_lock_acquire(lock)  /* nothing */
-# define stm_lock_release(lock)  /* nothing */
+# define spinlock_acquire(lock, targetvalue)  /* nothing */
+# define spinlock_release(lock)               /* nothing */
 #endif
 
 void pypy_debug_alloc_start(void *addr, const char *funcname)
@@ -29,23 +29,23 @@
   RPyAssert(p, "out of memory");
   p->addr = addr;
   p->funcname = funcname;
-  stm_lock_acquire(pypy_debug_alloc_lock);
+  spinlock_acquire(pypy_debug_alloc_lock, '+');
   p->next = pypy_debug_alloc_list;
   pypy_debug_alloc_list = p;
-  stm_lock_release(pypy_debug_alloc_lock);
+  spinlock_release(pypy_debug_alloc_lock);
 }
 
 void pypy_debug_alloc_stop(void *addr)
 {
   struct pypy_debug_alloc_s **p;
-  stm_lock_acquire(pypy_debug_alloc_lock);
+  spinlock_acquire(pypy_debug_alloc_lock, '-');
   for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next))
     if ((*p)->addr == addr)
       {
         struct pypy_debug_alloc_s *dying;
         dying = *p;
         *p = dying->next;
-        stm_lock_release(pypy_debug_alloc_lock);
+        spinlock_release(pypy_debug_alloc_lock);
         free(dying);
         return;
       }
diff --git a/rpython/translator/stm/src_stm/fprintcolor.c b/rpython/translator/stm/src_stm/fprintcolor.c
--- a/rpython/translator/stm/src_stm/fprintcolor.c
+++ b/rpython/translator/stm/src_stm/fprintcolor.c
@@ -25,12 +25,8 @@
 static __thread revision_t tcolor = 0;
 static revision_t tnextid = 0;
 
-
-int threadcolor_printf(const char *format, ...)
+int dprintfcolor(void)
 {
-    char buffer[2048];
-    va_list ap;
-    int result;
     if (tcolor == 0) {
         while (1) {
             tcolor = tnextid;
@@ -39,7 +35,15 @@
         }
         tcolor = 31 + tcolor % 6;
     }
-    int size = (int)sprintf(buffer, "\033[%dm", (int)tcolor);
+    return tcolor;
+}
+
+int threadcolor_printf(const char *format, ...)
+{
+    char buffer[2048];
+    va_list ap;
+    int result;
+    int size = (int)sprintf(buffer, "\033[%dm", dprintfcolor());
     assert(size >= 0);
 
     va_start(ap, format);
diff --git a/rpython/translator/stm/src_stm/fprintcolor.h b/rpython/translator/stm/src_stm/fprintcolor.h
--- a/rpython/translator/stm/src_stm/fprintcolor.h
+++ b/rpython/translator/stm/src_stm/fprintcolor.h
@@ -10,6 +10,7 @@
 #ifdef _GC_DEBUG
 
 #define dprintf(args)   threadcolor_printf args
+int dprintfcolor(void);
 
 int threadcolor_printf(const char *format, ...)
      __attribute__((format (printf, 1, 2)));
@@ -17,5 +18,6 @@
 #else
 
 #define dprintf(args)   do { } while(0)
+#define dprintfcolor()  0
 
 #endif
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-c856b6a0d157
+c856b6a0d157+
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -9,4 +9,5 @@
     include_dirs = [cdir, cdir2],
     includes = ['src_stm/stmgc.h'],
     pre_include_bits = ['#define RPY_STM 1'],
+    separate_module_sources = ['\n#include "src_stm/stmgc.c"\n'],
 )

From noreply at buildbot.pypy.org  Sat Jun 29 21:58:14 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 29 Jun 2013 21:58:14 +0200 (CEST)
Subject: [pypy-commit] buildbot numpy-tests: these steps work by hand,
 not sure about the last one
Message-ID: <20130629195814.5FF9F1C0255@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: numpy-tests
Changeset: r829:ce457ef29b96
Date: 2013-06-29 22:56 +0300
http://bitbucket.org/pypy/buildbot/changeset/ce457ef29b96/

Log:	these steps work by hand, not sure about the last one

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -771,27 +771,38 @@
         else:
             self.addStep(ShellCmd(
                 description="decompress pypy-c",
-                command=['tar', '--extract', '--file=pypy_build'+ extension, '--strip-components=1', '--directory=.'],
-                workdir='pypy-c'))
+                command=['tar', '--extract', '--file=pypy_build'+ extension,
+                         '--strip-components=1', '--directory=.'],
+                workdir='pypy-c/download'))
 
-        # copy pypy-c to the expected location within the pypy source checkout
+        # virtualenv the download
         self.addStep(ShellCmd(
-            description="move pypy-c",
-            command=['cp', '-v', 'pypy-c/bin/pypy', 'build/pypy/goal/pypy-c'],
-            workdir='.'))
-        # copy generated and copied header files to build/include
+            description="create virtualenv",
+            command=['virtualenv','-p', 'download/bin/pypy', 'install'],
+            workdir='pypy-c'))
+
         self.addStep(ShellCmd(
-            description="move header files",
-            command=['cp', '-vr', 'pypy-c/include', 'build'],
-            workdir='.'))
-        # copy ctypes_resource_cache generated during translation
-        self.addStep(ShellCmd(
-            description="move ctypes resource cache",
-            command=['cp', '-rv', 'pypy-c/lib_pypy/ctypes_config_cache', 'build/lib_pypy'],
-            workdir='.'))
+            description="install nose",
+            command=['download/bin/pip', 'install','nose'],
+            workdir='pypy-c'))
 
         # obtain a pypy-compatible branch of numpy
         numpy_url = 'https://github.com/mattip/numpy'
         numpy_pypy_branch = 'pypy'
         update_git(platform, factory, numpy_url, 'numpy_src', use_branch=True,
               force_branch=numpy_pypy_branch)
+
+        self.addStep(ShellCmd(
+            description="install numpy",
+            command=['download/bin/python', 'setup.py','install'],
+            workdir='numpy_src'))
+
+        self.addStep(ShellCmd(
+            description="test numpy",
+            command=['download/bin/python', '-c', '"import numpy;numpy.test()"',
+                     '> pytest-numpy.log','2>&1'],
+            logfiles={'pytestLog': 'pytest-numpy.log'},
+            timeout=4000,
+            workdir='numpy_src',
+            #env={"PYTHONPATH": ['download']}, # shouldn't be needed, but what if it is set externally?
+        ))

From noreply at buildbot.pypy.org  Sat Jun 29 22:17:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 22:17:08 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: in-progress
Message-ID: <20130629201708.1517D1C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65106:22d6efdb107a
Date: 2013-06-29 22:06 +0200
http://bitbucket.org/pypy/pypy/changeset/22d6efdb107a/

Log:	in-progress

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -23,6 +23,20 @@
 
     TRANSLATION_PARAMS = {}
 
+    def __init__(self, config, **kwds):
+        MovingGCBase.__init__(self, config, **kwds)
+
+        def stmcb_size(obj):
+            typeid = self.get_type_id(obj)
+            size = self.fixed_size(typeid)
+            if self.is_varsize(typeid):
+                lenofs = self.varsize_offset_to_length(typeid)
+                length = (obj + lenofs).signed[0]
+                size += self.varsize_item_sizes(typeid) * length
+            return size
+
+        
+
     def get_type_id(self, obj):
         return llop.stm_get_tid(llgroup.HALFWORD, obj)
 
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -5,9 +5,13 @@
 
 cdir = os.path.abspath(os.path.join(cdir2, '..', 'stm'))
 
+separate_source = '''
+#include "src_stm/stmgc.c"
+'''
+
 eci = ExternalCompilationInfo(
     include_dirs = [cdir, cdir2],
     includes = ['src_stm/stmgc.h'],
     pre_include_bits = ['#define RPY_STM 1'],
-    separate_module_sources = ['\n#include "src_stm/stmgc.c"\n'],
+    separate_module_sources = [separate_source],
 )

From noreply at buildbot.pypy.org  Sat Jun 29 22:17:16 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 22:17:16 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: hg merge default
Message-ID: <20130629201716.4AA101C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65107:98aa431967af
Date: 2013-06-29 22:16 +0200
http://bitbucket.org/pypy/pypy/changeset/98aa431967af/

Log:	hg merge default

diff too long, truncating to 2000 out of 48802 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -6,6 +6,7 @@
 .idea
 .project
 .pydevproject
+__pycache__
 
 syntax: regexp
 ^testresult$
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,6 @@
 d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
+ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,7 @@
     Impara, Germany
     Change Maker, Sweden 
     University of California Berkeley, USA
+    Google Inc.
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/Makefile b/Makefile
new file mode 100644
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,7 @@
+
+all: pypy-c
+
+pypy-c:
+	@echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM"
+	@sleep 3
+	rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -22,3 +22,16 @@
 and send us feedback!
 
     the pypy-dev team <pypy-dev at python.org>
+
+Building
+========
+
+build with::
+
+  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+
+This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
+to use virtualenv with the resulting pypy-c as the interpreter, you can
+find more details about various installation schemes here:
+
+http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -8,7 +8,7 @@
 
 __revision__ = "$Id$"
 
-import sys, os, string, re
+import sys, os, string, re, imp
 from types import *
 from site import USER_BASE, USER_SITE
 from distutils.core import Command
@@ -33,6 +33,11 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
 
 class build_ext (Command):
 
@@ -677,10 +682,18 @@
         # OS/2 has an 8 character module (extension) limit :-(
         if os.name == "os2":
             ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
+        # PyPy tweak: first try to get the C extension suffix from
+        # 'imp'.  If it fails we fall back to the 'SO' config var, like
+        # the previous version of this code did.  This should work for
+        # CPython too.  The point is that on PyPy with cpyext, the
+        # config var 'SO' is just ".so" but we want to return
+        # ".pypy-VERSION.so" instead.
+        so_ext = _get_c_extension_suffix()
+        if so_ext is None:
+            so_ext = get_config_var('SO')     # fall-back
         # extensions in debug_mode are named 'module_d.pyd' under windows
-        so_ext = get_config_var('SO')
         if os.name == 'nt' and self.debug:
-            return os.path.join(*ext_path) + '_d' + so_ext
+            so_ext = '_d.pyd'
         return os.path.join(*ext_path) + so_ext
 
     def get_export_symbols (self, ext):
diff --git a/lib-python/2.7/distutils/command/install.py b/lib-python/2.7/distutils/command/install.py
--- a/lib-python/2.7/distutils/command/install.py
+++ b/lib-python/2.7/distutils/command/install.py
@@ -474,8 +474,8 @@
 
     def select_scheme (self, name):
         # it's the caller's problem if they supply a bad name!
-        if hasattr(sys, 'pypy_version_info') and not (
-                name.endswith('_user') or name.endswith('_home')):
+        if (hasattr(sys, 'pypy_version_info') and
+            not name.endswith(('_user', '_home'))):
             name = 'pypy'
         scheme = INSTALL_SCHEMES[name]
         for key in SCHEME_KEYS:
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -1,30 +1,16 @@
-"""Provide access to Python's configuration information.  The specific
-configuration variables available depend heavily on the platform and
-configuration.  The values may be retrieved using
-get_config_var(name), and the list of variables is available via
-get_config_vars().keys().  Additional convenience functions are also
-available.
-
-Written by:   Fred L. Drake, Jr.
-Email:        <fdrake at acm.org>
-"""
-
-__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
-
-import sys
-
 
 # The content of this file is redirected from
 # sysconfig_cpython or sysconfig_pypy.
+# All underscore names are imported too, because
+# people like to use undocumented sysconfig._xxx
+# directly.
 
+import sys
 if '__pypy__' in sys.builtin_module_names:
-    from distutils.sysconfig_pypy import *
-    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
-    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+    from distutils import sysconfig_pypy as _sysconfig_module
 else:
-    from distutils.sysconfig_cpython import *
-    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
-    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
+    from distutils import sysconfig_cpython as _sysconfig_module
+globals().update(_sysconfig_module.__dict__)
 
 _USE_CLANG = None
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -9,7 +9,7 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
 import os
 import re
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -1,9 +1,17 @@
-"""PyPy's minimal configuration information.
+"""Provide access to Python's configuration information.
+This is actually PyPy's minimal configuration information.
+
+The specific configuration variables available depend heavily on the
+platform and configuration.  The values may be retrieved using
+get_config_var(name), and the list of variables is available via
+get_config_vars().keys().  Additional convenience functions are also
+available.
 """
 
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+
 import sys
 import os
-import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -49,16 +57,11 @@
 
 _config_vars = None
 
-def _get_so_extension():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
-
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = _get_so_extension() or ".so"
+    g['SO'] = ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
@@ -71,7 +74,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = _get_so_extension() or ".pyd"
+    g['SO'] = ".pyd"
     g['SOABI'] = g['SO'].rsplit('.')[0]
 
     global _config_vars
@@ -119,13 +122,13 @@
     optional C speedup components.
     """
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-fPIC', '-Wimplicit'])
+        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CFLAGS" in os.environ:
-            cflags = os.environ["CFLAGS"]
-            compiler.compiler.append(cflags)
-            compiler.compiler_so.append(cflags)
-            compiler.linker_so.append(cflags)
+            cflags = os.environ["CFLAGS"].split()
+            compiler.compiler.extend(cflags)
+            compiler.compiler_so.extend(cflags)
+            compiler.linker_so.extend(cflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/2.7/json/decoder.py b/lib-python/2.7/json/decoder.py
--- a/lib-python/2.7/json/decoder.py
+++ b/lib-python/2.7/json/decoder.py
@@ -162,7 +162,7 @@
         if nextchar == '}':
             if object_pairs_hook is not None:
                 result = object_pairs_hook(pairs)
-                return result, end
+                return result, end + 1
             pairs = {}
             if object_hook is not None:
                 pairs = object_hook(pairs)
diff --git a/lib-python/2.7/json/tests/test_decode.py b/lib-python/2.7/json/tests/test_decode.py
--- a/lib-python/2.7/json/tests/test_decode.py
+++ b/lib-python/2.7/json/tests/test_decode.py
@@ -44,6 +44,7 @@
                                     object_pairs_hook=OrderedDict,
                                     object_hook=lambda x: None),
                          OrderedDict(p))
+        self.assertEqual(self.loads("{}", object_pairs_hook=list), [])
 
 
 class TestPyDecode(TestDecode, PyTest): pass
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -134,21 +134,30 @@
 DEBUG = 10
 NOTSET = 0
 
-_levelNames = {
-    CRITICAL : 'CRITICAL',
-    ERROR : 'ERROR',
-    WARNING : 'WARNING',
-    INFO : 'INFO',
-    DEBUG : 'DEBUG',
-    NOTSET : 'NOTSET',
-    'CRITICAL' : CRITICAL,
-    'ERROR' : ERROR,
-    'WARN' : WARNING,
-    'WARNING' : WARNING,
-    'INFO' : INFO,
-    'DEBUG' : DEBUG,
-    'NOTSET' : NOTSET,
+# NOTE(flaper87): This is different from
+# python's stdlib module since pypy's
+# dicts are much faster when their
+# keys are all of the same type.
+# Introduced in commit 9de7b40c586f
+_levelToName = {
+    CRITICAL: 'CRITICAL',
+    ERROR: 'ERROR',
+    WARNING: 'WARNING',
+    INFO: 'INFO',
+    DEBUG: 'DEBUG',
+    NOTSET: 'NOTSET',
 }
+_nameToLevel = {
+    'CRITICAL': CRITICAL,
+    'ERROR': ERROR,
+    'WARN': WARNING,
+    'WARNING': WARNING,
+    'INFO': INFO,
+    'DEBUG': DEBUG,
+    'NOTSET': NOTSET,
+}
+_levelNames = dict(_levelToName)
+_levelNames.update(_nameToLevel)   # backward compatibility
 
 def getLevelName(level):
     """
@@ -164,7 +173,11 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    return _levelNames.get(level, ("Level %s" % level))
+
+    # NOTE(flaper87): Check also in _nameToLevel
+    # if value is None.
+    return (_levelToName.get(level) or
+            _nameToLevel.get(level, ("Level %s" % level)))
 
 def addLevelName(level, levelName):
     """
@@ -174,8 +187,8 @@
     """
     _acquireLock()
     try:    #unlikely to cause an exception, but you never know...
-        _levelNames[level] = levelName
-        _levelNames[levelName] = level
+        _levelToName[level] = levelName
+        _nameToLevel[levelName] = level
     finally:
         _releaseLock()
 
@@ -183,9 +196,9 @@
     if isinstance(level, int):
         rv = level
     elif str(level) == level:
-        if level not in _levelNames:
+        if level not in _nameToLevel:
             raise ValueError("Unknown level: %r" % level)
-        rv = _levelNames[level]
+        rv = _nameToLevel[level]
     else:
         raise TypeError("Level not an integer or a valid string: %r" % level)
     return rv
@@ -277,7 +290,7 @@
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - long(ct)) * 1000
+        self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads and thread:
             self.thread = thread.get_ident()
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -156,7 +156,7 @@
         h = klass(*args)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            h.setLevel(logging._levelNames[level])
+            h.setLevel(level)
         if len(fmt):
             h.setFormatter(formatters[fmt])
         if issubclass(klass, logging.handlers.MemoryHandler):
@@ -187,7 +187,7 @@
     opts = cp.options(sectname)
     if "level" in opts:
         level = cp.get(sectname, "level")
-        log.setLevel(logging._levelNames[level])
+        log.setLevel(level)
     for h in root.handlers[:]:
         root.removeHandler(h)
     hlist = cp.get(sectname, "handlers")
@@ -237,7 +237,7 @@
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
-            logger.setLevel(logging._levelNames[level])
+            logger.setLevel(level)
         for h in logger.handlers[:]:
             logger.removeHandler(h)
         logger.propagate = propagate
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -193,5 +193,6 @@
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
+jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -1953,7 +1953,11 @@
                 if key is None:
                     callback(None, modname, '')
                 else:
-                    desc = split(__import__(modname).__doc__ or '', '\n')[0]
+                    try:
+                        module_doc = __import__(modname).__doc__
+                    except ImportError:
+                        module_doc = None
+                    desc = split(module_doc or '', '\n')[0]
                     if find(lower(modname + ' - ' + desc), key) >= 0:
                         callback(None, modname, desc)
 
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -96,6 +96,7 @@
 
 
 _realsocket = socket
+_type = type
 
 # WSA error codes
 if sys.platform.lower().startswith("win"):
@@ -173,31 +174,37 @@
 
     __doc__ = _realsocket.__doc__
 
+    __slots__ = ["_sock", "__weakref__"]
+
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+        elif _type(_sock) is _realsocket:
+            _sock._reuse()
+        # PyPy note about refcounting: implemented with _reuse()/_drop()
+        # on the class '_socket.socket'.  Python 3 did it differently
+        # with a reference counter on this class 'socket._socketobject'
+        # instead, but it is a less compatible change (breaks eventlet).
         self._sock = _sock
-        self._io_refs = 0
-        self._closed = False
 
     def send(self, data, flags=0):
-        return self._sock.send(data, flags=flags)
+        return self._sock.send(data, flags)
     send.__doc__ = _realsocket.send.__doc__
 
     def recv(self, buffersize, flags=0):
-        return self._sock.recv(buffersize, flags=flags)
+        return self._sock.recv(buffersize, flags)
     recv.__doc__ = _realsocket.recv.__doc__
 
     def recv_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recv_into(buffer, nbytes, flags)
     recv_into.__doc__ = _realsocket.recv_into.__doc__
 
     def recvfrom(self, buffersize, flags=0):
-        return self._sock.recvfrom(buffersize, flags=flags)
+        return self._sock.recvfrom(buffersize, flags)
     recvfrom.__doc__ = _realsocket.recvfrom.__doc__
 
     def recvfrom_into(self, buffer, nbytes=0, flags=0):
-        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+        return self._sock.recvfrom_into(buffer, nbytes, flags)
     recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
 
     def sendto(self, data, param2, param3=None):
@@ -208,13 +215,17 @@
     sendto.__doc__ = _realsocket.sendto.__doc__
 
     def close(self):
-        # This function should not reference any globals. See issue #808164.
+        s = self._sock
+        if type(s) is _realsocket:
+            s._drop()
         self._sock = _closedsocket()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+        sockobj = _socketobject(_sock=sock)
+        sock._drop()    # already a copy in the _socketobject()
+        return sockobj, addr
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -228,24 +239,7 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        self._io_refs += 1
-        return _fileobject(self, mode, bufsize)
-
-    def _decref_socketios(self):
-        if self._io_refs > 0:
-            self._io_refs -= 1
-        if self._closed:
-            self.close()
-
-    def _real_close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._sock.close()
-
-    def close(self):
-        # This function should not reference any globals. See issue #808164.
-        self._closed = True
-        if self._io_refs <= 0:
-            self._real_close()
+        return _fileobject(self._sock, mode, bufsize)
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
@@ -286,6 +280,8 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
+        if type(sock) is _realsocket:
+            sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -320,11 +316,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._sock:
-                if self._close:
-                    self._sock.close()
-                else:
-                    self._sock._decref_socketios()
+            s = self._sock
+            if type(s) is _realsocket:
+                s._drop()
+            if self._close:
+                self._sock.close()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -75,7 +75,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 """
diff --git a/lib-python/2.7/test/test_codecs.py b/lib-python/2.7/test/test_codecs.py
--- a/lib-python/2.7/test/test_codecs.py
+++ b/lib-python/2.7/test/test_codecs.py
@@ -2,7 +2,11 @@
 import unittest
 import codecs
 import locale
-import sys, StringIO, _testcapi
+import sys, StringIO
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 class Queue(object):
     """
@@ -1387,7 +1391,7 @@
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
-            if encoding not in broken_incremental_coders:
+            if encoding not in broken_incremental_coders and _testcapi:
                 # check incremental decoder/encoder (fetched via the Python
                 # and C API) and iterencode()/iterdecode()
                 try:
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -3592,6 +3592,9 @@
         list.__init__(a, sequence=[0, 1, 2])
         self.assertEqual(a, [0, 1, 2])
 
+    @unittest.skipIf(test_support.check_impl_detail(pypy=True) and
+                     sys.platform == 'win32',
+                     "XXX: https://bugs.pypy.org/issue1461")
     def test_recursive_call(self):
         # Testing recursive __call__() by setting to instance of class...
         class A(object):
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -53,25 +53,26 @@
     pass
 
 dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        41
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 LOAD_FAST                0 (x)
-             12 BUILD_LIST_FROM_ARG      0
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
+ %-4d         0 JUMP_IF_NOT_DEBUG       41 (to 44)
+              3 LOAD_CONST               1 (0)
+              6 POP_JUMP_IF_TRUE        44
+              9 LOAD_GLOBAL              0 (AssertionError)
+             12 LOAD_FAST                0 (x)
+             15 BUILD_LIST_FROM_ARG      0
+             18 GET_ITER
+        >>   19 FOR_ITER                12 (to 34)
+             22 STORE_FAST               1 (s)
+             25 LOAD_FAST                1 (s)
+             28 LIST_APPEND              2
+             31 JUMP_ABSOLUTE           19
 
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 CALL_FUNCTION            1
-             38 RAISE_VARARGS            1
+ %-4d   >>   34 LOAD_CONST               2 (1)
+             37 BINARY_ADD
+             38 CALL_FUNCTION            1
+             41 RAISE_VARARGS            1
 
- %-4d   >>   41 LOAD_CONST               0 (None)
-             44 RETURN_VALUE
+ %-4d   >>   44 LOAD_CONST               0 (None)
+             47 RETURN_VALUE
 """%(bug1333982.func_code.co_firstlineno + 1,
      bug1333982.func_code.co_firstlineno + 2,
      bug1333982.func_code.co_firstlineno + 3)
diff --git a/lib-python/2.7/test/test_fileio.py b/lib-python/2.7/test/test_fileio.py
--- a/lib-python/2.7/test/test_fileio.py
+++ b/lib-python/2.7/test/test_fileio.py
@@ -318,7 +318,6 @@
         self.assertRaises(ValueError, _FileIO, -10)
         self.assertRaises(OSError, _FileIO, make_bad_fd())
         if sys.platform == 'win32':
-            raise unittest.SkipTest('Set _invalid_parameter_handler for low level io')
             import msvcrt
             self.assertRaises(IOError, msvcrt.get_osfhandle, make_bad_fd())
 
diff --git a/lib-python/2.7/test/test_logging.py b/lib-python/2.7/test/test_logging.py
--- a/lib-python/2.7/test/test_logging.py
+++ b/lib-python/2.7/test/test_logging.py
@@ -65,7 +65,8 @@
             self.saved_handlers = logging._handlers.copy()
             self.saved_handler_list = logging._handlerList[:]
             self.saved_loggers = logger_dict.copy()
-            self.saved_level_names = logging._levelNames.copy()
+            self.saved_name_to_level = logging._nameToLevel.copy()
+            self.saved_level_to_name = logging._levelToName.copy()
         finally:
             logging._releaseLock()
 
@@ -97,8 +98,10 @@
         self.root_logger.setLevel(self.original_logging_level)
         logging._acquireLock()
         try:
-            logging._levelNames.clear()
-            logging._levelNames.update(self.saved_level_names)
+            logging._levelToName.clear()
+            logging._levelToName.update(self.saved_level_to_name)
+            logging._nameToLevel.clear()
+            logging._nameToLevel.update(self.saved_name_to_level)
             logging._handlers.clear()
             logging._handlers.update(self.saved_handlers)
             logging._handlerList[:] = self.saved_handler_list
@@ -275,6 +278,24 @@
     def test_invalid_name(self):
         self.assertRaises(TypeError, logging.getLogger, any)
 
+    def test_get_level_name(self):
+        """Test getLevelName returns level constant."""
+        # NOTE(flaper87): Bug #1517
+        self.assertEqual(logging.getLevelName('NOTSET'), 0)
+        self.assertEqual(logging.getLevelName('DEBUG'), 10)
+        self.assertEqual(logging.getLevelName('INFO'), 20)
+        self.assertEqual(logging.getLevelName('WARN'), 30)
+        self.assertEqual(logging.getLevelName('WARNING'), 30)
+        self.assertEqual(logging.getLevelName('ERROR'), 40)
+        self.assertEqual(logging.getLevelName('CRITICAL'), 50)
+
+        self.assertEqual(logging.getLevelName(0), 'NOTSET')
+        self.assertEqual(logging.getLevelName(10), 'DEBUG')
+        self.assertEqual(logging.getLevelName(20), 'INFO')
+        self.assertEqual(logging.getLevelName(30), 'WARNING')
+        self.assertEqual(logging.getLevelName(40), 'ERROR')
+        self.assertEqual(logging.getLevelName(50), 'CRITICAL')
+
 class BasicFilterTest(BaseTest):
 
     """Test the bundled Filter class."""
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -7,7 +7,8 @@
 import subprocess
 from copy import copy, deepcopy
 
-from test.test_support import run_unittest, TESTFN, unlink, get_attribute
+from test.test_support import (run_unittest, TESTFN, unlink, get_attribute,
+                               import_module)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -236,7 +237,10 @@
 
     def test_get_config_h_filename(self):
         config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
+        # import_module skips the test when the CPython C Extension API
+        # appears to not be supported
+        self.assertTrue(os.path.isfile(config_h) or
+                        not import_module('_testcapi'), config_h)
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -1,6 +1,9 @@
 """Test cases for traceback module"""
 
-from _testcapi import traceback_print
+try:
+    from _testcapi import traceback_print
+except ImportError:
+    traceback_print = None
 from StringIO import StringIO
 import sys
 import unittest
@@ -176,6 +179,8 @@
 class TracebackFormatTests(unittest.TestCase):
 
     def test_traceback_format(self):
+        if traceback_print is None:
+            raise unittest.SkipTest('Requires _testcapi')
         try:
             raise KeyError('blah')
         except KeyError:
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -1609,7 +1609,10 @@
         self.assertEqual("{}".format(u), '__unicode__ overridden')
 
     def test_encode_decimal(self):
-        from _testcapi import unicode_encodedecimal
+        try:
+            from _testcapi import unicode_encodedecimal
+        except ImportError:
+            raise unittest.SkipTest('Requires _testcapi')
         self.assertEqual(unicode_encodedecimal(u'123'),
                          b'123')
         self.assertEqual(unicode_encodedecimal(u'\u0663.\u0661\u0664'),
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -130,7 +130,7 @@
     RegrTest('test_bz2.py', usemodules='bz2'),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
-    RegrTest('test_capi.py'),
+    RegrTest('test_capi.py', usemodules='cpyext'),
     RegrTest('test_cd.py'),
     RegrTest('test_cfgparser.py'),
     RegrTest('test_cgi.py'),
@@ -177,7 +177,7 @@
     RegrTest('test_cprofile.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
     RegrTest('test_csv.py', usemodules='_csv'),
-    RegrTest('test_ctypes.py', usemodules="_rawffi thread"),
+    RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
     RegrTest('test_curses.py'),
     RegrTest('test_datetime.py', usemodules='binascii struct'),
     RegrTest('test_dbm.py'),
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -120,6 +120,7 @@
         return self._buffer[0] != 0
 
     contents = property(getcontents, setcontents)
+    _obj = property(getcontents) # byref interface
 
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -20,7 +20,7 @@
                 or tp._type_ not in "iIhHbBlLqQ"):
                 #XXX: are those all types?
                 #     we just dont get the type name
-                #     in the interp levle thrown TypeError
+                #     in the interp level thrown TypeError
                 #     from rawffi if there are more
                 raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
@@ -166,9 +166,7 @@
         if self is StructOrUnion:
             return
         if '_fields_' not in self.__dict__:
-            self._fields_ = []
-            self._names = []
-            _set_shape(self, [], self._is_union)
+            self._fields_ = []  # As a side-effet, this also sets the ffishape.
 
     __setattr__ = struct_setattr
 
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,60 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_ctypes_test.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_ctypes_test' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_ctypes_test']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
-    imp.load_module('_ctypes_test', fp, filename, description)
-
-
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_ctypes_test'")
 try:
     import _ctypes
     _ctypes.PyObj_FromPtr = None
@@ -62,4 +9,5 @@
 except ImportError:
     pass    # obscure condition of _ctypes_test.py being imported by py.test
 else:
-    compile_shared()
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -476,6 +476,15 @@
 def _chtype(ch):
     return int(ffi.cast("chtype", ch))
 
+def _texttype(text):
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, unicode):
+        return str(text)   # default encoding
+    else:
+        raise TypeError("str or unicode expected, got a '%s' object"
+                        % (type(text).__name__,))
+
 
 def _extract_yx(args):
     if len(args) >= 2:
@@ -589,6 +598,7 @@
 
     @_argspec(1, 1, 2)
     def addstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -602,6 +612,7 @@
 
     @_argspec(2, 1, 2)
     def addnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -780,6 +791,7 @@
 
     @_argspec(1, 1, 2)
     def insstr(self, y, x, text, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -793,6 +805,7 @@
 
     @_argspec(2, 1, 2)
     def insnstr(self, y, x, text, n, attr=None):
+        text = _texttype(text)
         if attr is not None:
             attr_old = lib.getattrs(self._win)
             lib.wattrset(self._win, attr)
@@ -1197,6 +1210,7 @@
 
 
 def putp(text):
+    text = _texttype(text)
     return _check_ERR(lib.putp(text), "putp")
 
 
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -1,4 +1,4 @@
-"""eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
+__doc__ = """eclguba: flagnk naq frznagvpf bs clguba, fcrrq bs p, erfgevpgvbaf bs wnin naq pbzcvyre reebe zrffntrf nf crargenoyr nf ZHZCF
 pglcrf unf n fcva bs 1/3
 ' ' vf n fcnpr gbb
 Clguba 2.k rfg cerfdhr zbeg, ivir Clguba!
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_testcapi.py
@@ -0,0 +1,61 @@
+import os, sys, imp
+import tempfile
+
+def _get_c_extension_suffix():
+    for ext, mod, typ in imp.get_suffixes():
+        if typ == imp.C_EXTENSION:
+            return ext
+
+
+def compile_shared(csource, modulename):
+    """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
+    and import it.
+    """
+    thisdir = os.path.dirname(__file__)
+    output_dir = tempfile.mkdtemp()
+
+    from distutils.ccompiler import new_compiler
+
+    compiler = new_compiler()
+    compiler.output_dir = output_dir
+
+    # Compile .c file
+    include_dir = os.path.join(thisdir, '..', 'include')
+    if sys.platform == 'win32':
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
+    else:
+        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
+    res = compiler.compile([os.path.join(thisdir, csource)],
+                           include_dirs=[include_dir],
+                           extra_preargs=ccflags)
+    object_filename = res[0]
+
+    # set link options
+    output_filename = modulename + _get_c_extension_suffix()
+    if sys.platform == 'win32':
+        # XXX libpypy-c.lib is currently not installed automatically
+        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
+        libraries = [library, 'oleaut32']
+        extra_ldargs = ['/MANIFEST',  # needed for VC10
+                        '/EXPORT:init' + modulename]
+    else:
+        libraries = []
+        extra_ldargs = []
+
+    # link the dynamic library
+    compiler.link_shared_object(
+        [object_filename],
+        output_filename,
+        libraries=libraries,
+        extra_preargs=extra_ldargs)
+
+    # Now import the newly created library, it will replace the original
+    # module in sys.modules
+    fp, filename, description = imp.find_module(modulename, path=[output_dir])
+    imp.load_module(modulename, fp, filename, description)
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,57 +1,7 @@
-import os, sys
-import tempfile
-
-def compile_shared():
-    """Compile '_testcapi.c' into an extension module, and import it
-    """
-    thisdir = os.path.dirname(__file__)
-    output_dir = tempfile.mkdtemp()
-
-    from distutils.ccompiler import new_compiler
-    from distutils import sysconfig
-
-    compiler = new_compiler()
-    compiler.output_dir = output_dir
-
-    # Compile .c file
-    include_dir = os.path.join(thisdir, '..', 'include')
-    if sys.platform == 'win32':
-        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
-    else:
-        ccflags = ['-fPIC', '-Wimplicit-function-declaration']
-    res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
-                           include_dirs=[include_dir],
-                           extra_preargs=ccflags)
-    object_filename = res[0]
-
-    # set link options
-    output_filename = '_testcapi' + sysconfig.get_config_var('SO')
-    if sys.platform == 'win32':
-        # XXX libpypy-c.lib is currently not installed automatically
-        library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
-        if not os.path.exists(library + '.lib'):
-            #For a nightly build
-            library = os.path.join(thisdir, '..', 'include', 'python27')
-        if not os.path.exists(library + '.lib'):
-            # For a local translation
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c')
-        libraries = [library, 'oleaut32']
-        extra_ldargs = ['/MANIFEST',  # needed for VC10
-                        '/EXPORT:init_testcapi']
-    else:
-        libraries = []
-        extra_ldargs = []
-
-    # link the dynamic library
-    compiler.link_shared_object(
-        [object_filename],
-        output_filename,
-        libraries=libraries,
-        extra_preargs=extra_ldargs)
-
-    # Now import the newly created library, it will replace our module in sys.modules
-    import imp
-    fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
-    imp.load_module('_testcapi', fp, filename, description)
-
-compile_shared()
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testcapi'")
+else:
+    import _pypy_testcapi
+    _pypy_testcapi.compile_shared('_testcapimodule.c', '_testcapi')
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/__init__.py
@@ -0,0 +1,48 @@
+# _tkinter package -- low-level interface to libtk and libtcl.
+#
+# This is an internal module, applications should "import Tkinter" instead.
+#
+# This version is based on cffi, and is a translation of _tkinter.c
+# from CPython, version 2.7.4.
+
+class TclError(Exception):
+    pass
+
+import cffi
+try:
+    from .tklib import tklib, tkffi
+except cffi.VerificationError:
+    raise ImportError("Tk headers and development libraries are required")
+
+from .app import TkApp
+
+TK_VERSION = tkffi.string(tklib.get_tk_version())
+TCL_VERSION = tkffi.string(tklib.get_tcl_version())
+
+READABLE = tklib.TCL_READABLE
+WRITABLE = tklib.TCL_WRITABLE
+EXCEPTION = tklib.TCL_EXCEPTION
+
+def create(screenName=None, baseName=None, className=None,
+           interactive=False, wantobjects=False, wantTk=True,
+           sync=False, use=None):
+    return TkApp(screenName, baseName, className,
+                 interactive, wantobjects, wantTk, sync, use)
+
+def _flatten(item):
+    def _flatten1(output, item, depth):
+        if depth > 1000:
+            raise ValueError("nesting too deep in _flatten")
+        if not isinstance(item, (list, tuple)):
+            raise TypeError("argument must be sequence")
+        # copy items to output tuple
+        for o in item:
+            if isinstance(o, (list, tuple)):
+                _flatten1(output, o, depth + 1)
+            elif o is not None:
+                output.append(o)
+
+    result = []
+    _flatten1(result, item, 0)
+    return tuple(result)
+    
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/app.py
@@ -0,0 +1,389 @@
+# The TkApp class.
+
+from .tklib import tklib, tkffi
+from . import TclError
+from .tclobj import TclObject, FromObj, AsObj, TypeCache
+
+import sys
+
+def varname_converter(input):
+    if isinstance(input, TclObject):
+        return input.string
+    return input
+
+
+def Tcl_AppInit(app):
+    if tklib.Tcl_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+    skip_tk_init = tklib.Tcl_GetVar(
+        app.interp, "_tkinter_skip_tk_init", tklib.TCL_GLOBAL_ONLY)
+    if skip_tk_init and tkffi.string(skip_tk_init) == "1":
+        return
+
+    if tklib.Tk_Init(app.interp) == tklib.TCL_ERROR:
+        app.raiseTclError()
+
+class _CommandData(object):
+    def __new__(cls, app, name, func):
+        self = object.__new__(cls)
+        self.app = app
+        self.name = name
+        self.func = func
+        handle = tkffi.new_handle(self)
+        app._commands[name] = handle  # To keep the command alive
+        return tkffi.cast("ClientData", handle)
+
+    @tkffi.callback("Tcl_CmdProc")
+    def PythonCmd(clientData, interp, argc, argv):
+        self = tkffi.from_handle(clientData)
+        assert self.app.interp == interp
+        try:
+            args = [tkffi.string(arg) for arg in argv[1:argc]]
+            result = self.func(*args)
+            obj = AsObj(result)
+            tklib.Tcl_SetObjResult(interp, obj)
+        except:
+            self.app.errorInCmd = True
+            self.app.exc_info = sys.exc_info()
+            return tklib.TCL_ERROR
+        else:
+            return tklib.TCL_OK
+
+    @tkffi.callback("Tcl_CmdDeleteProc")
+    def PythonCmdDelete(clientData):
+        self = tkffi.from_handle(clientData)
+        app = self.app
+        del app._commands[self.name]
+        return
+
+
+class TkApp(object):
+    def __new__(cls, screenName, baseName, className,
+                interactive, wantobjects, wantTk, sync, use):
+        if not wantobjects:
+            raise NotImplementedError("wantobjects=True only")
+        self = object.__new__(cls)
+        self.interp = tklib.Tcl_CreateInterp()
+        self._wantobjects = wantobjects
+        self.threaded = bool(tklib.Tcl_GetVar2Ex(
+            self.interp, "tcl_platform", "threaded",
+            tklib.TCL_GLOBAL_ONLY))
+        self.thread_id = tklib.Tcl_GetCurrentThread()
+        self.dispatching = False
+        self.quitMainLoop = False
+        self.errorInCmd = False
+
+        self._typeCache = TypeCache()
+        self._commands = {}
+
+        # Delete the 'exit' command, which can screw things up
+        tklib.Tcl_DeleteCommand(self.interp, "exit")
+
+        if screenName is not None:
+            tklib.Tcl_SetVar2(self.interp, "env", "DISPLAY", screenName,
+                              tklib.TCL_GLOBAL_ONLY)
+
+        if interactive:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+        else:
+            tklib.Tcl_SetVar(self.interp, "tcl_interactive", "0",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # This is used to get the application class for Tk 4.1 and up
+        argv0 = className.lower()
+        tklib.Tcl_SetVar(self.interp, "argv0", argv0,
+                         tklib.TCL_GLOBAL_ONLY)
+
+        if not wantTk:
+            tklib.Tcl_SetVar(self.interp, "_tkinter_skip_tk_init", "1",
+                             tklib.TCL_GLOBAL_ONLY)
+
+        # some initial arguments need to be in argv
+        if sync or use:
+            args = ""
+            if sync:
+                args += "-sync"
+            if use:
+                if sync:
+                    args += " "
+                args += "-use " + use
+
+            tklib.Tcl_SetVar(self.interp, "argv", args,
+                             tklib.TCL_GLOBAL_ONLY)
+
+        Tcl_AppInit(self)
+        # EnableEventHook()
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DeleteInterp(self.interp)
+        # DisableEventHook()
+
+    def raiseTclError(self):
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+        raise TclError(tkffi.string(tklib.Tcl_GetStringResult(self.interp)))
+
+    def wantobjects(self):
+        return self._wantobjects
+
+    def _check_tcl_appartment(self):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise RuntimeError("Calling Tcl from different appartment")
+
+    def loadtk(self):
+        # We want to guard against calling Tk_Init() multiple times
+        err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
+        if err == tklib.TCL_ERROR:
+            self.raiseTclError()
+        tk_exists = tklib.Tcl_GetStringResult(self.interp)
+        if not tk_exists or tkffi.string(tk_exists) != "1":
+            err = tklib.Tk_Init(self.interp)
+            if err == tklib.TCL_ERROR:
+                self.raiseTclError()
+
+    def _var_invoke(self, func, *args, **kwargs):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # The current thread is not the interpreter thread.
+            # Marshal the call to the interpreter thread, then wait
+            # for completion.
+            raise NotImplementedError("Call from another thread")
+        return func(*args, **kwargs)
+
+    def _getvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+        if not res:
+            self.raiseTclError()
+        assert self._wantobjects
+        return FromObj(self, res)
+
+    def _setvar(self, name1, value, global_only=False):
+        name1 = varname_converter(name1)
+        newval = AsObj(value)
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                  newval, flags)
+        if not res:
+            self.raiseTclError()
+
+    def _unsetvar(self, name1, name2=None, global_only=False):
+        name1 = varname_converter(name1)
+        if not name2:
+            name2 = tkffi.NULL
+        flags=tklib.TCL_LEAVE_ERR_MSG
+        if global_only:
+            flags |= tklib.TCL_GLOBAL_ONLY
+        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def getvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2)
+
+    def globalgetvar(self, name1, name2=None):
+        return self._var_invoke(self._getvar, name1, name2, global_only=True)
+
+    def setvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value)
+
+    def globalsetvar(self, name1, value):
+        return self._var_invoke(self._setvar, name1, value, global_only=True)
+
+    def unsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2)
+
+    def globalunsetvar(self, name1, name2=None):
+        return self._var_invoke(self._unsetvar, name1, name2, global_only=True)
+
+    # COMMANDS
+
+    def createcommand(self, cmdName, func):
+        if not callable(func):
+            raise TypeError("command not callable")
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        clientData = _CommandData(self, cmdName, func)
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_CreateCommand(
+            self.interp, cmdName, _CommandData.PythonCmd,
+            clientData, _CommandData.PythonCmdDelete)
+        if not res:
+            raise TclError("can't create Tcl command")
+
+    def deletecommand(self, cmdName):
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            raise NotImplementedError("Call from another thread")
+
+        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        if res == -1:
+            raise TclError("can't delete Tcl command")
+
+    def call(self, *args):
+        flags = tklib.TCL_EVAL_DIRECT | tklib.TCL_EVAL_GLOBAL
+
+        # If args is a single tuple, replace with contents of tuple
+        if len(args) == 1 and isinstance(args[0], tuple):
+            args = args[0]
+
+        if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
+            # We cannot call the command directly. Instead, we must
+            # marshal the parameters to the interpreter thread.
+            raise NotImplementedError("Call from another thread")
+
+        objects = tkffi.new("Tcl_Obj*[]", len(args))
+        argc = len(args)
+        try:
+            for i, arg in enumerate(args):
+                if arg is None:
+                    argc = i
+                    break
+                obj = AsObj(arg)
+                tklib.Tcl_IncrRefCount(obj)
+                objects[i] = obj
+
+            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            else:
+                result = self._callResult()
+        finally:
+            for obj in objects:
+                if obj:
+                    tklib.Tcl_DecrRefCount(obj)
+        return result
+
+    def _callResult(self):
+        assert self._wantobjects
+        value = tklib.Tcl_GetObjResult(self.interp)
+        # Not sure whether the IncrRef is necessary, but something
+        # may overwrite the interpreter result while we are
+        # converting it.
+        tklib.Tcl_IncrRefCount(value)
+        res = FromObj(self, value)
+        tklib.Tcl_DecrRefCount(value)
+        return res
+
+    def eval(self, script):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_Eval(self.interp, script)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def evalfile(self, filename):
+        self._check_tcl_appartment()
+        res = tklib.Tcl_EvalFile(self.interp, filename)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+
+    def split(self, arg):
+        if isinstance(arg, tuple):
+            return self._splitObj(arg)
+        else:
+            return self._split(arg)
+
+    def splitlist(self, arg):
+        if isinstance(arg, tuple):
+            return arg
+        if isinstance(arg, unicode):
+            arg = arg.encode('utf8')
+
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(self.interp, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+        result = tuple(tkffi.string(argv[0][i])
+                       for i in range(argc[0]))
+        tklib.Tcl_Free(argv[0])
+        return result
+
+    def _splitObj(self, arg):
+        if isinstance(arg, tuple):
+            size = len(arg)
+            # Recursively invoke SplitObj for all tuple items.
+            # If this does not return a new object, no action is
+            # needed.
+            result = None
+            newelems = (self._splitObj(elem) for elem in arg)
+            for elem, newelem in zip(arg, newelems):
+                if elem is not newelem:
+                    return newelems
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+            if res == tklib.TCL_ERROR:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(arg)
+        return arg
+
+    def _split(self, arg):
+        argc = tkffi.new("int*")
+        argv = tkffi.new("char***")
+        res = tklib.Tcl_SplitList(tkffi.NULL, arg, argc, argv)
+        if res == tklib.TCL_ERROR:
+            # Not a list.
+            # Could be a quoted string containing funnies, e.g. {"}.
+            # Return the string itself.
+            return arg
+
+        try:
+            if argc[0] == 0:
+                return ""
+            elif argc[0] == 1:
+                return argv[0][0]
+            else:
+                return (self._split(argv[0][i])
+                        for i in range(argc[0]))
+        finally:
+            tklib.Tcl_Free(argv[0])
+
+    def getboolean(self, s):
+        if isinstance(s, int):
+            return s
+        v = tkffi.new("int*")
+        res = tklib.Tcl_GetBoolean(self.interp, s, v)
+        if res == tklib.TCL_ERROR:
+            self.raiseTclError()
+
+    def mainloop(self, threshold):
+        self._check_tcl_appartment()
+        self.dispatching = True
+        while (tklib.Tk_GetNumMainWindows() > threshold and
+               not self.quitMainLoop and not self.errorInCmd):
+
+            if self.threaded:
+                result = tklib.Tcl_DoOneEvent(0)
+            else:
+                raise NotImplementedError("TCL configured without threads")
+
+            if result < 0:
+                break
+        self.dispatching = False
+        self.quitMainLoop = False
+        if self.errorInCmd:
+            self.errorInCmd = False
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
+    def quit(self):
+        self.quitMainLoop = True
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -0,0 +1,114 @@
+# TclObject, conversions with Python objects
+
+from .tklib import tklib, tkffi
+
+class TypeCache(object):
+    def __init__(self):
+        self.BooleanType = tklib.Tcl_GetObjType("boolean")
+        self.ByteArrayType = tklib.Tcl_GetObjType("bytearray")
+        self.DoubleType = tklib.Tcl_GetObjType("double")
+        self.IntType = tklib.Tcl_GetObjType("int")
+        self.ListType = tklib.Tcl_GetObjType("list")
+        self.ProcBodyType = tklib.Tcl_GetObjType("procbody")
+        self.StringType = tklib.Tcl_GetObjType("string")
+        
+
+def FromObj(app, value):
+    """Convert a TclObj pointer into a Python object."""
+    typeCache = app._typeCache
+    if not value.typePtr:
+        buf = tkffi.buffer(value.bytes, value.length)
+        result = buf[:]
+        # If the result contains any bytes with the top bit set, it's
+        # UTF-8 and we should decode it to Unicode.
+        try:
+            result.decode('ascii')
+        except UnicodeDecodeError:
+            result = result.decode('utf8')
+        return result
+
+    elif value.typePtr == typeCache.BooleanType:
+        return result
+    elif value.typePtr == typeCache.ByteArrayType:
+        return result
+    elif value.typePtr == typeCache.DoubleType:
+        return value.internalRep.doubleValue
+    elif value.typePtr == typeCache.IntType:
+        return value.internalRep.longValue
+    elif value.typePtr == typeCache.ListType:
+        size = tkffi.new('int*')
+        status = tklib.Tcl_ListObjLength(app.interp, value, size)
+        if status == tklib.TCL_ERROR:
+            app.raiseTclError()
+        result = []
+        tcl_elem = tkffi.new("Tcl_Obj**")
+        for i in range(size[0]):
+            status = tklib.Tcl_ListObjIndex(app.interp,
+                                            value, i, tcl_elem)
+            if status == tklib.TCL_ERROR:
+                app.raiseTclError()
+            result.append(FromObj(app, tcl_elem[0]))
+        return tuple(result)
+    elif value.typePtr == typeCache.ProcBodyType:
+        return result
+    elif value.typePtr == typeCache.StringType:
+        buf = tklib.Tcl_GetUnicode(value)
+        length = tklib.Tcl_GetCharLength(value)
+        buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
+        return buf.decode('utf-16')
+
+    return TclObject(value)
+
+def AsObj(value):
+    if isinstance(value, str):
+        return tklib.Tcl_NewStringObj(value, len(value))
+    elif isinstance(value, bool):
+        return tklib.Tcl_NewBooleanObj(value)
+    elif isinstance(value, int):
+        return tklib.Tcl_NewLongObj(value)
+    elif isinstance(value, float):
+        return tklib.Tcl_NewDoubleObj(value)
+    elif isinstance(value, tuple):
+        argv = tkffi.new("Tcl_Obj*[]", len(value))
+        for i in range(len(value)):
+            argv[i] = AsObj(value[i])
+        return tklib.Tcl_NewListObj(len(value), argv)
+    elif isinstance(value, unicode):
+        encoded = value.encode('utf-16')[2:]
+        buf = tkffi.new("char[]", encoded)
+        inbuf = tkffi.cast("Tcl_UniChar*", buf)
+        return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2)
+    elif isinstance(value, TclObject):
+        tklib.Tcl_IncrRefCount(value._value)
+        return value._value
+    else:
+        return AsObj(str(value))
+
+class TclObject(object):
+    def __new__(cls, value):
+        self = object.__new__(cls)
+        tklib.Tcl_IncrRefCount(value)
+        self._value = value
+        self._string = None
+        return self
+
+    def __del__(self):
+        tklib.Tcl_DecrRefCount(self._value)
+
+    def __str__(self):
+        if self._string and isinstance(self._string, str):
+            return self._string
+        return tkffi.string(tklib.Tcl_GetString(self._value))
+
+    @property
+    def string(self):
+        if self._string is None:
+            length = tkffi.new("int*")
+            s = tklib.Tcl_GetStringFromObj(self._value, length)
+            value = tkffi.buffer(s, length[0])[:]
+            try:
+                value.decode('ascii')
+            except UnicodeDecodeError:
+                value = value.decode('utf8')
+            self._string = value
+        return self._string
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_tkinter/tklib.py
@@ -0,0 +1,114 @@
+# C bindings with libtcl and libtk.
+
+from cffi import FFI
+
+tkffi = FFI()
+
+tkffi.cdef("""
+char *get_tk_version();
+char *get_tcl_version();
+#define TCL_READABLE ...
+#define TCL_WRITABLE ...
+#define TCL_EXCEPTION ...
+#define TCL_ERROR ...
+#define TCL_OK ...
+
+#define TCL_LEAVE_ERR_MSG ...
+#define TCL_GLOBAL_ONLY ...
+#define TCL_EVAL_DIRECT ...
+#define TCL_EVAL_GLOBAL ...
+
+typedef unsigned short Tcl_UniChar;
+typedef ... Tcl_Interp;
+typedef ...* Tcl_ThreadId;
+typedef ...* Tcl_Command;
+
+typedef struct Tcl_ObjType {
+    char *name;
+    ...;
+} Tcl_ObjType;
+typedef struct Tcl_Obj {
+    char *bytes;
+    int length;
+    Tcl_ObjType *typePtr;
+    union {                     /* The internal representation: */
+        long longValue;         /*   - an long integer value. */
+        double doubleValue;     /*   - a double-precision floating value. */
+        struct {                /*   - internal rep as two pointers. */
+            void *ptr1;
+            void *ptr2;
+        } twoPtrValue;
+    } internalRep;
+    ...;
+} Tcl_Obj;
+
+Tcl_Interp *Tcl_CreateInterp();
+void Tcl_DeleteInterp(Tcl_Interp* interp);
+int Tcl_Init(Tcl_Interp* interp);
+int Tk_Init(Tcl_Interp* interp);
+
+void Tcl_Free(char* ptr);
+
+const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags);
+const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags);
+const char *Tcl_GetVar(Tcl_Interp* interp, const char* varName, int flags);
+Tcl_Obj *Tcl_SetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, Tcl_Obj* newValuePtr, int flags);
+Tcl_Obj *Tcl_GetVar2Ex(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+int Tcl_UnsetVar2(Tcl_Interp* interp, const char* name1, const char* name2, int flags);
+const Tcl_ObjType *Tcl_GetObjType(const char* typeName);
+
+Tcl_Obj *Tcl_NewStringObj(const char* bytes, int length);
+Tcl_Obj *Tcl_NewUnicodeObj(const Tcl_UniChar* unicode, int numChars);
+Tcl_Obj *Tcl_NewLongObj(long longValue);
+Tcl_Obj *Tcl_NewBooleanObj(int boolValue);
+Tcl_Obj *Tcl_NewDoubleObj(double doubleValue);
+
+void Tcl_IncrRefCount(Tcl_Obj* objPtr);
+void Tcl_DecrRefCount(Tcl_Obj* objPtr);
+
+int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
+char *Tcl_GetString(Tcl_Obj* objPtr);
+char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+
+Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
+int Tcl_GetCharLength(Tcl_Obj* objPtr);
+
+Tcl_Obj *Tcl_NewListObj(int objc, Tcl_Obj* const objv[]);
+int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
+int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
+int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+
+int Tcl_Eval(Tcl_Interp* interp, const char* script);
+int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);
+int Tcl_EvalObjv(Tcl_Interp* interp, int objc, Tcl_Obj** objv, int flags);
+Tcl_Obj *Tcl_GetObjResult(Tcl_Interp* interp);
+const char *Tcl_GetStringResult(Tcl_Interp* interp);
+void Tcl_SetObjResult(Tcl_Interp* interp, Tcl_Obj* objPtr);
+
+typedef void* ClientData;
+typedef int Tcl_CmdProc(
+        ClientData clientData,
+        Tcl_Interp *interp,
+        int argc,
+        const char *argv[]);
+typedef void Tcl_CmdDeleteProc(
+        ClientData clientData);
+Tcl_Command Tcl_CreateCommand(Tcl_Interp* interp, const char* cmdName, Tcl_CmdProc proc, ClientData clientData, Tcl_CmdDeleteProc deleteProc);
+int Tcl_DeleteCommand(Tcl_Interp* interp, const char* cmdName);
+
+Tcl_ThreadId Tcl_GetCurrentThread();
+int Tcl_DoOneEvent(int flags);
+
+int Tk_GetNumMainWindows();
+""")
+
+tklib = tkffi.verify("""
+#include <tcl.h>
+#include <tk.h>
+
+char *get_tk_version() { return TK_VERSION; }
+char *get_tcl_version() { return TCL_VERSION; }
+""",
+include_dirs=['/usr/include/tcl'],
+libraries=['tcl', 'tk'],
+)
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.6"
-__version_info__ = (0, 6)
+__version__ = "0.7"
+__version_info__ = (0, 7)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -73,15 +73,15 @@
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        BVoidP = self._get_cached_btype(model.voidp_type)
+        self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(BVoidP, 0)
+                FFI.NULL = self.cast(self.BVoidP, 0)
                 FFI.CData, FFI.CType = backend._get_types()
         else:
             # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(BVoidP, 0)
+            self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False):
@@ -346,6 +346,12 @@
         self._cdefsources.extend(ffi_to_include._cdefsources)
         self._cdefsources.append(']')
 
+    def new_handle(self, x):
+        return self._backend.newp_handle(self.BVoidP, x)
+
+    def from_handle(self, x):
+        return self._backend.from_handle(x)
+
 
 def _make_ffi_library(ffi, libname, flags):
     import os
@@ -355,13 +361,13 @@
     backend = ffi._backend
     try:
         if '.' not in name and '/' not in name:
-            raise OSError
+            raise OSError("library not found: %r" % (name,))
         backendlib = backend.load_library(name, flags)
     except OSError:
         import ctypes.util
         path = ctypes.util.find_library(name)
         if path is None:
-            raise OSError("library not found: %r" % (name,))
+            raise     # propagate the original OSError
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
@@ -370,7 +376,10 @@
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
             BType = ffi._get_cached_btype(tp)
-            value = backendlib.load_function(BType, name)
+            try:
+                value = backendlib.load_function(BType, name)
+            except KeyError as e:
+                raise AttributeError('%s: %s' % (name, e))
             library.__dict__[name] = value
             return
         #
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -16,6 +16,7 @@
 class CTypesData(object):
     __metaclass__ = CTypesType
     __slots__ = ['__weakref__']
+    __name__ = '<cdata>'
 
     def __init__(self, *args):
         raise TypeError("cannot instantiate %r" % (self.__class__,))
@@ -491,6 +492,8 @@
         elif BItem in (getbtype(model.PrimitiveType('signed char')),
                        getbtype(model.PrimitiveType('unsigned char'))):
             kind = 'bytep'
+        elif BItem is getbtype(model.void_type):
+            kind = 'voidp'
         else:
             kind = 'generic'
         #
@@ -546,13 +549,13 @@
             def __setitem__(self, index, value):
                 self._as_ctype_ptr[index] = BItem._to_ctypes(value)
 
-            if kind == 'charp':
+            if kind == 'charp' or kind == 'voidp':
                 @classmethod
-                def _arg_to_ctypes(cls, value):
-                    if isinstance(value, bytes):
-                        return ctypes.c_char_p(value)
+                def _arg_to_ctypes(cls, *value):
+                    if value and isinstance(value[0], bytes):
+                        return ctypes.c_char_p(value[0])
                     else:
-                        return super(CTypesPtr, cls)._arg_to_ctypes(value)
+                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
 
             if kind == 'charp' or kind == 'bytep':
                 def _to_string(self, maxlen):
@@ -704,7 +707,7 @@
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
-            _reftypename = '%s %s &' % (kind, name)
+            _reftypename = '%s &' % (name,)
             _kind = kind
         #
         CTypesStructOrUnion._fix_class()
@@ -931,7 +934,7 @@
         #
         class CTypesEnum(CTypesInt):
             __slots__ = []
-            _reftypename = 'enum %s &' % name
+            _reftypename = '%s &' % name
 
             def _get_own_repr(self):
                 value = self._value
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -244,6 +244,10 @@
         self.forcename = forcename
         self.build_c_name_with_marker()
 
+    def get_official_name(self):
+        assert self.c_name_with_marker.endswith('&')
+        return self.c_name_with_marker[:-1]
+
 
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
@@ -357,7 +361,9 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_struct_type', self.name, key=self)
+        
+        return global_cache(self, ffi, 'new_struct_type',
+                            self.get_official_name(), key=self)
 
 
 class UnionType(StructOrUnion):
@@ -365,7 +371,8 @@
 
     def build_backend_type(self, ffi, finishlist):
         finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type', self.name, key=self)
+        return global_cache(self, ffi, 'new_union_type',
+                            self.get_official_name(), key=self)
 
 
 class EnumType(StructOrUnionOrEnum):
@@ -388,7 +395,8 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         base_btype = self.build_baseinttype(ffi, finishlist)
-        return global_cache(self, ffi, 'new_enum_type', self.name,
+        return global_cache(self, ffi, 'new_enum_type',
+                            self.get_official_name(),
                             self.enumerators, self.enumvalues,
                             base_btype, key=self)
 
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -15,6 +15,20 @@
     def patch_extension_kwds(self, kwds):
         pass
 
+    def find_module(self, module_name, path, so_suffix):
+        try:
+            f, filename, descr = imp.find_module(module_name, path)
+        except ImportError:
+            return None
+        if f is not None:
+            f.close()
+        # Note that after a setuptools installation, there are both .py
+        # and .so files with the same basename.  The code here relies on
+        # imp.find_module() locating the .so in priority.
+        if descr[0] != so_suffix:
+            return None
+        return filename
+
     def collect_types(self):
         self._typesdict = {}
         self._generate("collecttype")
@@ -142,6 +156,9 @@
         class FFILibrary(object):
             _cffi_python_module = module
             _cffi_ffi = self.ffi
+            _cffi_dir = []
+            def __dir__(self):
+                return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
         module._cffi_setup(lst, ffiplatform.VerificationError, library)
         #
@@ -427,9 +444,9 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
-        for fname, ftype, _ in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():

From noreply at buildbot.pypy.org  Sat Jun 29 22:19:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 22:19:26 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Fix this,
	lost because of a mistake during merge
Message-ID: <20130629201926.1D1351C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65108:96da7cbe93df
Date: 2013-06-29 22:18 +0200
http://bitbucket.org/pypy/pypy/changeset/96da7cbe93df/

Log:	Fix this, lost because of a mistake during merge

diff --git a/rpython/translator/c/src/g_prerequisite.h b/rpython/translator/c/src/g_prerequisite.h
--- a/rpython/translator/c/src/g_prerequisite.h
+++ b/rpython/translator/c/src/g_prerequisite.h
@@ -12,4 +12,15 @@
 #include <stddef.h>
 
 
+#ifdef __GNUC__
+# define RPY_VARLENGTH   /* nothing: [RPY_VARLENGTH] => [] */
+# define RPY_LENGTH0     0       /* array decl [0] are ok  */
+# define RPY_DUMMY_VARLENGTH     char _dummy[0];
+#else
+# define RPY_VARLENGTH   1       /* [RPY_VARLENGTH] => [1] */
+# define RPY_LENGTH0     1       /* array decl [0] are bad */
+# define RPY_DUMMY_VARLENGTH     /* nothing */
+#endif
+
+
 #include "src/align.h"

From noreply at buildbot.pypy.org  Sat Jun 29 23:01:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 23:01:27 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Implement stmcb_size().
Message-ID: <20130629210127.31CE61C06B1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65109:d39a80a156da
Date: 2013-06-29 23:00 +0200
http://bitbucket.org/pypy/pypy/changeset/d39a80a156da/

Log:	Implement stmcb_size().

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -23,20 +23,6 @@
 
     TRANSLATION_PARAMS = {}
 
-    def __init__(self, config, **kwds):
-        MovingGCBase.__init__(self, config, **kwds)
-
-        def stmcb_size(obj):
-            typeid = self.get_type_id(obj)
-            size = self.fixed_size(typeid)
-            if self.is_varsize(typeid):
-                lenofs = self.varsize_offset_to_length(typeid)
-                length = (obj + lenofs).signed[0]
-                size += self.varsize_item_sizes(typeid) * length
-            return size
-
-        
-
     def get_type_id(self, obj):
         return llop.stm_get_tid(llgroup.HALFWORD, obj)
 
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -219,6 +219,7 @@
             return annhelper.graph2const(graph)
         self._getfn = getfn
 
+        self.autoregister_ptrs = []
         self.frameworkgc_setup_ptr = getfn(frameworkgc_setup, [],
                                            annmodel.s_None)
         # for tests
diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -4,6 +4,20 @@
 
 class StmFrameworkGCTransformer(BaseFrameworkGCTransformer):
 
+    def _declare_functions(self, GCClass, getfn, s_gc, s_typeid16):
+        from rpython.annotator import model as annmodel
+        from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+        BaseFrameworkGCTransformer._declare_functions(self, GCClass, getfn,
+                                                      s_gc, s_typeid16)
+        gc = self.gcdata.gc
+        #
+        def pypy_stmcb_size(obj):
+            return gc.get_size(obj)
+        pypy_stmcb_size.c_name = "pypy_stmcb_size"
+        self.autoregister_ptrs.append(
+            getfn(pypy_stmcb_size, [annmodel.SomeAddress()],
+                  annmodel.SomeInteger()))
+
     def build_root_walker(self):
         return StmRootWalker(self)
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -590,6 +590,7 @@
     OP_STM_PUSH_ROOT = _OP_STM
     OP_STM_POP_ROOT_INTO = _OP_STM
     OP_STM_ALLOCATE = _OP_STM
+    OP_STM_GET_TID = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -345,6 +345,8 @@
         return FrameworkGcRuntimeTypeInfo_OpaqueNode
 
     def gc_startup_code(self):
+        for c_fnptr in self.db.gctransformer.autoregister_ptrs:
+            self.db.get(c_fnptr.value)
         fnptr = self.db.gctransformer.frameworkgc_setup_ptr.value
         yield '%s();' % (self.db.get(fnptr),)
 
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -85,6 +85,11 @@
     result = funcgen.expr(op.result)
     return '%s = stm_allocate(%s, %s);' % (result, arg0, arg1)
 
+def stm_get_tid(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    result = funcgen.expr(op.result)
+    return '%s = stm_get_tid((gcptr)%s);' % (result, arg0)
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -6,6 +6,14 @@
 cdir = os.path.abspath(os.path.join(cdir2, '..', 'stm'))
 
 separate_source = '''
+#include "src_stm/stmgc.h"
+
+extern Signed pypy_stmcb_size(void*);
+
+inline size_t stmcb_size(gcptr obj) {
+    return pypy_stmcb_size(obj);
+}
+
 #include "src_stm/stmgc.c"
 '''
 

From noreply at buildbot.pypy.org  Sat Jun 29 23:10:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 23:10:40 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: stmcb_trace().
Message-ID: <20130629211040.853AC1C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65110:c6848441ba04
Date: 2013-06-29 23:10 +0200
http://bitbucket.org/pypy/pypy/changeset/c6848441ba04/

Log:	stmcb_trace().

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -21,6 +21,8 @@
     HDR = rffi.COpaque('struct stm_object_s')
     typeid_is_in_field = None
 
+    VISIT_FPTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
+
     TRANSLATION_PARAMS = {}
 
     def get_type_id(self, obj):
diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -17,6 +17,16 @@
         self.autoregister_ptrs.append(
             getfn(pypy_stmcb_size, [annmodel.SomeAddress()],
                   annmodel.SomeInteger()))
+        #
+        def invokecallback(root, visit_fn):
+            visit_fn(root)
+        def pypy_stmcb_trace(obj, visit_fn):
+            gc.trace(obj, invokecallback, visit_fn)
+        pypy_stmcb_trace.c_name = "pypy_stmcb_trace"
+        self.autoregister_ptrs.append(
+            getfn(pypy_stmcb_trace, [annmodel.SomeAddress(),
+                                     annmodel.SomePtr(GCClass.VISIT_FPTR)],
+                  annmodel.s_None))
 
     def build_root_walker(self):
         return StmRootWalker(self)
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -9,11 +9,16 @@
 #include "src_stm/stmgc.h"
 
 extern Signed pypy_stmcb_size(void*);
+extern void pypy_stmcb_trace(void*, void(*)(void*));
 
 inline size_t stmcb_size(gcptr obj) {
     return pypy_stmcb_size(obj);
 }
 
+inline void stmcb_trace(gcptr obj, void visit(gcptr *)) {
+    pypy_stmcb_trace(obj, (void(*)(void*))visit);
+}
+
 #include "src_stm/stmgc.c"
 '''
 

From noreply at buildbot.pypy.org  Sat Jun 29 23:19:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 23:19:58 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: stm_initialize()
Message-ID: <20130629211958.9B0E61C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65111:3451bcd67240
Date: 2013-06-29 23:17 +0200
http://bitbucket.org/pypy/pypy/changeset/3451bcd67240/

Log:	stm_initialize()

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -3,7 +3,7 @@
 from rpython/translator/stm/src_stm/.
 """
 
-from rpython.memory.gc.base import MovingGCBase
+from rpython.memory.gc.base import GCBase, MovingGCBase
 from rpython.rtyper.lltypesystem import lltype, llmemory, llgroup, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
@@ -25,6 +25,14 @@
 
     TRANSLATION_PARAMS = {}
 
+    def setup(self):
+        # Hack: MovingGCBase.setup() sets up stuff related to id(), which
+        # we implement differently anyway.  So directly call GCBase.setup().
+        GCBase.setup(self)
+        #
+        llop.stm_initialize(lltype.Void)
+
+
     def get_type_id(self, obj):
         return llop.stm_get_tid(llgroup.HALFWORD, obj)
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -421,6 +421,7 @@
     # direct_calls and maybe several casts, but it looks less heavy-weight
     # to keep them as operations until the genc stage)
 
+    'stm_initialize':         LLOp(),
     'stm_barrier':            LLOp(sideeffects=False),
     'stm_allocate':           LLOp(sideeffects=False),
     'stm_become_inevitable':  LLOp(),
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -584,6 +584,7 @@
             from rpython.translator.stm.funcgen import op_stm
             self.__class__.op_stm = op_stm
         return self.op_stm(op)
+    OP_STM_INITIALIZE = _OP_STM
     OP_STM_BECOME_INEVITABLE = _OP_STM
     OP_STM_BARRIER = _OP_STM
     OP_STM_PTR_EQ = _OP_STM
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -22,19 +22,8 @@
             self.obj.prebuilt_hash)
 
 
-def stm_start_transaction(funcgen, op):
-    xxx
-    # only for testing.  With stmgc, this operation should have been handled
-    # already by gctransform.
-    assert funcgen.db.translator.config.translation.gc == 'none'
-    return 'stm_nogc_start_transaction();'
-
-def stm_stop_transaction(funcgen, op):
-    xxx
-    # only for testing.  With stmgc, this operation should have been handled
-    # already by gctransform.
-    assert funcgen.db.translator.config.translation.gc == 'none'
-    return 'stm_nogc_stop_transaction();'
+def stm_initialize(funcgen, op):
+    return 'stm_initialize();'
 
 _STM_BARRIER_FUNCS = {   # XXX try to see if some combinations can be shorter
     'P2R': 'stm_read_barrier',
diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py
--- a/rpython/translator/stm/stmgcintf.py
+++ b/rpython/translator/stm/stmgcintf.py
@@ -6,6 +6,8 @@
 cdir = os.path.abspath(os.path.join(cdir2, '..', 'stm'))
 
 separate_source = '''
+#define _GC_DEBUG   2       /* XXX move elsewhere */
+
 #include "src_stm/stmgc.h"
 
 extern Signed pypy_stmcb_size(void*);

From noreply at buildbot.pypy.org  Sat Jun 29 23:19:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Jun 2013 23:19:59 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: First first part of this test passes!
 :-) probably because it's small
Message-ID: <20130629211959.D83271C0255@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65112:b1bac73d4f54
Date: 2013-06-29 23:19 +0200
http://bitbucket.org/pypy/pypy/changeset/b1bac73d4f54/

Log:	First first part of this test passes! :-) probably because it's
	small enough to avoid any minor collection.

diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -20,6 +20,8 @@
             return 0
         #
         t, cbuilder = self.compile(entry_point, backendopt=True)
+        data = cbuilder.cmdexec('5')
+        assert '< 5 >' in data, "got: %r" % (data,)
         data = cbuilder.cmdexec('42')
         assert '< 42 >' in data, "got: %r" % (data,)
 

From noreply at buildbot.pypy.org  Sat Jun 29 23:56:18 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 29 Jun 2013 23:56:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
	fix missing method in ScalarIterator
Message-ID: <20130629215618.4344C1C0255@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65113:9918529ceecd
Date: 2013-06-30 00:55 +0300
http://bitbucket.org/pypy/pypy/changeset/9918529ceecd/

Log:	test, fix missing method in ScalarIterator

diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -13,6 +13,9 @@
     def next(self):
         self.called_once = True
 
+    def next_skip_x(self, n):
+        self.called_once = True
+
     def getitem(self):
         return self.v.get_scalar_value()
 
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -37,7 +37,7 @@
 we can go faster.
 All the calculations happen in next()
 
-next_skip_x() tries to do the iteration for a number of steps at once,
+next_skip_x(steps) tries to do the iteration for a number of steps at once,
 but then we cannot gaurentee that we only overflow one single shape
 dimension, perhaps we could overflow times in one big step.
 """
diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py
--- a/pypy/module/micronumpy/test/test_iter.py
+++ b/pypy/module/micronumpy/test/test_iter.py
@@ -1,4 +1,5 @@
 from pypy.module.micronumpy.iter import MultiDimViewIterator
+from pypy.module.micronumpy.arrayimpl.scalar import ScalarIterator
 
 class MockArray(object):
     size = 1
@@ -8,7 +9,7 @@
         #Let's get started, simple iteration in C order with
         #contiguous layout => strides[-1] is 1
         start = 0
-        shape = [3, 5] 
+        shape = [3, 5]
         strides = [5, 1]
         backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
         assert backstrides == [10, 4]
@@ -47,7 +48,7 @@
         #iteration in C order with #contiguous layout => strides[-1] is 1
         #skip less than the shape
         start = 0
-        shape = [3, 5] 
+        shape = [3, 5]
         strides = [5, 1]
         backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
         assert backstrides == [10, 4]
@@ -89,3 +90,9 @@
         assert i.indexes == [0,1]
         assert i.offset == 3
         assert i.done()
+
+    def test_scalar_iter(self):
+        i = ScalarIterator(MockArray)
+        i.next()
+        i.next_skip_x(3)
+        assert i.done()

From noreply at buildbot.pypy.org  Sun Jun 30 01:25:29 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun, 30 Jun 2013 01:25:29 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: the 'fast' version of float parsing is
 not much faster than the other, just kill the additional complexity
Message-ID: <20130629232529.AB5C51C0255@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65114:d431e45e8e2e
Date: 2013-06-29 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/d431e45e8e2e/

Log:	the 'fast' version of float parsing is not much faster than the
	other, just kill the additional complexity

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -48,7 +48,7 @@
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 class JSONDecoder(object):
-    def __init__(self, space, s, precise_float=True):
+    def __init__(self, space, s):
         self.space = space
         self.s = s
         # we put our string in a raw buffer so:
@@ -59,7 +59,6 @@
         self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
-        self.precise_float = precise_float
 
     def close(self):
         rffi.free_charp(self.ll_chars)
@@ -138,21 +137,16 @@
         if ch == '.':
             if not self.ll_chars[i+1].isdigit():
                 self._raise("Expected digit at char %d", i+1)
-            return self.parse_float(start, i, intval, ovf_maybe)
+            return self.decode_float(start)
         elif ch == 'e' or ch == 'E':
-            return self.parse_float(start, i, intval, ovf_maybe)
+            return self.decode_float(start)
         elif ovf_maybe:
             return self.decode_int_slow(start)
 
         self.pos = i
         return self.space.wrap(intval)
 
-    def parse_float(self, start, i, intval, ovf_maybe):
-        if self.precise_float or ovf_maybe:
-            return self.parse_float_precise(start)
-        return self.parse_float_fast(i, intval)
-
-    def parse_float_precise(self, i):
+    def decode_float(self, i):
         from rpython.rlib import rdtoa
         start = rffi.ptradd(self.ll_chars, i)
         floatval = rdtoa.dg_strtod(start, self.end_ptr)
@@ -160,27 +154,6 @@
         self.pos = i + diff
         return self.space.wrap(floatval)
 
-    def parse_float_fast(self, i, intval):
-        exp = 0
-        frcval = 0
-        ch = self.ll_chars[i]
-        if ch == '.':
-            i, frcval, frccount = self.parse_digits(i+1)
-            frcval = neg_pow_10(frcval, frccount)
-            ch = self.ll_chars[i]
-        # check for the optional exponent part
-        if ch == 'E' or ch == 'e':
-            i, ovf_maybe, exp = self.parse_integer(i+1, allow_leading_0=True)
-        #
-        floatval = float(intval) + frcval
-        if exp != 0:
-            try:
-                floatval = floatval * math.pow(10, exp)
-            except OverflowError:
-                floatval = rfloat.INFINITY
-        self.pos = i
-        return self.space.wrap(floatval)
-
     def decode_int_slow(self, i):
         start = i
         if self.ll_chars[i] == '-':
@@ -390,13 +363,12 @@
         lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
         return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
 
- at unwrap_spec(precise_float=bool)
-def loads(space, w_s, precise_float=True):
+def loads(space, w_s):
     if space.isinstance_w(w_s, space.w_unicode):
         raise OperationError(space.w_TypeError,
                              space.wrap("Expected utf8-encoded str, got unicode"))
     s = space.str_w(w_s)
-    decoder = JSONDecoder(space, s, precise_float)
+    decoder = JSONDecoder(space, s)
     try:
         w_res = decoder.decode_any(0)
         i = decoder.skip_whitespace(decoder.pos)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -106,10 +106,6 @@
             res = _fastjson.loads(s)
             assert type(res) is type(val)
             assert res == val
-            #
-            res = _fastjson.loads(s, precise_float=False)
-            assert type(res) is type(val)
-            assert res == val
         #
         check('42', 42)
         check('-42', -42)
@@ -140,7 +136,6 @@
         import _fastjson
         def error(s):
             raises(ValueError, _fastjson.loads, s)
-            raises(ValueError, _fastjson.loads, s, False)
         #
         error('  42   abc')
         error('.123')

From noreply at buildbot.pypy.org  Sun Jun 30 01:25:30 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun, 30 Jun 2013 01:25:30 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: inline parse_digits into parse_integer
Message-ID: <20130629232530.E2BE81C06B1@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65115:007efd4cf4be
Date: 2013-06-29 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/007efd4cf4be/

Log:	inline parse_digits into parse_integer

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -130,7 +130,7 @@
 
     def decode_numeric(self, i):
         start = i
-        i, ovf_maybe, intval = self.parse_integer(i, allow_leading_0=False)
+        i, ovf_maybe, intval = self.parse_integer(i)
         #
         # check for the optional fractional part
         ch = self.ll_chars[i]
@@ -164,29 +164,22 @@
         self.pos = i
         return self.space.call_function(self.space.w_int, self.space.wrap(s))
 
-    def parse_integer(self, i, allow_leading_0=False):
+    def parse_integer(self, i):
         "Parse a decimal number with an optional minus sign"
-        start = i
         sign = 1
+        # parse the sign
         if self.ll_chars[i] == '-':
             sign = -1
             i += 1
         elif self.ll_chars[i] == '+':
             i += 1
-        elif not allow_leading_0 and self.ll_chars[i] == '0':
+        #
+        if self.ll_chars[i] == '0':
             i += 1
             return i, False, 0
-        i, intval, _ = self.parse_digits(i)
-        # if the number has more digits than OVF_DIGITS, it might have
-        # overflowed
-        ovf_maybe = (i-start >= OVF_DIGITS)
-        return i, ovf_maybe, sign * intval
-    parse_integer._always_inline_ = True
 
-    def parse_digits(self, i):
-        "Parse a sequence of digits as a decimal number. No sign allowed"
         intval = 0
-        start = i 
+        start = i
         while True:
             ch = self.ll_chars[i]
             if ch.isdigit():
@@ -197,7 +190,11 @@
         count = i - start
         if count == 0:
             self._raise("Expected digit at char %d", i)
-        return i, intval, count
+        # if the number has more digits than OVF_DIGITS, it might have
+        # overflowed
+        ovf_maybe = (count >= OVF_DIGITS)
+        return i, ovf_maybe, sign * intval
+    parse_integer._always_inline_ = True
 
     def decode_array(self, i):
         w_list = self.space.newlist([])
@@ -226,7 +223,6 @@
                 self._raise("Unexpected '%s' when decoding array (char %d)",
                             ch, self.pos)
 
-
     def decode_object(self, i):
         start = i
         w_dict = self.space.newdict()

From noreply at buildbot.pypy.org  Sun Jun 30 01:25:32 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Sun, 30 Jun 2013 01:25:32 +0200 (CEST)
Subject: [pypy-commit] pypy fastjson: add support for non-standard constants
 Infinity, -Infinity and NaN
Message-ID: <20130629232532.15FC21C0255@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fastjson
Changeset: r65116:c64aefd4f08d
Date: 2013-06-30 01:24 +0200
http://bitbucket.org/pypy/pypy/changeset/c64aefd4f08d/

Log:	add support for non-standard constants Infinity, -Infinity and NaN

diff --git a/pypy/module/_fastjson/interp_decoder.py b/pypy/module/_fastjson/interp_decoder.py
--- a/pypy/module/_fastjson/interp_decoder.py
+++ b/pypy/module/_fastjson/interp_decoder.py
@@ -97,7 +97,15 @@
             return self.decode_true(i+1)
         elif ch == 'f':
             return self.decode_false(i+1)
-        elif ch.isdigit() or ch == '-':
+        elif ch == 'I':
+            return self.decode_infinity(i+1)
+        elif ch == 'N':
+            return self.decode_nan(i+1)
+        elif ch == '-':
+            if self.ll_chars[i+1] == 'I':
+                return self.decode_infinity(i+2, sign=-1)
+            return self.decode_numeric(i)
+        elif ch.isdigit():
             return self.decode_numeric(i)
         else:
             self._raise("No JSON object could be decoded: unexpected '%s' at char %d",
@@ -128,6 +136,25 @@
             return self.space.w_False
         self._raise("Error when decoding false at char %d", i)
 
+    def decode_infinity(self, i, sign=1):
+        if (self.ll_chars[i]   == 'n' and
+            self.ll_chars[i+1] == 'f' and
+            self.ll_chars[i+2] == 'i' and
+            self.ll_chars[i+3] == 'n' and
+            self.ll_chars[i+4] == 'i' and
+            self.ll_chars[i+5] == 't' and
+            self.ll_chars[i+6] == 'y'):
+            self.pos = i+7
+            return self.space.wrap(rfloat.INFINITY * sign)
+        self._raise("Error when decoding Infinity at char %d", i)
+
+    def decode_nan(self, i):
+        if (self.ll_chars[i]   == 'a' and
+            self.ll_chars[i+1] == 'N'):
+            self.pos = i+2
+            return self.space.wrap(rfloat.NAN)
+        self._raise("Error when decoding NaN at char %d", i)
+
     def decode_numeric(self, i):
         start = i
         i, ovf_maybe, intval = self.parse_integer(i)
diff --git a/pypy/module/_fastjson/test/test__fastjson.py b/pypy/module/_fastjson/test/test__fastjson.py
--- a/pypy/module/_fastjson/test/test__fastjson.py
+++ b/pypy/module/_fastjson/test/test__fastjson.py
@@ -98,7 +98,6 @@
         s = '"\xe0"' # this is an invalid UTF8 sequence inside a string
         raises(UnicodeDecodeError, "_fastjson.loads(s)")
 
-
     def test_decode_numeric(self):
         import sys
         import _fastjson
@@ -131,6 +130,15 @@
         check(x, float(x))
         #
         check('1E400', float('inf'))
+        ## # these are non-standard but supported by CPython json
+        check('Infinity', float('inf'))
+        check('-Infinity', float('-inf'))
+
+    def test_nan(self):
+        import math
+        import _fastjson
+        res = _fastjson.loads('NaN')
+        assert math.isnan(res)
 
     def test_decode_numeric_invalid(self):
         import _fastjson

From noreply at buildbot.pypy.org  Sun Jun 30 05:54:53 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 30 Jun 2013 05:54:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: These should be immutable!
Message-ID: <20130630035453.DCBB51C0255@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r65117:068f69b29ef0
Date: 2013-06-29 20:53 -0700
http://bitbucket.org/pypy/pypy/changeset/068f69b29ef0/

Log:	These should be immutable!

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -201,10 +201,27 @@
         def main(n):
             i = 0
             while i < n:
-                s = set([1,2,3])
+                s = set([1, 2, 3])
                 i += 1
         log = self.run(main, [1000])
         assert log.result == main(1000)
         loop, = log.loops_by_filename(self.filepath)
         opnames = log.opnames(loop.allops())
         assert opnames.count('new_with_vtable') == 0
+
+    def test_specialised_tuple(self):
+        def main(n):
+            import pypyjit
+
+            f = lambda: None
+            tup = (n, n)
+            while n > 0:
+                tup[0]  # ID: getitem
+                pypyjit.residual_call(f)
+                n -= 1
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        ops = loop.ops_by_id('getitem')
+        assert log.opnames(ops) == []
diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py
--- a/pypy/objspace/std/specialisedtupleobject.py
+++ b/pypy/objspace/std/specialisedtupleobject.py
@@ -18,6 +18,8 @@
     iter_n = unrolling_iterable(range(typelen))
 
     class cls(W_AbstractTupleObject):
+        _immutable_fields_ = ['value%s' % i for i in iter_n]
+
         def __init__(self, space, *values_w):
             self.space = space
             assert len(values_w) == typelen

From noreply at buildbot.pypy.org  Sun Jun 30 05:54:55 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 30 Jun 2013 05:54:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20130630035455.1F5551C0255@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r65118:896493e4c586
Date: 2013-06-29 20:54 -0700
http://bitbucket.org/pypy/pypy/changeset/896493e4c586/

Log:	merged upstream

diff --git a/lib_pypy/ctypes_config_cache/syslog.ctc.py b/lib_pypy/ctypes_config_cache/syslog.ctc.py
deleted file mode 100644
--- a/lib_pypy/ctypes_config_cache/syslog.ctc.py
+++ /dev/null
@@ -1,75 +0,0 @@
-"""
-'ctypes_configure' source for syslog.py.
-Run this to rebuild _syslog_cache.py.
-"""
-
-from ctypes_configure.configure import (configure,
-    ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger)
-import dumpcache
-
-
-_CONSTANTS = (
-    'LOG_EMERG',
-    'LOG_ALERT',
-    'LOG_CRIT',
-    'LOG_ERR',
-    'LOG_WARNING',
-    'LOG_NOTICE',
-    'LOG_INFO',
-    'LOG_DEBUG',
-
-    'LOG_PID',
-    'LOG_CONS',
-    'LOG_NDELAY',
-
-    'LOG_KERN',
-    'LOG_USER',
-    'LOG_MAIL',
-    'LOG_DAEMON',
-    'LOG_AUTH',
-    'LOG_LPR',
-    'LOG_LOCAL0',
-    'LOG_LOCAL1',
-    'LOG_LOCAL2',
-    'LOG_LOCAL3',
-    'LOG_LOCAL4',
-    'LOG_LOCAL5',
-    'LOG_LOCAL6',
-    'LOG_LOCAL7',
-)
-_OPTIONAL_CONSTANTS = (
-    'LOG_NOWAIT',
-    'LOG_PERROR',
-
-    'LOG_SYSLOG',
-    'LOG_CRON',
-    'LOG_UUCP',
-    'LOG_NEWS',
-)
-
-# Constant aliases if there are not defined
-_ALIAS = (
-    ('LOG_SYSLOG', 'LOG_DAEMON'),
-    ('LOG_CRON', 'LOG_DAEMON'),
-    ('LOG_NEWS', 'LOG_MAIL'),
-    ('LOG_UUCP', 'LOG_MAIL'),
-)
-
-class SyslogConfigure:
-    _compilation_info_ = ExternalCompilationInfo(includes=['sys/syslog.h'])
-for key in _CONSTANTS:
-    setattr(SyslogConfigure, key, ConstantInteger(key))
-for key in _OPTIONAL_CONSTANTS:
-    setattr(SyslogConfigure, key, DefinedConstantInteger(key))
-
-config = configure(SyslogConfigure)
-for key in _OPTIONAL_CONSTANTS:
-    if config[key] is None:
-        del config[key]
-for alias, key in _ALIAS:
-    config.setdefault(alias, config[key])
-
-all_constants = config.keys()
-all_constants.sort()
-config['ALL_CONSTANTS'] = tuple(all_constants)
-dumpcache.dumpcache2('syslog', config)
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -8,6 +8,7 @@
 
 from ctypes import Structure, c_char_p, c_int, POINTER
 from ctypes_support import standard_c_lib as libc
+import _structseq
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -23,32 +24,13 @@
         ('gr_mem', POINTER(c_char_p)),
         )
 
-class Group(object):
-    def __init__(self, gr_name, gr_passwd, gr_gid, gr_mem):
-        self.gr_name = gr_name
-        self.gr_passwd = gr_passwd
-        self.gr_gid = gr_gid
-        self.gr_mem = gr_mem
+class struct_group:
+    __metaclass__ = _structseq.structseqtype
 
-    def __getitem__(self, item):
-        if item == 0:
-            return self.gr_name
-        elif item == 1:
-            return self.gr_passwd
-        elif item == 2:
-            return self.gr_gid
-        elif item == 3:
-            return self.gr_mem
-        else:
-            raise IndexError(item)
-
-    def __len__(self):
-        return 4
-
-    def __repr__(self):
-        return str((self.gr_name, self.gr_passwd, self.gr_gid, self.gr_mem))
-
-    # whatever else...
+    gr_name   = _structseq.structseqfield(0)
+    gr_passwd = _structseq.structseqfield(1)
+    gr_gid    = _structseq.structseqfield(2)
+    gr_mem    = _structseq.structseqfield(3)
 
 libc.getgrgid.argtypes = [gid_t]
 libc.getgrgid.restype = POINTER(GroupStruct)
@@ -71,8 +53,8 @@
     while res.contents.gr_mem[i]:
         mem.append(res.contents.gr_mem[i])
         i += 1
-    return Group(res.contents.gr_name, res.contents.gr_passwd,
-                 res.contents.gr_gid, mem)
+    return struct_group((res.contents.gr_name, res.contents.gr_passwd,
+                         res.contents.gr_gid, mem))
 
 @builtinify
 def getgrgid(gid):
diff --git a/lib_pypy/syslog.py b/lib_pypy/syslog.py
--- a/lib_pypy/syslog.py
+++ b/lib_pypy/syslog.py
@@ -1,3 +1,4 @@
+# this cffi version was rewritten based on the
 # ctypes implementation: Victor Stinner, 2008-05-08
 """
 This module provides an interface to the Unix syslog library routines.
@@ -9,34 +10,84 @@
 if sys.platform == 'win32':
     raise ImportError("No syslog on Windows")
 
-# load the platform-specific cache made by running syslog.ctc.py
-from ctypes_config_cache._syslog_cache import *
-
-from ctypes_support import standard_c_lib as libc
-from ctypes import c_int, c_char_p
+from cffi import FFI
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
 
+ffi = FFI()
 
-# Real prototype is:
-# void syslog(int priority, const char *format, ...);
-# But we also need format ("%s") and one format argument (message)
-_syslog = libc.syslog
-_syslog.argtypes = (c_int, c_char_p, c_char_p)
-_syslog.restype = None
+ffi.cdef("""
+/* mandatory constants */
+#define LOG_EMERG ...
+#define LOG_ALERT ...
+#define LOG_CRIT ...
+#define LOG_ERR ...
+#define LOG_WARNING ...
+#define LOG_NOTICE ...
+#define LOG_INFO ...
+#define LOG_DEBUG ...
 
-_openlog = libc.openlog
-_openlog.argtypes = (c_char_p, c_int, c_int)
-_openlog.restype = None
+#define LOG_PID ...
+#define LOG_CONS ...
+#define LOG_NDELAY ...
 
-_closelog = libc.closelog
-_closelog.argtypes = None
-_closelog.restype = None
+#define LOG_KERN ...
+#define LOG_USER ...
+#define LOG_MAIL ...
+#define LOG_DAEMON ...
+#define LOG_AUTH ...
+#define LOG_LPR ...
+#define LOG_LOCAL0 ...
+#define LOG_LOCAL1 ...
+#define LOG_LOCAL2 ...
+#define LOG_LOCAL3 ...
+#define LOG_LOCAL4 ...
+#define LOG_LOCAL5 ...
+#define LOG_LOCAL6 ...
+#define LOG_LOCAL7 ...
 
-_setlogmask = libc.setlogmask
-_setlogmask.argtypes = (c_int,)
-_setlogmask.restype = c_int
+/* optional constants, gets defined to -919919 if missing */
+#define LOG_NOWAIT ...
+#define LOG_PERROR ...
+
+/* aliased constants, gets defined as some other constant if missing */
+#define LOG_SYSLOG ...
+#define LOG_CRON ...
+#define LOG_UUCP ...
+#define LOG_NEWS ...
+
+/* functions */
+void openlog(const char *ident, int option, int facility);
+void syslog(int priority, const char *format, const char *string);
+// NB. the signature of syslog() is specialized to the only case we use
+void closelog(void);
+int setlogmask(int mask);
+""")
+
+lib = ffi.verify("""
+#include <syslog.h>
+
+#ifndef LOG_NOWAIT
+#define LOG_NOWAIT -919919
+#endif
+#ifndef LOG_PERROR
+#define LOG_PERROR -919919
+#endif
+#ifndef LOG_SYSLOG
+#define LOG_SYSLOG LOG_DAEMON
+#endif
+#ifndef LOG_CRON
+#define LOG_CRON LOG_DAEMON
+#endif
+#ifndef LOG_UUCP
+#define LOG_UUCP LOG_MAIL
+#endif
+#ifndef LOG_NEWS
+#define LOG_NEWS LOG_MAIL
+#endif
+""")
+
 
 _S_log_open = False
 _S_ident_o = None
@@ -52,12 +103,17 @@
     return None
 
 @builtinify
-def openlog(ident=None, logoption=0, facility=LOG_USER):
+def openlog(ident=None, logoption=0, facility=lib.LOG_USER):
     global _S_ident_o, _S_log_open
     if ident is None:
         ident = _get_argv()
-    _S_ident_o = c_char_p(ident)    # keepalive
-    _openlog(_S_ident_o, logoption, facility)
+    if ident is None:
+        _S_ident_o = ffi.NULL
+    elif isinstance(ident, str):
+        _S_ident_o = ffi.new("char[]", ident)    # keepalive
+    else:
+        raise TypeError("'ident' must be a string or None")
+    lib.openlog(_S_ident_o, logoption, facility)
     _S_log_open = True
 
 @builtinify
@@ -69,19 +125,19 @@
     # if log is not opened, open it now
     if not _S_log_open:
         openlog()
-    _syslog(priority, "%s", message)
+    lib.syslog(priority, "%s", message)
 
 @builtinify
 def closelog():
     global _S_log_open, S_ident_o
     if _S_log_open:
-        _closelog()
+        lib.closelog()
         _S_log_open = False
         _S_ident_o = None
 
 @builtinify
 def setlogmask(mask):
-    return _setlogmask(mask)
+    return lib.setlogmask(mask)
 
 @builtinify
 def LOG_MASK(pri):
@@ -91,8 +147,15 @@
 def LOG_UPTO(pri):
     return (1 << (pri + 1)) - 1
 
-__all__ = ALL_CONSTANTS + (
+__all__ = []
+
+for name in sorted(lib.__dict__):
+    if name.startswith('LOG_'):
+        value = getattr(lib, name)
+        if value != -919919:
+            globals()[name] = value
+            __all__.append(name)
+
+__all__ = tuple(__all__) + (
     'openlog', 'syslog', 'closelog', 'setlogmask',
     'LOG_MASK', 'LOG_UPTO')
-
-del ALL_CONSTANTS
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -101,6 +101,9 @@
             raise OperationError(
                 space.w_ValueError, space.wrap(message))
 
+    def check_closed_w(self, space):
+        self._check_closed(space)
+
     def closed_get_w(self, space):
         return space.newbool(self.__IOBase_closed)
 
@@ -277,6 +280,7 @@
     _checkReadable = interp2app(check_readable_w),
     _checkWritable = interp2app(check_writable_w),
     _checkSeekable = interp2app(check_seekable_w),
+    _checkClosed = interp2app(W_IOBase.check_closed_w),
     closed = GetSetProperty(W_IOBase.closed_get_w),
     __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, cls=W_IOBase),
     __weakref__ = make_weakref_descr(W_IOBase),
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -22,7 +22,9 @@
         import io
         with io.BufferedIOBase() as f:
             assert not f.closed
+            f._checkClosed()
         assert f.closed
+        raises(ValueError, f._checkClosed)
 
     def test_iter(self):
         import io
diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -13,6 +13,9 @@
     def next(self):
         self.called_once = True
 
+    def next_skip_x(self, n):
+        self.called_once = True
+
     def getitem(self):
         return self.v.get_scalar_value()
 
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -37,7 +37,7 @@
 we can go faster.
 All the calculations happen in next()
 
-next_skip_x() tries to do the iteration for a number of steps at once,
+next_skip_x(steps) tries to do the iteration for a number of steps at once,
 but then we cannot gaurentee that we only overflow one single shape
 dimension, perhaps we could overflow times in one big step.
 """
diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py
--- a/pypy/module/micronumpy/test/test_iter.py
+++ b/pypy/module/micronumpy/test/test_iter.py
@@ -1,4 +1,5 @@
 from pypy.module.micronumpy.iter import MultiDimViewIterator
+from pypy.module.micronumpy.arrayimpl.scalar import ScalarIterator
 
 class MockArray(object):
     size = 1
@@ -8,7 +9,7 @@
         #Let's get started, simple iteration in C order with
         #contiguous layout => strides[-1] is 1
         start = 0
-        shape = [3, 5] 
+        shape = [3, 5]
         strides = [5, 1]
         backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
         assert backstrides == [10, 4]
@@ -47,7 +48,7 @@
         #iteration in C order with #contiguous layout => strides[-1] is 1
         #skip less than the shape
         start = 0
-        shape = [3, 5] 
+        shape = [3, 5]
         strides = [5, 1]
         backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
         assert backstrides == [10, 4]
@@ -89,3 +90,9 @@
         assert i.indexes == [0,1]
         assert i.offset == 3
         assert i.done()
+
+    def test_scalar_iter(self):
+        i = ScalarIterator(MockArray)
+        i.next()
+        i.next_skip_x(3)
+        assert i.done()
diff --git a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
--- a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
+++ b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py
@@ -32,14 +32,6 @@
     return d
 
 
-def test_syslog():
-    try:
-        import lib_pypy.syslog
-    except ImportError:
-        py.test.skip('no syslog on this platform')
-    d = run('syslog.ctc.py', '_syslog_cache.py')
-    assert 'LOG_NOTICE' in d
-
 def test_resource():
     try:
         import lib_pypy.resource
diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -5,6 +5,22 @@
 except ImportError:
     py.test.skip("No grp module on this platform")
 
+def test_basic():
+    g = grp.getgrnam("root")
+    assert g.gr_gid == 0
+    assert g.gr_mem == ['root'] or g.gr_mem == []
+    assert g.gr_name == 'root'
+    assert isinstance(g.gr_passwd, str)    # usually just 'x', don't hope :-)
+
 def test_extra():
     py.test.raises(TypeError, grp.getgrnam, False)
     py.test.raises(TypeError, grp.getgrnam, None)
+
+def test_struct_group():
+    g = grp.struct_group((10, 20, 30, 40))
+    assert len(g) == 4
+    assert list(g) == [10, 20, 30, 40]
+    assert g.gr_name == 10
+    assert g.gr_passwd == 20
+    assert g.gr_gid == 30
+    assert g.gr_mem == 40
diff --git a/pypy/module/test_lib_pypy/test_syslog.py b/pypy/module/test_lib_pypy/test_syslog.py
--- a/pypy/module/test_lib_pypy/test_syslog.py
+++ b/pypy/module/test_lib_pypy/test_syslog.py
@@ -1,15 +1,15 @@
 from __future__ import absolute_import
-import py
+import sys, py
 try:
     from lib_pypy import syslog
 except ImportError:
     py.test.skip('no syslog on this platform')
+except AssertionError:
+    if '__pypy__' in sys.builtin_module_names:
+        raise
+    py.test.skip('AssertionError during import (wrong cffi version?)')
 
 # XXX very minimal test
 
-from lib_pypy.ctypes_config_cache import rebuild
-rebuild.rebuild_one('syslog.ctc.py')
-
-
 def test_syslog():
     assert hasattr(syslog, 'LOG_ALERT')
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -69,6 +69,7 @@
     subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
     if not sys.platform == 'win32':
         subprocess.check_call([str(pypy_c), '-c', 'import _curses'])
+        subprocess.check_call([str(pypy_c), '-c', 'import syslog'])
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
         rename_pypy_c += '.exe'
     binaries = [(pypy_c, rename_pypy_c)]

From noreply at buildbot.pypy.org  Sun Jun 30 11:02:34 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 11:02:34 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Support gcremovetypeptr with stm,
 and in fact recommend it, as it is done by
Message-ID: <20130630090234.2A0CD1C13B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65119:6bca9bb3fd22
Date: 2013-06-30 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/6bca9bb3fd22/

Log:	Support gcremovetypeptr with stm, and in fact recommend it, as it is
	done by direct reads of the h_tid.

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -76,6 +76,9 @@
                      "stmgc": [("translation.gctransformer", "framework"),
                                ("translation.gcrootfinder", "stm")],
                      },
+                 suggests = {
+                     "stmgc": [("translation.gcremovetypeptr", True)],
+                     },
                   cmdline="--gc"),
     ChoiceOption("gctransformer", "GC transformer that is used - internal",
                  ["boehm", "ref", "framework", "none"],
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -87,6 +87,9 @@
             elif T == WeakRef:
                 REALT = self.gcpolicy.get_real_weakref_type()
                 node = self.gettypedefnode(REALT)
+            elif isinstance(T, OpaqueType) and T.__name__ == "struct stm_object_s":
+                from rpython.translator.stm.funcgen import StmHeaderOpaqueDefNode
+                node = StmHeaderOpaqueDefNode(self, T)
             else:
                 raise NoCorrespondingNode("don't know about %r" % (T,))
             self.structdefnodes[key] = node
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -1,8 +1,26 @@
 from rpython.translator.c.support import c_string_constant, cdecl
-from rpython.translator.c.node import ContainerNode
+from rpython.translator.c.node import Node, ContainerNode
 from rpython.translator.c.primitive import name_small_integer
 
 
+class StmHeaderOpaqueDefNode(Node):
+    typetag = 'struct'
+
+    def __init__(self, db, T):
+        Node.__init__(self, db)
+        self.T = T
+        self.name = 'stm_object_s'
+
+    def setup(self):
+        pass
+
+    def definition(self):
+        return []
+
+    def c_struct_field_name(self, _):
+        return 'h_tid'
+
+
 class StmHeader_OpaqueNode(ContainerNode):
     nodekind = 'stmhdr'
     globalcontainer = True

From noreply at buildbot.pypy.org  Sun Jun 30 11:02:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 11:02:35 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Fails on 64-bit too with larger numbers
Message-ID: <20130630090235.6E8701C13D5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65120:e24d8e7514fe
Date: 2013-06-30 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/e24d8e7514fe/

Log:	Fails on 64-bit too with larger numbers

diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -24,6 +24,8 @@
         assert '< 5 >' in data, "got: %r" % (data,)
         data = cbuilder.cmdexec('42')
         assert '< 42 >' in data, "got: %r" % (data,)
+        data = cbuilder.cmdexec('260')
+        assert '< 260 >' in data, "got: %r" % (data,)
 
     def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo2.entry_point)

From noreply at buildbot.pypy.org  Sun Jun 30 11:02:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 11:02:36 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Argh!
Message-ID: <20130630090236.98ED61C13FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65121:fbbf6a7ae469
Date: 2013-06-30 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/fbbf6a7ae469/

Log:	Argh!

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -62,7 +62,7 @@
                              offset_to_length):
         # XXX be careful about overflows, and call optimized versions
         totalsize = size + itemsize * length
-        obj = llop.stm_allocate(llmemory.Address, typeid16, totalsize)
+        obj = llop.stm_allocate(llmemory.Address, totalsize, typeid16)
         (obj + offset_to_length).signed[0] = length
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 

From noreply at buildbot.pypy.org  Sun Jun 30 11:47:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 11:47:32 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: hash, id
Message-ID: <20130630094732.0466E1C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65122:74d273f9c9cd
Date: 2013-06-30 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/74d273f9c9cd/

Log:	hash, id

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -592,6 +592,8 @@
     OP_STM_POP_ROOT_INTO = _OP_STM
     OP_STM_ALLOCATE = _OP_STM
     OP_STM_GET_TID = _OP_STM
+    OP_STM_HASH = _OP_STM
+    OP_STM_ID = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -97,6 +97,16 @@
     result = funcgen.expr(op.result)
     return '%s = stm_get_tid((gcptr)%s);' % (result, arg0)
 
+def stm_hash(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    result = funcgen.expr(op.result)
+    return '%s = stm_hash((gcptr)%s);' % (result, arg0)
+
+def stm_id(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    result = funcgen.expr(op.result)
+    return '%s = stm_id((gcptr)%s);' % (result, arg0)
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -27,6 +27,33 @@
         data = cbuilder.cmdexec('260')
         assert '< 260 >' in data, "got: %r" % (data,)
 
+    def test_hash_id(self):
+        from rpython.rlib.objectmodel import compute_identity_hash
+        from rpython.rlib.objectmodel import compute_unique_id
+        FOO = lltype.GcStruct('FOO')
+        prebuilt = lltype.malloc(FOO)
+        prebuilt_hash = lltype.identityhash(prebuilt)
+        #
+        def w(num, x):
+            print '%d>>>' % num, compute_identity_hash(x), compute_unique_id(x)
+        #
+        def entry_point(argv):
+            w(1, prebuilt)
+            w(2, lltype.malloc(FOO))
+            return 0
+        #
+        t, cbuilder = self.compile(entry_point, backendopt=True)
+        assert prebuilt_hash == lltype.identityhash(prebuilt)
+        data = cbuilder.cmdexec('')
+        data = data.split()
+        i1 = data.index('1>>>')
+        i2 = data.index('2>>>')
+        int(data[i1 + 1])
+        int(data[i1 + 2])
+        int(data[i2 + 1])
+        int(data[i2 + 2])
+        assert int(data[i1 + 1]) == prebuilt_hash
+
     def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo2.entry_point)
         data, dataerr = cbuilder.cmdexec('4 5000', err=True,

From noreply at buildbot.pypy.org  Sun Jun 30 18:28:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 18:28:20 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Updates
Message-ID: <20130630162820.25D011C0634@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r319:1ba74b051c7b
Date: 2013-06-30 18:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/1ba74b051c7b/

Log:	Updates

diff --git a/c4/fprintcolor.c b/c4/fprintcolor.c
--- a/c4/fprintcolor.c
+++ b/c4/fprintcolor.c
@@ -24,12 +24,8 @@
 static __thread revision_t tcolor = 0;
 static revision_t tnextid = 0;
 
-
-int threadcolor_printf(const char *format, ...)
+int dprintfcolor(void)
 {
-    char buffer[2048];
-    va_list ap;
-    int result;
     if (tcolor == 0) {
         while (1) {
             tcolor = tnextid;
@@ -38,7 +34,15 @@
         }
         tcolor = 31 + tcolor % 6;
     }
-    int size = (int)sprintf(buffer, "\033[%dm", (int)tcolor);
+    return tcolor;
+}
+
+int threadcolor_printf(const char *format, ...)
+{
+    char buffer[2048];
+    va_list ap;
+    int result;
+    int size = (int)sprintf(buffer, "\033[%dm", dprintfcolor());
     assert(size >= 0);
 
     va_start(ap, format);
diff --git a/c4/fprintcolor.h b/c4/fprintcolor.h
--- a/c4/fprintcolor.h
+++ b/c4/fprintcolor.h
@@ -9,6 +9,7 @@
 #ifdef _GC_DEBUG
 
 #define dprintf(args)   threadcolor_printf args
+int dprintfcolor(void);
 
 int threadcolor_printf(const char *format, ...)
      __attribute__((format (printf, 1, 2)));
@@ -16,5 +17,6 @@
 #else
 
 #define dprintf(args)   do { } while(0)
+#define dprintfcolor()  0
 
 #endif
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -64,6 +64,7 @@
     gcptr P;
     char *cur = d->nursery_current;
     char *end = cur + size;
+    assert((size & 3) == 0);
     d->nursery_current = end;
     if (end > d->nursery_nextlimit) {
         P = allocate_next_section(size, tid);

From noreply at buildbot.pypy.org  Sun Jun 30 18:29:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 18:29:04 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: The next test passes
Message-ID: <20130630162904.93DDD1C0634@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65123:1bacb9bddd05
Date: 2013-06-30 18:28 +0200
http://bitbucket.org/pypy/pypy/changeset/1bacb9bddd05/

Log:	The next test passes

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -4,7 +4,8 @@
 """
 
 from rpython.memory.gc.base import GCBase, MovingGCBase
-from rpython.rtyper.lltypesystem import lltype, llmemory, llgroup, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, llgroup, llarena
+from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
 
@@ -62,6 +63,7 @@
                              offset_to_length):
         # XXX be careful about overflows, and call optimized versions
         totalsize = size + itemsize * length
+        totalsize = llarena.round_up_for_allocation(totalsize)
         obj = llop.stm_allocate(llmemory.Address, totalsize, typeid16)
         (obj + offset_to_length).signed[0] = length
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -1,3 +1,6 @@
+from rpython.annotator import model as annmodel
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gctransform.framework import (
      BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr)
 
@@ -5,8 +8,6 @@
 class StmFrameworkGCTransformer(BaseFrameworkGCTransformer):
 
     def _declare_functions(self, GCClass, getfn, s_gc, s_typeid16):
-        from rpython.annotator import model as annmodel
-        from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
         BaseFrameworkGCTransformer._declare_functions(self, GCClass, getfn,
                                                       s_gc, s_typeid16)
         gc = self.gcdata.gc
@@ -48,8 +49,13 @@
 
 class StmRootWalker(BaseRootWalker):
 
-    def need_thread_support(self, gctransform, getfn):
-        pass
+    def need_thread_support(self, gctransformer, getfn):
+        def thread_start():
+            llop.stm_initialize(lltype.Void)
+        def thread_die():
+            llop.stm_finalize(lltype.Void)
+        self.thread_start_ptr = getfn(thread_start, [], annmodel.s_None)
+        self.thread_die_ptr = getfn(thread_die, [], annmodel.s_None)
 
     def walk_stack_roots(self, collect_stack_root):
         raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -422,6 +422,7 @@
     # to keep them as operations until the genc stage)
 
     'stm_initialize':         LLOp(),
+    'stm_finalize':           LLOp(),
     'stm_barrier':            LLOp(sideeffects=False),
     'stm_allocate':           LLOp(sideeffects=False),
     'stm_become_inevitable':  LLOp(),
@@ -432,6 +433,8 @@
     'stm_hash':               LLOp(sideeffects=False),
     'stm_push_root':          LLOp(),
     'stm_pop_root_into':      LLOp(),
+    'stm_commit_transaction': LLOp(),
+    'stm_begin_inevitable_transaction': LLOp(),
 
     # __________ address operations __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -585,6 +585,7 @@
             self.__class__.op_stm = op_stm
         return self.op_stm(op)
     OP_STM_INITIALIZE = _OP_STM
+    OP_STM_FINALIZE = _OP_STM
     OP_STM_BECOME_INEVITABLE = _OP_STM
     OP_STM_BARRIER = _OP_STM
     OP_STM_PTR_EQ = _OP_STM
@@ -594,6 +595,8 @@
     OP_STM_GET_TID = _OP_STM
     OP_STM_HASH = _OP_STM
     OP_STM_ID = _OP_STM
+    OP_STM_COMMIT_TRANSACTION = _OP_STM
+    OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -1,3 +1,4 @@
+from rpython.flowspace.model import Constant
 from rpython.translator.c.support import c_string_constant, cdecl
 from rpython.translator.c.node import Node, ContainerNode
 from rpython.translator.c.primitive import name_small_integer
@@ -43,6 +44,9 @@
 def stm_initialize(funcgen, op):
     return 'stm_initialize();'
 
+def stm_finalize(funcgen, op):
+    return 'stm_finalize();'
+
 _STM_BARRIER_FUNCS = {   # XXX try to see if some combinations can be shorter
     'P2R': 'stm_read_barrier',
     'G2R': 'stm_read_barrier',
@@ -83,6 +87,8 @@
 
 def stm_pop_root_into(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
+    if isinstance(op.args[0], Constant):
+        return '/* %s = */ stm_pop_root();' % (arg0,)
     return '%s = (%s)stm_pop_root();' % (
         arg0, cdecl(funcgen.lltypename(op.args[0]), ''))
 
@@ -107,6 +113,12 @@
     result = funcgen.expr(op.result)
     return '%s = stm_id((gcptr)%s);' % (result, arg0)
 
+def stm_commit_transaction(funcgen, op):
+    return 'stm_commit_transaction();'
+
+def stm_begin_inevitable_transaction(funcgen, op):
+    return 'stm_begin_inevitable_transaction();'
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c
--- a/rpython/translator/stm/src_stm/nursery.c
+++ b/rpython/translator/stm/src_stm/nursery.c
@@ -65,6 +65,7 @@
     gcptr P;
     char *cur = d->nursery_current;
     char *end = cur + size;
+    assert((size & 3) == 0);
     d->nursery_current = end;
     if (end > d->nursery_nextlimit) {
         P = allocate_next_section(size, tid);
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-c856b6a0d157+
+1ba74b051c7b
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -42,7 +42,7 @@
             w(2, lltype.malloc(FOO))
             return 0
         #
-        t, cbuilder = self.compile(entry_point, backendopt=True)
+        t, cbuilder = self.compile(entry_point)
         assert prebuilt_hash == lltype.identityhash(prebuilt)
         data = cbuilder.cmdexec('')
         data = data.split()
@@ -54,6 +54,31 @@
         int(data[i2 + 2])
         assert int(data[i1 + 1]) == prebuilt_hash
 
+    def test_start_thread(self):
+        from rpython.rlib import rthread
+        class Global:
+            value = 1
+            seen = None
+        glob = Global()
+        #
+        def threadfn():
+            rthread.gc_thread_start()
+            x = Global()
+            x.value = 0
+            glob.seen = x
+            rthread.gc_thread_die()
+        def entry_point(argv):
+            glob.seen = None
+            rthread.start_new_thread(threadfn, ())
+            while glob.seen is None:
+                llop.stm_commit_transaction(lltype.Void)
+                llop.stm_begin_inevitable_transaction(lltype.Void)
+            return glob.seen.value
+        #
+        t, cbuilder = self.compile(entry_point)
+        cbuilder.cmdexec('')
+        # assert did not crash
+
     def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo2.entry_point)
         data, dataerr = cbuilder.cmdexec('4 5000', err=True,

From noreply at buildbot.pypy.org  Sun Jun 30 19:14:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 19:14:10 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Re-add counting reads
Message-ID: <20130630171410.AEAD61C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r320:f88a2e8e6b8e
Date: 2013-06-30 19:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/f88a2e8e6b8e/

Log:	Re-add counting reads

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -213,6 +213,7 @@
   /* The risks are that the following assert fails, because the flag was
      added just now by a parallel thread during stealing... */
   /*assert(!(P->h_tid & GCFLAG_NURSERY_MOVED));*/
+  d->count_reads++;
   fxcache_add(&d->recent_reads_cache, P);
   return P;
 
@@ -600,6 +601,7 @@
       record_write_barrier(W);
     }
 
+  d->count_reads++;
   spinlock_release(d->public_descriptor->collection_lock);
 
   dprintf(("write_barrier: %p -> %p -> %p\n", P, R, W));

From noreply at buildbot.pypy.org  Sun Jun 30 19:14:11 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 19:14:11 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Redo the hack about END_MARKERs in the
	shadowstack.
Message-ID: <20130630171411.C773F1C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r321:7c03d107c940
Date: 2013-06-30 19:11 +0200
http://bitbucket.org/pypy/stmgc/changeset/7c03d107c940/

Log:	Redo the hack about END_MARKERs in the shadowstack.

diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -320,13 +320,20 @@
 
 static void mark_roots(gcptr *root, gcptr *end)
 {
-    //assert(*root == END_MARKER);
-    //root++;
+    assert(*root == END_MARKER_ON);
+    root++;
+
     while (root != end) {
-        gcptr o = *root;
-        visit(root);
-        dprintf(("visit stack root: %p -> %p\n", o, *root));
-        (void)o;   /* silence "warning: unused variable 'o'" */
+        gcptr item = *root;
+        if (((revision_t)item) & ~((revision_t)END_MARKER_OFF |
+                                   (revision_t)END_MARKER_ON)) {
+            /* 'item' is a regular, non-null pointer */
+            visit(root);
+            dprintf(("visit stack root: %p -> %p\n", item, *root));
+        }
+        else if (item == END_MARKER_OFF) {
+            *root = END_MARKER_ON;
+        }
         root++;
     }
 }
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -323,12 +323,30 @@
 
 static void mark_young_roots(struct tx_descriptor *d)
 {
-    gcptr *root = d->shadowstack;
+    /* we walk the shadowstack from the end, replacing any END_MARKER_OFF
+       found with END_MARKER_ON.  When we reach an END_MARKER_ON, we know
+       that we have already seen the rest of the stack in the previous
+       nursery collection, so we stop.
+    */
     gcptr *end = *d->shadowstack_end_ref;
 
-    /* XXX use a way to avoid walking all roots again and again */
-    for (; root != end; root++) {
-        visit_if_young(root);
+    while (1) {
+        assert(end > d->shadowstack);
+        gcptr item = *--end;
+
+        if (((revision_t)item) & ~((revision_t)END_MARKER_OFF |
+                                   (revision_t)END_MARKER_ON)) {
+            /* 'item' is a regular, non-null pointer */
+            visit_if_young(end);
+        }
+        else if (item != NULL) {
+            if (item == END_MARKER_OFF)
+                *end = END_MARKER_ON;
+            else {
+                assert(item == END_MARKER_ON);
+                break;
+            }
+        }
     }
 }
 
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -18,6 +18,9 @@
 # error "GC_NURSERY must be a multiple of GC_NURSERY_SECTION"
 #endif
 
+#define END_MARKER_OFF  ((gcptr) 16)
+#define END_MARKER_ON   ((gcptr) 24)
+
 
 #define NURSERY_FIELDS_DECL                                             \
     /* the nursery */                                                   \
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -25,14 +25,14 @@
     }
     stm_shadowstack = d->shadowstack;
     d->shadowstack_end_ref = &stm_shadowstack;
-    //stm_push_root(END_MARKER);
+    stm_push_root(END_MARKER_ON);
 }
 
 static void done_shadowstack(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    //gcptr x = stm_pop_root();
-    //assert(x == END_MARKER);
+    gcptr x = stm_pop_root();
+    assert(x == END_MARKER_ON);
     assert(stm_shadowstack == d->shadowstack);
     stm_shadowstack = NULL;
     free(d->shadowstack);
@@ -77,6 +77,7 @@
     long volatile v_atomic;
 
     stm_push_root(arg);
+    stm_push_root(END_MARKER_OFF);
 
     if (!(v_atomic = thread_descriptor->atomic))
         CommitTransaction();
@@ -98,14 +99,7 @@
     long counter, result;
     counter = v_counter;
     d->atomic = v_atomic;
-    stm_shadowstack = v_saved_value + 1;    /* skip the 'arg', pushed above */
-    //    if (!d->atomic) {
-    //        /* In non-atomic mode, we are now between two transactions.
-    //           It means that in the next transaction's collections we know
-    //           that we won't need to access the shadow stack beyond its
-    //           current position.  So we add an end marker. */
-    //        stm_push_root(END_MARKER);
-    //    }
+    stm_shadowstack = v_saved_value + 2;   /*skip the two values pushed above*/
 
     do {
         v_counter = counter + 1;
@@ -135,7 +129,7 @@
         /* invoke the callback in the new transaction */
         arg = v_saved_value[0];
         result = callback(arg, counter);
-        assert(stm_shadowstack == v_saved_value + 1);
+        assert(stm_shadowstack == v_saved_value + 2);
 
         v_atomic = d->atomic;
         if (!d->atomic)
@@ -154,7 +148,9 @@
         BeginInevitableTransaction();
     }
 
-    stm_pop_root();      /* pop the 'arg' */
+    gcptr x = stm_pop_root();   /* pop the END_MARKER */
+    assert(x == END_MARKER_OFF || x == END_MARKER_ON);
+    stm_pop_root();             /* pop the 'arg' */
     assert(stm_shadowstack == v_saved_value);
 }
 
diff --git a/c4/test/test_random.py b/c4/test/test_random.py
--- a/c4/test/test_random.py
+++ b/c4/test/test_random.py
@@ -305,12 +305,16 @@
         #
         self.roots = self.roots_outside_perform[:]
         self.startrev()
-        # XXX stm_perform_transaction() adds one root for the unused arg
+        # XXX stm_perform_transaction() adds one root for the unused arg,
+        # and one for an END_MARKER
+        end_marker = lib.stm_pop_root()
+        assert int(ffi.cast("revision_t", end_marker)) in [16, 24]
         arg = lib.stm_pop_root()
         assert arg == ffi.NULL
         self.pop_roots()
         self.push_roots()
         lib.stm_push_root(arg)
+        lib.stm_push_root(end_marker)
         self.check_valid(self.roots)
         #
         try:

From noreply at buildbot.pypy.org  Sun Jun 30 19:14:13 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 19:14:13 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Port here the changes done to
	test_random.
Message-ID: <20130630171413.06F7E1C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r322:ad8b5874acca
Date: 2013-06-30 19:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/ad8b5874acca/

Log:	Port here the changes done to test_random.

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -463,11 +463,14 @@
     // copy_roots(td.roots_outside_perform, td.roots, td.num_roots);
 
     // refresh td.roots:
+    gcptr end_marker = stm_pop_root();
+    assert(end_marker == END_MARKER_ON || end_marker == END_MARKER_OFF);
     arg1 = stm_pop_root();
     assert(arg1 == NULL);
     pop_roots();
     push_roots();
     stm_push_root(arg1);
+    stm_push_root(end_marker);
 
     int p = run_me();
     if (p == -1) // maybe restart transaction

From noreply at buildbot.pypy.org  Sun Jun 30 19:48:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 19:48:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Tests for should_break_transaction()
 and set_transaction_length().
Message-ID: <20130630174841.75F0D1C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r323:d97c6375a312
Date: 2013-06-30 19:36 +0200
http://bitbucket.org/pypy/stmgc/changeset/d97c6375a312/

Log:	Tests for should_break_transaction() and set_transaction_length().

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -86,6 +86,8 @@
   gcptr P = G;
   revision_t v;
 
+  d->count_reads++;
+
  restart_all:
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
@@ -213,7 +215,6 @@
   /* The risks are that the following assert fails, because the flag was
      added just now by a parallel thread during stealing... */
   /*assert(!(P->h_tid & GCFLAG_NURSERY_MOVED));*/
-  d->count_reads++;
   fxcache_add(&d->recent_reads_cache, P);
   return P;
 
@@ -601,7 +602,6 @@
       record_write_barrier(W);
     }
 
-  d->count_reads++;
   spinlock_release(d->public_descriptor->collection_lock);
 
   dprintf(("write_barrier: %p -> %p -> %p\n", P, R, W));
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -57,7 +57,8 @@
 /* start a new transaction, calls callback(), and when it returns
    finish that transaction.  callback() is called with the 'arg'
    provided, and with a retry_counter number.  Must save roots around
-   this call. */
+   this call.  The callback() is called repeatedly as long as it
+   returns a value > 0. */
 void stm_perform_transaction(gcptr arg, int (*callback)(gcptr, int));
 
 /* finish the current transaction, start a new one, or turn the current
@@ -70,8 +71,11 @@
 /* debugging: check if we're currently running a transaction or not. */
 int stm_in_transaction(void);
 
-/* change the default transaction length */
+/* change the default transaction length, and ask if now would be a good
+   time to break the transaction (by returning from the 'callback' above
+   with a positive value). */
 void stm_set_transaction_length(long length_max);
+_Bool stm_should_break_transaction(void);
 
 /* callback: get the size of an object */
 extern size_t stmcb_size(gcptr);
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -6,6 +6,7 @@
 
 __thread gcptr *stm_shadowstack;
 static unsigned long stm_regular_length_limit = 10000;
+static revision_t sync_required = 0;
 
 void stm_set_transaction_length(long length_max)
 {
@@ -16,6 +17,38 @@
     stm_regular_length_limit = length_max;
 }
 
+_Bool stm_should_break_transaction(void)
+{
+    struct tx_descriptor *d = thread_descriptor;
+
+    /* a single comparison to handle all cases:
+
+     - first, if sync_required == -1, this should return True.
+
+     - if d->atomic, then we should return False.  This is done by
+       forcing reads_size_limit to ULONG_MAX as soon as atomic > 0.
+
+     - otherwise, if is_inevitable(), then we should return True.
+       This is done by forcing both reads_size_limit and
+       reads_size_limit_nonatomic to 0 in that case.
+
+     - finally, the default case: return True if d->count_reads is
+       greater than reads_size_limit == reads_size_limit_nonatomic.
+    */
+#ifdef _GC_DEBUG
+    /* reads_size_limit is ULONG_MAX if d->atomic, or else it is equal to
+       reads_size_limit_nonatomic. */
+    assert(d->reads_size_limit == (d->atomic ? ULONG_MAX :
+                                   d->reads_size_limit_nonatomic));
+    /* if is_inevitable(), reads_size_limit_nonatomic should be 0
+       (and thus reads_size_limit too, if !d->atomic.) */
+    if (d->active == 2)
+        assert(d->reads_size_limit_nonatomic == 0);
+#endif
+
+    return (sync_required | d->count_reads) >= d->reads_size_limit;
+}
+
 static void init_shadowstack(void)
 {
     struct tx_descriptor *d = thread_descriptor;
@@ -67,8 +100,6 @@
 
 /************************************************************/
 
-static revision_t sync_required = 0;
-
 void stm_perform_transaction(gcptr arg, int (*callback)(gcptr, int))
 {   /* must save roots around this call */
     jmp_buf _jmpbuf;
@@ -235,7 +266,7 @@
        which prevents any other thread from running in a transaction.
        Warning, may block waiting for rwlock_in_transaction while another
        thread runs a major GC itself! */
-    ACCESS_ONCE(sync_required) = 1;
+    ACCESS_ONCE(sync_required) = -1;
     stm_stop_sharedlock();
     start_exclusivelock();
     ACCESS_ONCE(sync_required) = 0;
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -61,6 +61,7 @@
     void stm_commit_transaction(void);
     void stm_begin_inevitable_transaction(void);
     void stm_set_transaction_length(long length_max);
+    _Bool stm_should_break_transaction(void);
 
     /* extra non-public code */
     void printfcolor(char *msg);
@@ -665,5 +666,7 @@
     assert (r % 4) == 0
     return ffi.cast("gcptr", r)
 
+should_break_transaction = lib.stm_should_break_transaction
+
     
 nrb_protected = ffi.cast("gcptr", -1)
diff --git a/c4/test/test_atomic.py b/c4/test/test_atomic.py
new file mode 100644
--- /dev/null
+++ b/c4/test/test_atomic.py
@@ -0,0 +1,33 @@
+import py
+from support import *
+
+
+def setup_function(f):
+    lib.stm_clear_between_tests()
+    lib.stm_initialize_tests(getattr(f, 'max_aborts', 0))
+
+def teardown_function(_):
+    lib.stm_finalize()
+
+
+def test_should_break_transaction():
+    # we're inevitable here, so:
+    assert should_break_transaction() == True
+    #
+    @perform_transaction
+    def run(retry_counter):
+        assert should_break_transaction() == False
+
+def test_set_transaction_length():
+    lib.stm_set_transaction_length(5)    # breaks after 4 read-or-writes
+    plist = [palloc(HDR) for i in range(6)]
+    should_br = ['?'] * (len(plist) + 1)
+    #
+    @perform_transaction
+    def run(retry_counter):
+        should_br[0] = should_break_transaction()
+        for i in range(len(plist)):
+            lib.stm_write_barrier(plist[i])
+            should_br[i + 1] = should_break_transaction()
+    #
+    assert should_br == [False, False, False, False, True, True, True]

From noreply at buildbot.pypy.org  Sun Jun 30 19:48:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 19:48:42 +0200 (CEST)
Subject: [pypy-commit] stmgc default: stm_atomic().
Message-ID: <20130630174842.9282A1C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r324:59eb9a85c3d1
Date: 2013-06-30 19:48 +0200
http://bitbucket.org/pypy/stmgc/changeset/59eb9a85c3d1/

Log:	stm_atomic().

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -903,6 +903,15 @@
   d->reads_size_limit = d->atomic ? ULONG_MAX : d->reads_size_limit_nonatomic;
 }
 
+long stm_atomic(long delta)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  d->atomic += delta;
+  assert(d->atomic >= 0);
+  update_reads_size_limit(d);
+  return d->atomic;
+}
+
 static void init_transaction(struct tx_descriptor *d)
 {
   assert(d->active == 0);
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -77,6 +77,13 @@
 void stm_set_transaction_length(long length_max);
 _Bool stm_should_break_transaction(void);
 
+/* change the atomic counter by 'delta' and return the new value.  Used
+   with +1 to enter or with -1 to leave atomic mode, or with 0 to just
+   know the current value of the counter.  The current transaction is
+   *never* interrupted as long as this counter is positive. */
+long stm_atomic(long delta);
+
+
 /* callback: get the size of an object */
 extern size_t stmcb_size(gcptr);
 
diff --git a/c4/test/support.py b/c4/test/support.py
--- a/c4/test/support.py
+++ b/c4/test/support.py
@@ -62,6 +62,7 @@
     void stm_begin_inevitable_transaction(void);
     void stm_set_transaction_length(long length_max);
     _Bool stm_should_break_transaction(void);
+    long stm_atomic(long delta);
 
     /* extra non-public code */
     void printfcolor(char *msg);
diff --git a/c4/test/test_atomic.py b/c4/test/test_atomic.py
--- a/c4/test/test_atomic.py
+++ b/c4/test/test_atomic.py
@@ -31,3 +31,28 @@
             should_br[i + 1] = should_break_transaction()
     #
     assert should_br == [False, False, False, False, True, True, True]
+
+def test_stm_atomic():
+    assert lib.stm_atomic(0) == 0
+    x = lib.stm_atomic(+1)
+    assert x == 1
+    x = lib.stm_atomic(+1)
+    assert x == 2
+    x = lib.stm_atomic(-1)
+    assert x == 1
+    x = lib.stm_atomic(0)
+    assert x == 1
+    x = lib.stm_atomic(-1)
+    assert x == 0
+
+def test_transaction_atomic_mode():
+    assert lib.stm_in_transaction()
+    lib.stm_commit_transaction()
+    assert not lib.stm_in_transaction()
+    lib.stm_begin_inevitable_transaction()
+    assert lib.stm_in_transaction()
+    lib.stm_atomic(+1)
+    lib.stm_commit_transaction()
+    assert lib.stm_in_transaction()
+    lib.stm_begin_inevitable_transaction()
+    lib.stm_atomic(-1)

From noreply at buildbot.pypy.org  Sun Jun 30 19:58:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 19:58:32 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Import stmgc/59eb9a85c3d1.
Message-ID: <20130630175832.3F8D71C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65124:674cdae07936
Date: 2013-06-30 19:49 +0200
http://bitbucket.org/pypy/pypy/changeset/674cdae07936/

Log:	Import stmgc/59eb9a85c3d1.

diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
--- a/rpython/translator/stm/src_stm/et.c
+++ b/rpython/translator/stm/src_stm/et.c
@@ -87,6 +87,8 @@
   gcptr P = G;
   revision_t v;
 
+  d->count_reads++;
+
  restart_all:
   if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED)
     {
@@ -902,6 +904,15 @@
   d->reads_size_limit = d->atomic ? ULONG_MAX : d->reads_size_limit_nonatomic;
 }
 
+long stm_atomic(long delta)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  d->atomic += delta;
+  assert(d->atomic >= 0);
+  update_reads_size_limit(d);
+  return d->atomic;
+}
+
 static void init_transaction(struct tx_descriptor *d)
 {
   assert(d->active == 0);
diff --git a/rpython/translator/stm/src_stm/gcpage.c b/rpython/translator/stm/src_stm/gcpage.c
--- a/rpython/translator/stm/src_stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/gcpage.c
@@ -321,13 +321,20 @@
 
 static void mark_roots(gcptr *root, gcptr *end)
 {
-    //assert(*root == END_MARKER);
-    //root++;
+    assert(*root == END_MARKER_ON);
+    root++;
+
     while (root != end) {
-        gcptr o = *root;
-        visit(root);
-        dprintf(("visit stack root: %p -> %p\n", o, *root));
-        (void)o;   /* silence "warning: unused variable 'o'" */
+        gcptr item = *root;
+        if (((revision_t)item) & ~((revision_t)END_MARKER_OFF |
+                                   (revision_t)END_MARKER_ON)) {
+            /* 'item' is a regular, non-null pointer */
+            visit(root);
+            dprintf(("visit stack root: %p -> %p\n", item, *root));
+        }
+        else if (item == END_MARKER_OFF) {
+            *root = END_MARKER_ON;
+        }
         root++;
     }
 }
diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c
--- a/rpython/translator/stm/src_stm/nursery.c
+++ b/rpython/translator/stm/src_stm/nursery.c
@@ -324,12 +324,30 @@
 
 static void mark_young_roots(struct tx_descriptor *d)
 {
-    gcptr *root = d->shadowstack;
+    /* we walk the shadowstack from the end, replacing any END_MARKER_OFF
+       found with END_MARKER_ON.  When we reach an END_MARKER_ON, we know
+       that we have already seen the rest of the stack in the previous
+       nursery collection, so we stop.
+    */
     gcptr *end = *d->shadowstack_end_ref;
 
-    /* XXX use a way to avoid walking all roots again and again */
-    for (; root != end; root++) {
-        visit_if_young(root);
+    while (1) {
+        assert(end > d->shadowstack);
+        gcptr item = *--end;
+
+        if (((revision_t)item) & ~((revision_t)END_MARKER_OFF |
+                                   (revision_t)END_MARKER_ON)) {
+            /* 'item' is a regular, non-null pointer */
+            visit_if_young(end);
+        }
+        else if (item != NULL) {
+            if (item == END_MARKER_OFF)
+                *end = END_MARKER_ON;
+            else {
+                assert(item == END_MARKER_ON);
+                break;
+            }
+        }
     }
 }
 
diff --git a/rpython/translator/stm/src_stm/nursery.h b/rpython/translator/stm/src_stm/nursery.h
--- a/rpython/translator/stm/src_stm/nursery.h
+++ b/rpython/translator/stm/src_stm/nursery.h
@@ -19,6 +19,9 @@
 # error "GC_NURSERY must be a multiple of GC_NURSERY_SECTION"
 #endif
 
+#define END_MARKER_OFF  ((gcptr) 16)
+#define END_MARKER_ON   ((gcptr) 24)
+
 
 #define NURSERY_FIELDS_DECL                                             \
     /* the nursery */                                                   \
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-1ba74b051c7b
+59eb9a85c3d1
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -58,7 +58,8 @@
 /* start a new transaction, calls callback(), and when it returns
    finish that transaction.  callback() is called with the 'arg'
    provided, and with a retry_counter number.  Must save roots around
-   this call. */
+   this call.  The callback() is called repeatedly as long as it
+   returns a value > 0. */
 void stm_perform_transaction(gcptr arg, int (*callback)(gcptr, int));
 
 /* finish the current transaction, start a new one, or turn the current
@@ -71,8 +72,18 @@
 /* debugging: check if we're currently running a transaction or not. */
 int stm_in_transaction(void);
 
-/* change the default transaction length */
+/* change the default transaction length, and ask if now would be a good
+   time to break the transaction (by returning from the 'callback' above
+   with a positive value). */
 void stm_set_transaction_length(long length_max);
+_Bool stm_should_break_transaction(void);
+
+/* change the atomic counter by 'delta' and return the new value.  Used
+   with +1 to enter or with -1 to leave atomic mode, or with 0 to just
+   know the current value of the counter.  The current transaction is
+   *never* interrupted as long as this counter is positive. */
+long stm_atomic(long delta);
+
 
 /* callback: get the size of an object */
 extern size_t stmcb_size(gcptr);
diff --git a/rpython/translator/stm/src_stm/stmsync.c b/rpython/translator/stm/src_stm/stmsync.c
--- a/rpython/translator/stm/src_stm/stmsync.c
+++ b/rpython/translator/stm/src_stm/stmsync.c
@@ -7,6 +7,7 @@
 
 __thread gcptr *stm_shadowstack;
 static unsigned long stm_regular_length_limit = 10000;
+static revision_t sync_required = 0;
 
 void stm_set_transaction_length(long length_max)
 {
@@ -17,6 +18,38 @@
     stm_regular_length_limit = length_max;
 }
 
+_Bool stm_should_break_transaction(void)
+{
+    struct tx_descriptor *d = thread_descriptor;
+
+    /* a single comparison to handle all cases:
+
+     - first, if sync_required == -1, this should return True.
+
+     - if d->atomic, then we should return False.  This is done by
+       forcing reads_size_limit to ULONG_MAX as soon as atomic > 0.
+
+     - otherwise, if is_inevitable(), then we should return True.
+       This is done by forcing both reads_size_limit and
+       reads_size_limit_nonatomic to 0 in that case.
+
+     - finally, the default case: return True if d->count_reads is
+       greater than reads_size_limit == reads_size_limit_nonatomic.
+    */
+#ifdef _GC_DEBUG
+    /* reads_size_limit is ULONG_MAX if d->atomic, or else it is equal to
+       reads_size_limit_nonatomic. */
+    assert(d->reads_size_limit == (d->atomic ? ULONG_MAX :
+                                   d->reads_size_limit_nonatomic));
+    /* if is_inevitable(), reads_size_limit_nonatomic should be 0
+       (and thus reads_size_limit too, if !d->atomic.) */
+    if (d->active == 2)
+        assert(d->reads_size_limit_nonatomic == 0);
+#endif
+
+    return (sync_required | d->count_reads) >= d->reads_size_limit;
+}
+
 static void init_shadowstack(void)
 {
     struct tx_descriptor *d = thread_descriptor;
@@ -26,14 +59,14 @@
     }
     stm_shadowstack = d->shadowstack;
     d->shadowstack_end_ref = &stm_shadowstack;
-    //stm_push_root(END_MARKER);
+    stm_push_root(END_MARKER_ON);
 }
 
 static void done_shadowstack(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    //gcptr x = stm_pop_root();
-    //assert(x == END_MARKER);
+    gcptr x = stm_pop_root();
+    assert(x == END_MARKER_ON);
     assert(stm_shadowstack == d->shadowstack);
     stm_shadowstack = NULL;
     free(d->shadowstack);
@@ -68,8 +101,6 @@
 
 /************************************************************/
 
-static revision_t sync_required = 0;
-
 void stm_perform_transaction(gcptr arg, int (*callback)(gcptr, int))
 {   /* must save roots around this call */
     jmp_buf _jmpbuf;
@@ -78,6 +109,7 @@
     long volatile v_atomic;
 
     stm_push_root(arg);
+    stm_push_root(END_MARKER_OFF);
 
     if (!(v_atomic = thread_descriptor->atomic))
         CommitTransaction();
@@ -99,14 +131,7 @@
     long counter, result;
     counter = v_counter;
     d->atomic = v_atomic;
-    stm_shadowstack = v_saved_value + 1;    /* skip the 'arg', pushed above */
-    //    if (!d->atomic) {
-    //        /* In non-atomic mode, we are now between two transactions.
-    //           It means that in the next transaction's collections we know
-    //           that we won't need to access the shadow stack beyond its
-    //           current position.  So we add an end marker. */
-    //        stm_push_root(END_MARKER);
-    //    }
+    stm_shadowstack = v_saved_value + 2;   /*skip the two values pushed above*/
 
     do {
         v_counter = counter + 1;
@@ -136,7 +161,7 @@
         /* invoke the callback in the new transaction */
         arg = v_saved_value[0];
         result = callback(arg, counter);
-        assert(stm_shadowstack == v_saved_value + 1);
+        assert(stm_shadowstack == v_saved_value + 2);
 
         v_atomic = d->atomic;
         if (!d->atomic)
@@ -155,7 +180,9 @@
         BeginInevitableTransaction();
     }
 
-    stm_pop_root();      /* pop the 'arg' */
+    gcptr x = stm_pop_root();   /* pop the END_MARKER */
+    assert(x == END_MARKER_OFF || x == END_MARKER_ON);
+    stm_pop_root();             /* pop the 'arg' */
     assert(stm_shadowstack == v_saved_value);
 }
 
@@ -240,7 +267,7 @@
        which prevents any other thread from running in a transaction.
        Warning, may block waiting for rwlock_in_transaction while another
        thread runs a major GC itself! */
-    ACCESS_ONCE(sync_required) = 1;
+    ACCESS_ONCE(sync_required) = -1;
     stm_stop_sharedlock();
     start_exclusivelock();
     ACCESS_ONCE(sync_required) = 0;

From noreply at buildbot.pypy.org  Sun Jun 30 19:58:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 19:58:33 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Starting to adapt rpython/rlib.py:
	become_inevitable(), 
Message-ID: <20130630175833.AAFB11C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65125:5e8273640041
Date: 2013-06-30 19:58 +0200
http://bitbucket.org/pypy/pypy/changeset/5e8273640041/

Log:	Starting to adapt rpython/rlib.py: become_inevitable(),
	should_break_transaction()

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -1,24 +1,15 @@
-import thread, weakref
-from rpython.rlib.debug import ll_assert, fatalerror
-from rpython.rlib.objectmodel import keepalive_until_here, specialize
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rposix import get_errno, set_errno
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass, rstr
+from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr, llhelper
-from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
 from rpython.rtyper.extregistry import ExtRegistryEntry
 
 
-def is_inevitable():
-    return we_are_translated() and stmgcintf.StmOperations.is_inevitable()
-
 def become_inevitable():
     llop.stm_become_inevitable(lltype.Void)
 
 def should_break_transaction():
     return we_are_translated() and (
-        stmgcintf.StmOperations.should_break_transaction())
+        llop.stm_should_break_transaction(lltype.Bool))
 
 def set_transaction_length(length):
     stmgcintf.StmOperations.set_transaction_length(length)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -435,6 +435,7 @@
     'stm_pop_root_into':      LLOp(),
     'stm_commit_transaction': LLOp(),
     'stm_begin_inevitable_transaction': LLOp(),
+    'stm_should_break_transaction': LLOp(sideeffects=False),
 
     # __________ address operations __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -597,6 +597,7 @@
     OP_STM_ID = _OP_STM
     OP_STM_COMMIT_TRANSACTION = _OP_STM
     OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM
+    OP_STM_SHOULD_BREAK_TRANSACTION = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -119,6 +119,10 @@
 def stm_begin_inevitable_transaction(funcgen, op):
     return 'stm_begin_inevitable_transaction();'
 
+def stm_should_break_transaction(funcgen, op):
+    result = funcgen.expr(op.result)
+    return '%s = stm_should_break_transaction();' % (result,)
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -79,6 +79,22 @@
         cbuilder.cmdexec('')
         # assert did not crash
 
+    def test_become_inevitable(self):
+        def entry_point(argv):
+            rstm.become_inevitable()
+            return 0
+        t, cbuilder = self.compile(entry_point)
+        cbuilder.cmdexec('')
+        # assert did not crash
+
+    def test_should_break_transaction(self):
+        def entry_point(argv):
+            print '<', int(rstm.should_break_transaction()), '>'
+            return 0
+        t, cbuilder = self.compile(entry_point)
+        data = cbuilder.cmdexec('')
+        assert '< 1 >\n' in data
+
     def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo2.entry_point)
         data, dataerr = cbuilder.cmdexec('4 5000', err=True,

From noreply at buildbot.pypy.org  Sun Jun 30 20:00:28 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 20:00:28 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: set_transaction_length()
Message-ID: <20130630180028.0537D1C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65126:2a4c96f5318f
Date: 2013-06-30 19:59 +0200
http://bitbucket.org/pypy/pypy/changeset/2a4c96f5318f/

Log:	set_transaction_length()

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -12,7 +12,7 @@
         llop.stm_should_break_transaction(lltype.Bool))
 
 def set_transaction_length(length):
-    stmgcintf.StmOperations.set_transaction_length(length)
+    llop.stm_set_transaction_length(lltype.Void, length)
 
 def increment_atomic():
     stmgcintf.StmOperations.add_atomic(+1)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -436,6 +436,7 @@
     'stm_commit_transaction': LLOp(),
     'stm_begin_inevitable_transaction': LLOp(),
     'stm_should_break_transaction': LLOp(sideeffects=False),
+    'stm_set_transaction_length': LLOp(),
 
     # __________ address operations __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -598,6 +598,7 @@
     OP_STM_COMMIT_TRANSACTION = _OP_STM
     OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM
     OP_STM_SHOULD_BREAK_TRANSACTION = _OP_STM
+    OP_STM_SET_TRANSACTION_LENGTH = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -123,6 +123,10 @@
     result = funcgen.expr(op.result)
     return '%s = stm_should_break_transaction();' % (result,)
 
+def stm_set_transaction_length(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    return 'stm_set_transaction_length(%s);' % (arg0,)
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -95,6 +95,14 @@
         data = cbuilder.cmdexec('')
         assert '< 1 >\n' in data
 
+    def test_set_transaction_length(self):
+        def entry_point(argv):
+            rstm.set_transaction_length(123)
+            return 0
+        t, cbuilder = self.compile(entry_point)
+        cbuilder.cmdexec('')
+        # assert did not crash
+
     def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo2.entry_point)
         data, dataerr = cbuilder.cmdexec('4 5000', err=True,

From noreply at buildbot.pypy.org  Sun Jun 30 20:04:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 20:04:51 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: stm_atomic()
Message-ID: <20130630180451.4B3461C0168@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65127:0d53a387555d
Date: 2013-06-30 20:04 +0200
http://bitbucket.org/pypy/pypy/changeset/0d53a387555d/

Log:	stm_atomic()

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -15,13 +15,13 @@
     llop.stm_set_transaction_length(lltype.Void, length)
 
 def increment_atomic():
-    stmgcintf.StmOperations.add_atomic(+1)
+    llop.stm_change_atomic(lltype.Signed, +1)
 
 def decrement_atomic():
-    stmgcintf.StmOperations.add_atomic(-1)
+    llop.stm_change_atomic(lltype.Signed, -1)
 
 def is_atomic():
-    return stmgcintf.StmOperations.get_atomic()
+    return llop.stm_get_atomic(lltype.Signed, 0)
 
 def abort_info_push(instance, fieldnames):
     "Special-cased below."
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -437,6 +437,8 @@
     'stm_begin_inevitable_transaction': LLOp(),
     'stm_should_break_transaction': LLOp(sideeffects=False),
     'stm_set_transaction_length': LLOp(),
+    'stm_change_atomic':      LLOp(),
+    'stm_get_atomic':         LLOp(sideeffects=False),
 
     # __________ address operations __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -599,6 +599,8 @@
     OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM
     OP_STM_SHOULD_BREAK_TRANSACTION = _OP_STM
     OP_STM_SET_TRANSACTION_LENGTH = _OP_STM
+    OP_STM_CHANGE_ATOMIC = _OP_STM
+    OP_STM_GET_ATOMIC = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -127,6 +127,14 @@
     arg0 = funcgen.expr(op.args[0])
     return 'stm_set_transaction_length(%s);' % (arg0,)
 
+def stm_change_atomic(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    return 'stm_atomic(%s);' % (arg0,)
+
+def stm_get_atomic(funcgen, op):
+    result = funcgen.expr(op.result)
+    return '%s = stm_atomic(0);' % (result,)
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -103,6 +103,18 @@
         cbuilder.cmdexec('')
         # assert did not crash
 
+    def test_stm_atomic(self):
+        def entry_point(argv):
+            assert not rstm.is_atomic()
+            rstm.increment_atomic()
+            assert rstm.is_atomic()
+            rstm.decrement_atomic()
+            assert not rstm.is_atomic()
+            return 0
+        t, cbuilder = self.compile(entry_point)
+        cbuilder.cmdexec('')
+        # assert did not crash
+
     def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo2.entry_point)
         data, dataerr = cbuilder.cmdexec('4 5000', err=True,

From noreply at buildbot.pypy.org  Sun Jun 30 21:14:44 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Jun 2013 21:14:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
	fix for shape modification in view()
Message-ID: <20130630191444.D97C21C0634@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r65128:12ed8f9fbc83
Date: 2013-06-30 22:13 +0300
http://bitbucket.org/pypy/pypy/changeset/12ed8f9fbc83/

Log:	test, fix for shape modification in view()

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -630,7 +630,7 @@
         old_itemsize = self.get_dtype().get_size()
         new_itemsize = dtype.get_size()
         impl = self.implementation
-        new_shape = self.get_shape()
+        new_shape = self.get_shape()[:]
         dims = len(new_shape)
         if dims == 0:
             # Cannot resize scalars
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1411,16 +1411,17 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
-    def test_ndarray_view(self):
+    def test_view(self):
         from numpypy import array, int8, int16, dtype
-        x = array([(1, 2)], dtype=[('a', int8), ('b', int8)])
+        x = array((1, 2), dtype=int8)
+        assert x.shape == (2,)
         y = x.view(dtype=int16)
-        print y,y.shape
+        assert x.shape == (2,)
         assert y[0] == 513
         assert y.dtype == dtype('int16')
         y[0] = 670
-        assert x['a'] == -98
-        assert x['b'] == 2
+        assert x[0] == -98
+        assert x[1] == 2
         f = array([1000, -1234], dtype='i4')
         nnp = self.non_native_prefix
         d = f.view(dtype=nnp + 'i4')

From noreply at buildbot.pypy.org  Sun Jun 30 21:33:24 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 21:33:24 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Unused arg
Message-ID: <20130630193324.4DC921C0634@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65129:2588a015d619
Date: 2013-06-30 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/2588a015d619/

Log:	Unused arg

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -21,7 +21,7 @@
     llop.stm_change_atomic(lltype.Signed, -1)
 
 def is_atomic():
-    return llop.stm_get_atomic(lltype.Signed, 0)
+    return llop.stm_get_atomic(lltype.Signed)
 
 def abort_info_push(instance, fieldnames):
     "Special-cased below."

From noreply at buildbot.pypy.org  Sun Jun 30 21:33:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Jun 2013 21:33:25 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: threadlocals, and stm_ptr_eq()
Message-ID: <20130630193325.C51EC1C0634@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r65130:bf3b552315d6
Date: 2013-06-30 21:32 +0200
http://bitbucket.org/pypy/pypy/changeset/bf3b552315d6/

Log:	threadlocals, and stm_ptr_eq()

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -163,6 +163,7 @@
 
     def __init__(self, Cls):
         "NOT_RPYTHON: must be prebuilt"
+        import thread
         self.Cls = Cls
         self.local = thread._local()      # <- NOT_RPYTHON
         self.unique_id = ThreadLocalReference._COUNT
@@ -174,6 +175,8 @@
     @specialize.arg(0)
     def get(self):
         if we_are_translated():
+            from rpython.rtyper.lltypesystem import rclass
+            from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
             ptr = llop.stm_threadlocalref_get(rclass.OBJECTPTR, self.unique_id)
             return cast_base_ptr_to_instance(self.Cls, ptr)
         else:
@@ -183,6 +186,7 @@
     def set(self, value):
         assert isinstance(value, self.Cls) or value is None
         if we_are_translated():
+            from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr
             ptr = cast_instance_to_base_ptr(value)
             llop.stm_threadlocalref_set(lltype.Void, self.unique_id, ptr)
         else:
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -429,6 +429,7 @@
     'stm_minor_collect':      LLOp(),
     'stm_major_collect':      LLOp(),
     'stm_get_tid':            LLOp(canfold=True),
+    'stm_ptr_eq':             LLOp(canfold=True),
     'stm_id':                 LLOp(sideeffects=False),
     'stm_hash':               LLOp(sideeffects=False),
     'stm_push_root':          LLOp(),
@@ -440,6 +441,11 @@
     'stm_change_atomic':      LLOp(),
     'stm_get_atomic':         LLOp(sideeffects=False),
 
+    'stm_threadlocalref_get': LLOp(sideeffects=False),
+    'stm_threadlocalref_set': LLOp(),
+    'stm_threadlocal_get':    LLOp(sideeffects=False),
+    'stm_threadlocal_set':    LLOp(),
+
     # __________ address operations __________
 
     'boehm_malloc':         LLOp(),
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -601,6 +601,8 @@
     OP_STM_SET_TRANSACTION_LENGTH = _OP_STM
     OP_STM_CHANGE_ATOMIC = _OP_STM
     OP_STM_GET_ATOMIC = _OP_STM
+    OP_STM_THREADLOCAL_GET = _OP_STM
+    OP_STM_THREADLOCAL_SET = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -71,7 +71,8 @@
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
     result = funcgen.expr(op.result)
-    return '%s = stm_pointer_equal(%s, %s);' % (result, arg0, arg1)
+    return '%s = stm_pointer_equal((gcptr)%s, (gcptr)%s);' % (
+        result, arg0, arg1)
 
 def stm_become_inevitable(funcgen, op):
     try:
@@ -135,6 +136,15 @@
     result = funcgen.expr(op.result)
     return '%s = stm_atomic(0);' % (result,)
 
+def stm_threadlocal_get(funcgen, op):
+    result = funcgen.expr(op.result)
+    return '%s = (%s)stm_thread_local_obj;' % (
+        result, cdecl(funcgen.lltypename(op.result), ''))
+
+def stm_threadlocal_set(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    return 'stm_thread_local_obj = (gcptr)%s;' % (arg0,)
+
 
 def op_stm(funcgen, op):
     func = globals()[op.opname]
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -215,12 +215,6 @@
             assert t.get() is None
             t.set(x)
             assert t.get() is x
-            assert llop.stm_threadlocalref_llcount(lltype.Signed) == 1
-            p = llop.stm_threadlocalref_lladdr(llmemory.Address, 0)
-            adr = p.address[0]
-            adr2 = cast_instance_to_base_ptr(x)
-            adr2 = llmemory.cast_ptr_to_adr(adr2)
-            assert adr == adr2
             print "ok"
             return 0
         t, cbuilder = self.compile(main)
diff --git a/rpython/translator/stm/threadlocalref.py b/rpython/translator/stm/threadlocalref.py
--- a/rpython/translator/stm/threadlocalref.py
+++ b/rpython/translator/stm/threadlocalref.py
@@ -1,10 +1,13 @@
-from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.translator.unsimplify import varoftype
+from rpython.annotator import model as annmodel
+from rpython.rtyper import annlowlevel
+from rpython.rtyper.lltypesystem import lltype, rclass
+from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.flowspace.model import SpaceOperation, Constant
 
 
-def transform_tlref(graphs):
+def transform_tlref(t):
     ids = set()
+    graphs = t.graphs
     #
     for graph in graphs:
         for block in graph.iterblocks():
@@ -13,23 +16,35 @@
                 if (op.opname == 'stm_threadlocalref_set' or
                     op.opname == 'stm_threadlocalref_get'):
                     ids.add(op.args[0].value)
+    if not ids:
+        return
     #
     ids = sorted(ids)
-    ARRAY = lltype.FixedSizeArray(llmemory.Address, len(ids))
-    S = lltype.Struct('THREADLOCALREF', ('ptr', ARRAY),
-                      hints={'stm_thread_local': True})
-    ll_threadlocalref = lltype.malloc(S, immortal=True)
-    c_threadlocalref = Constant(ll_threadlocalref, lltype.Ptr(S))
-    c_fieldname = Constant('ptr', lltype.Void)
-    c_null = Constant(llmemory.NULL, llmemory.Address)
+    total = len(ids)
+    ARRAY = lltype.GcArray(rclass.OBJECTPTR)
     #
-    def getaddr(v_num, v_result):
-        v_array = varoftype(lltype.Ptr(ARRAY))
-        ops = [
-            SpaceOperation('getfield', [c_threadlocalref, c_fieldname],
-                           v_array),
-            SpaceOperation('direct_ptradd', [v_array, v_num], v_result)]
-        return ops
+    def ll_threadlocalref_get(index):
+        array = llop.stm_threadlocal_get(lltype.Ptr(ARRAY))
+        if not array:
+            return lltype.nullptr(rclass.OBJECTPTR.TO)
+        else:
+            return array[index]
+    #
+    def ll_threadlocalref_set(index, newvalue):
+        array = llop.stm_threadlocal_get(lltype.Ptr(ARRAY))
+        if not array:
+            array = lltype.malloc(ARRAY, total)
+            llop.stm_threadlocal_set(lltype.Void, array)
+        array[index] = newvalue
+    #
+    annhelper = annlowlevel.MixLevelHelperAnnotator(t.rtyper)
+    s_Int = annmodel.SomeInteger()
+    s_Ptr = annmodel.SomePtr(rclass.OBJECTPTR)
+    c_getter_ptr = annhelper.constfunc(ll_threadlocalref_get,
+                                       [s_Int], s_Ptr)
+    c_setter_ptr = annhelper.constfunc(ll_threadlocalref_set,
+                                       [s_Int, s_Ptr], annmodel.s_None)
+    annhelper.finish()
     #
     for graph in graphs:
         for block in graph.iterblocks():
@@ -38,26 +53,17 @@
                 if op.opname == 'stm_threadlocalref_set':
                     id = op.args[0].value
                     c_num = Constant(ids.index(id), lltype.Signed)
-                    v_addr = varoftype(lltype.Ptr(ARRAY))
-                    ops = getaddr(c_num, v_addr)
-                    ops.append(SpaceOperation('stm_threadlocalref_llset',
-                                              [v_addr, op.args[1]],
-                                              op.result))
+                    ops = [
+                        SpaceOperation('direct_call', [c_setter_ptr, c_num,
+                                                       op.args[1]],
+                                       op.result)
+                        ]
                     block.operations[i:i+1] = ops
                 elif op.opname == 'stm_threadlocalref_get':
                     id = op.args[0].value
                     c_num = Constant(ids.index(id), lltype.Signed)
-                    v_array = varoftype(lltype.Ptr(ARRAY))
                     ops = [
-                        SpaceOperation('getfield', [c_threadlocalref,
-                                                    c_fieldname],
-                                       v_array),
-                        SpaceOperation('getarrayitem', [v_array, c_num],
-                                       op.result)]
+                        SpaceOperation('direct_call', [c_getter_ptr, c_num],
+                                       op.result)
+                        ]
                     block.operations[i:i+1] = ops
-                elif op.opname == 'stm_threadlocalref_lladdr':
-                    block.operations[i:i+1] = getaddr(op.args[0], op.result)
-                elif op.opname == 'stm_threadlocalref_llcount':
-                    c_count = Constant(len(ids), lltype.Signed)
-                    op = SpaceOperation('same_as', [c_count], op.result)
-                    block.operations[i] = op
diff --git a/rpython/translator/stm/transform2.py b/rpython/translator/stm/transform2.py
--- a/rpython/translator/stm/transform2.py
+++ b/rpython/translator/stm/transform2.py
@@ -40,7 +40,7 @@
 
     def transform_threadlocalref(self):
         from rpython.translator.stm.threadlocalref import transform_tlref
-        transform_tlref(self.translator.graphs)
+        transform_tlref(self.translator)
 
     def start_log(self):
         from rpython.translator.c.support import log