[pypy-commit] pypy win64-stage1: Merge with default (2 mon)

ctismer noreply at buildbot.pypy.org
Tue Mar 13 00:05:02 CET 2012


Author: Christian Tismer <tismer at stackless.com>
Branch: win64-stage1
Changeset: r53377:0382f7ffd064
Date: 2012-03-12 16:04 -0700
http://bitbucket.org/pypy/pypy/changeset/0382f7ffd064/

Log:	Merge with default (2 mon)

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -27,7 +27,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2011
+PyPy Copyright holders 2003-2012
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -231,8 +231,10 @@
 sqlite.sqlite3_result_text.argtypes = [c_void_p, c_char_p, c_int, c_void_p]
 sqlite.sqlite3_result_text.restype = None
 
-sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int]
-sqlite.sqlite3_enable_load_extension.restype = c_int
+HAS_LOAD_EXTENSION = hasattr(sqlite, "sqlite3_enable_load_extension")
+if HAS_LOAD_EXTENSION:
+    sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int]
+    sqlite.sqlite3_enable_load_extension.restype = c_int
 
 ##########################################
 # END Wrapped SQLite C API and constants
@@ -708,13 +710,14 @@
         from sqlite3.dump import _iterdump
         return _iterdump(self)
 
-    def enable_load_extension(self, enabled):
-        self._check_thread()
-        self._check_closed()
+    if HAS_LOAD_EXTENSION:
+        def enable_load_extension(self, enabled):
+            self._check_thread()
+            self._check_closed()
 
-        rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled))
-        if rc != SQLITE_OK:
-            raise OperationalError("Error enabling load extension")
+            rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled))
+            if rc != SQLITE_OK:
+                raise OperationalError("Error enabling load extension")
 
 DML, DQL, DDL = range(3)
 
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -180,7 +180,12 @@
         if name is None:
             name = pyobj.func_name
         if signature is None:
-            signature = cpython_code_signature(pyobj.func_code)
+            if hasattr(pyobj, '_generator_next_method_of_'):
+                from pypy.interpreter.argument import Signature
+                signature = Signature(['entry'])     # haaaaaack
+                defaults = ()
+            else:
+                signature = cpython_code_signature(pyobj.func_code)
         if defaults is None:
             defaults = pyobj.func_defaults
         self.name = name
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -175,15 +175,15 @@
 RPython
 =================
 
-RPython Definition, not
------------------------
+RPython Definition
+------------------
 
-The list and exact details of the "RPython" restrictions are a somewhat
-evolving topic.  In particular, we have no formal language definition
-as we find it more practical to discuss and evolve the set of
-restrictions while working on the whole program analysis.  If you
-have any questions about the restrictions below then please feel
-free to mail us at pypy-dev at codespeak net.
+RPython is a restricted subset of Python that is amenable to static analysis.
+Although there are additions to the language and some things might surprisingly
+work, this is a rough list of restrictions that should be considered. Note
+that there are tons of special cased restrictions that you'll encounter
+as you go. The exact definition is "RPython is everything that our translation
+toolchain can accept" :)
 
 .. _`wrapped object`: coding-guide.html#wrapping-rules
 
@@ -198,7 +198,7 @@
   contain both a string and a int must be avoided.  It is allowed to
   mix None (basically with the role of a null pointer) with many other
   types: `wrapped objects`, class instances, lists, dicts, strings, etc.
-  but *not* with int and floats.
+  but *not* with int, floats or tuples.
 
 **constants**
 
@@ -209,9 +209,12 @@
   have this restriction, so if you need mutable global state, store it
   in the attributes of some prebuilt singleton instance.
 
+
+
 **control structures**
 
-  all allowed but yield, ``for`` loops restricted to builtin types
+  all allowed, ``for`` loops restricted to builtin types, generators
+  very restricted.
 
 **range**
 
@@ -226,7 +229,8 @@
 
 **generators**
 
-  generators are not supported.
+  generators are supported, but their exact scope is very limited. you can't
+  merge two different generator in one control point.
 
 **exceptions**
 
@@ -245,22 +249,27 @@
 
 **strings**
 
-  a lot of, but not all string methods are supported.  Indexes can be
+  a lot of, but not all string methods are supported and those that are
+  supported, not necesarilly accept all arguments.  Indexes can be
   negative.  In case they are not, then you get slightly more efficient
   code if the translator can prove that they are non-negative.  When
   slicing a string it is necessary to prove that the slice start and
-  stop indexes are non-negative.
+  stop indexes are non-negative. There is no implicit str-to-unicode cast
+  anywhere.
 
 **tuples**
 
   no variable-length tuples; use them to store or return pairs or n-tuples of
-  values. Each combination of types for elements and length constitute a separate
-  and not mixable type.
+  values. Each combination of types for elements and length constitute
+  a separate and not mixable type.
 
 **lists**
 
   lists are used as an allocated array.  Lists are over-allocated, so list.append()
-  is reasonably fast.  Negative or out-of-bound indexes are only allowed for the
+  is reasonably fast. However, if you use a fixed-size list, the code
+  is more efficient. Annotator can figure out most of the time that your
+  list is fixed-size, even when you use list comprehension.
+  Negative or out-of-bound indexes are only allowed for the
   most common operations, as follows:
 
   - *indexing*:
@@ -287,16 +296,14 @@
 
 **dicts**
 
-  dicts with a unique key type only, provided it is hashable. 
-  String keys have been the only allowed key types for a while, but this was generalized. 
-  After some re-optimization,
-  the implementation could safely decide that all string dict keys should be interned.
+  dicts with a unique key type only, provided it is hashable. Custom
+  hash functions and custom equality will not be honored.
+  Use ``pypy.rlib.objectmodel.r_dict`` for custom hash functions.
 
 
 **list comprehensions**
 
-  may be used to create allocated, initialized arrays.
-  After list over-allocation was introduced, there is no longer any restriction.
+  May be used to create allocated, initialized arrays.
 
 **functions**
 
@@ -334,9 +341,7 @@
 
 **objects**
 
-  in PyPy, wrapped objects are borrowed from the object space. Just like
-  in CPython, code that needs e.g. a dictionary can use a wrapped dict
-  and the object space operations on it.
+  Normal rules apply.
 
 This layout makes the number of types to take care about quite limited.
 
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -98,7 +98,6 @@
         "Abstract. Get the expected number of locals."
         raise TypeError, "abstract"
 
-    @jit.dont_look_inside
     def fast2locals(self):
         # Copy values from the fastlocals to self.w_locals
         if self.w_locals is None:
@@ -112,7 +111,6 @@
                 w_name = self.space.wrap(name)
                 self.space.setitem(self.w_locals, w_name, w_value)
 
-    @jit.dont_look_inside
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
         assert self.w_locals is not None
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -619,7 +619,8 @@
                                                   self.descr_reqcls,
                                                   args)
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
@@ -655,7 +656,8 @@
                                                   self.descr_reqcls,
                                                   args)
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
@@ -674,7 +676,8 @@
                                                   self.descr_reqcls,
                                                   args.prepend(w_obj))
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
@@ -690,7 +693,8 @@
             raise OperationError(space.w_SystemError,
                                  space.wrap("unexpected DescrMismatch error"))
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
@@ -708,7 +712,8 @@
                                            self.descr_reqcls,
                                            Arguments(space, [w1]))
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
@@ -726,7 +731,8 @@
                                            self.descr_reqcls,
                                            Arguments(space, [w1, w2]))
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
@@ -744,7 +750,8 @@
                                            self.descr_reqcls,
                                            Arguments(space, [w1, w2, w3]))
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
@@ -763,7 +770,8 @@
                                            Arguments(space,
                                                      [w1, w2, w3, w4]))
         except Exception, e:
-            raise self.handle_exception(space, e)
+            self.handle_exception(space, e)
+            w_result = None
         if w_result is None:
             w_result = space.w_None
         return w_result
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -39,6 +39,7 @@
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 from pypy.rlib.rarithmetic import intmask
+from pypy.rlib.objectmodel import compute_unique_id
 
 # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0,
 # better safe than sorry
@@ -58,7 +59,8 @@
         self.is_guard_not_invalidated = is_guard_not_invalidated
 
 DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed),
-                              ('bridge', lltype.Signed), # 0 or 1
+                              ('type', lltype.Char), # 'b'ridge, 'l'abel or
+                                                     # 'e'ntry point
                               ('number', lltype.Signed))
 
 class Assembler386(object):
@@ -147,12 +149,15 @@
     def finish_once(self):
         if self._debug:
             debug_start('jit-backend-counts')
-            for struct in self.loop_run_counters:
-                if struct.bridge:
-                    prefix = 'bridge '
+            for i in range(len(self.loop_run_counters)):
+                struct = self.loop_run_counters[i]
+                if struct.type == 'l':
+                    prefix = 'TargetToken(%d)' % struct.number
+                elif struct.type == 'b':
+                    prefix = 'bridge ' + str(struct.number)
                 else:
-                    prefix = 'loop '
-                debug_print(prefix + str(struct.number) + ':' + str(struct.i))
+                    prefix = 'entry ' + str(struct.number)
+                debug_print(prefix + ':' + str(struct.i))
             debug_stop('jit-backend-counts')
 
     def _build_float_constants(self):
@@ -422,8 +427,8 @@
 
         self.setup(looptoken)
         if log:
-            self._register_counter(False, looptoken.number)
-            operations = self._inject_debugging_code(looptoken, operations)
+            operations = self._inject_debugging_code(looptoken, operations,
+                                                     'e', looptoken.number)
 
         regalloc = RegAlloc(self, self.cpu.translate_support_code)
         #
@@ -489,8 +494,8 @@
 
         self.setup(original_loop_token)
         if log:
-            self._register_counter(True, descr_number)
-            operations = self._inject_debugging_code(faildescr, operations)
+            operations = self._inject_debugging_code(faildescr, operations,
+                                                     'b', descr_number)
 
         arglocs = self.rebuild_faillocs_from_descr(failure_recovery)
         if not we_are_translated():
@@ -597,17 +602,21 @@
         return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
 
-    def _register_counter(self, bridge, number):
-        if self._debug:
-            # YYY very minor leak -- we need the counters to stay alive
-            # forever, just because we want to report them at the end
-            # of the process
-            struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
-                                   track_allocation=False)
-            struct.i = 0
-            struct.bridge = int(bridge)
+    def _register_counter(self, tp, number, token):
+        # YYY very minor leak -- we need the counters to stay alive
+        # forever, just because we want to report them at the end
+        # of the process
+        struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
+                               track_allocation=False)
+        struct.i = 0
+        struct.type = tp
+        if tp == 'b' or tp == 'e':
             struct.number = number
-            self.loop_run_counters.append(struct)
+        else:
+            assert token
+            struct.number = compute_unique_id(token)
+        self.loop_run_counters.append(struct)            
+        return struct
 
     def _find_failure_recovery_bytecode(self, faildescr):
         adr_jump_offset = faildescr._x86_adr_jump_offset
@@ -651,27 +660,36 @@
             targettoken._x86_loop_code += rawstart
         self.target_tokens_currently_compiling = None
 
+    def _append_debugging_code(self, operations, tp, number, token):
+        counter = self._register_counter(tp, number, token)
+        c_adr = ConstInt(rffi.cast(lltype.Signed, counter))
+        box = BoxInt()
+        box2 = BoxInt()
+        ops = [ResOperation(rop.GETFIELD_RAW, [c_adr],
+                            box, descr=self.debug_counter_descr),
+               ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2),
+               ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
+                            None, descr=self.debug_counter_descr)]
+        operations.extend(ops)
+        
     @specialize.argtype(1)
-    def _inject_debugging_code(self, looptoken, operations):
+    def _inject_debugging_code(self, looptoken, operations, tp, number):
         if self._debug:
             # before doing anything, let's increase a counter
             s = 0
             for op in operations:
                 s += op.getopnum()
             looptoken._x86_debug_checksum = s
-            c_adr = ConstInt(rffi.cast(lltype.Signed,
-                                       self.loop_run_counters[-1]))
-            box = BoxInt()
-            box2 = BoxInt()
-            ops = [ResOperation(rop.GETFIELD_RAW, [c_adr],
-                                box, descr=self.debug_counter_descr),
-                   ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2),
-                   ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
-                                None, descr=self.debug_counter_descr)]
-            if operations[0].getopnum() == rop.LABEL:
-                operations = [operations[0]] + ops + operations[1:]
-            else:
-                operations =  ops + operations
+
+            newoperations = []
+            self._append_debugging_code(newoperations, tp, number,
+                                        None)
+            for op in operations:
+                newoperations.append(op)
+                if op.getopnum() == rop.LABEL:
+                    self._append_debugging_code(newoperations, 'l', number,
+                                                op.getdescr())
+            operations = newoperations
         return operations
 
     def _assemble(self, regalloc, operations):
diff --git a/pypy/jit/backend/x86/jump.py b/pypy/jit/backend/x86/jump.py
--- a/pypy/jit/backend/x86/jump.py
+++ b/pypy/jit/backend/x86/jump.py
@@ -17,7 +17,10 @@
         key = src._getregkey()
         if key in srccount:
             if key == dst_locations[i]._getregkey():
-                srccount[key] = -sys.maxint     # ignore a move "x = x"
+                # ignore a move "x = x"
+                # setting any "large enough" negative value is ok, but
+                # be careful of overflows, don't use -sys.maxint
+                srccount[key] = -len(dst_locations) - 1
                 pending_dests -= 1
             else:
                 srccount[key] += 1
diff --git a/pypy/jit/backend/x86/test/test_jump.py b/pypy/jit/backend/x86/test/test_jump.py
--- a/pypy/jit/backend/x86/test/test_jump.py
+++ b/pypy/jit/backend/x86/test/test_jump.py
@@ -385,3 +385,32 @@
             assert read(loc, WORD) == src_values1[i]
         for i, loc in enumerate(dst_locations2):
             assert read(loc, 8) == src_values2[i]
+
+
+def test_overflow_bug():
+    CASE = [
+        (-144, -248),   # \ cycle
+        (-248, -144),   # /
+        (-488, -416),   # \ two usages of -488
+        (-488, -480),   # /
+        (-488, -488),   # - one self-application of -488
+        ]
+    class FakeAssembler:
+        def regalloc_mov(self, src, dst):
+            print "mov", src, dst
+        def regalloc_push(self, x):
+            print "push", x
+        def regalloc_pop(self, x):
+            print "pop", x
+        def regalloc_immedmem2mem(self, x, y):
+            print "?????????????????????????"
+    def main():
+        srclocs = [StackLoc(9999, x, 'i') for x,y in CASE]
+        dstlocs = [StackLoc(9999, y, 'i') for x,y in CASE]
+        remap_frame_layout(FakeAssembler(), srclocs, dstlocs, eax)
+    # it works when run directly
+    main()
+    # but it used to crash when translated,
+    # because of a -sys.maxint-2 overflowing to sys.maxint
+    from pypy.rpython.test.test_llinterp import interpret
+    interpret(main, [])
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -420,8 +420,8 @@
         debug._log = None
         #
         assert ops_offset is looptoken._x86_ops_offset
-        # getfield_raw/int_add/setfield_raw + ops + None
-        assert len(ops_offset) == 3 + len(operations) + 1
+        # 2*(getfield_raw/int_add/setfield_raw) + ops + None
+        assert len(ops_offset) == 2*3 + len(operations) + 1
         assert (ops_offset[operations[0]] <=
                 ops_offset[operations[1]] <=
                 ops_offset[operations[2]] <=
@@ -519,16 +519,23 @@
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
 
+        targettoken, preambletoken = TargetToken(), TargetToken()
         loop = """
         [i0]
-        label(i0, descr=targettoken)
+        label(i0, descr=preambletoken)
         debug_merge_point('xyz', 0)
         i1 = int_add(i0, 1)
         i2 = int_ge(i1, 10)
         guard_false(i2) []
-        jump(i1, descr=targettoken)
+        label(i1, descr=targettoken)
+        debug_merge_point('xyz', 0)
+        i11 = int_add(i1, 1)
+        i12 = int_ge(i11, 10)
+        guard_false(i12) []
+        jump(i11, descr=targettoken)
         """
-        ops = parse(loop, namespace={'targettoken': TargetToken()})
+        ops = parse(loop, namespace={'targettoken': targettoken,
+                                     'preambletoken': preambletoken})
         debug._log = dlog = debug.DebugLog()
         try:
             self.cpu.assembler.set_debug(True)
@@ -537,11 +544,18 @@
             self.cpu.execute_token(looptoken, 0)
             # check debugging info
             struct = self.cpu.assembler.loop_run_counters[0]
-            assert struct.i == 10
+            assert struct.i == 1
+            struct = self.cpu.assembler.loop_run_counters[1]
+            assert struct.i == 1
+            struct = self.cpu.assembler.loop_run_counters[2]
+            assert struct.i == 9
             self.cpu.finish_once()
         finally:
             debug._log = None
-        assert ('jit-backend-counts', [('debug_print', 'loop -1:10')]) in dlog
+        l0 = ('debug_print', 'entry -1:1')
+        l1 = ('debug_print', preambletoken.repr_of_descr() + ':1')
+        l2 = ('debug_print', targettoken.repr_of_descr() + ':9')
+        assert ('jit-backend-counts', [l0, l1, l2]) in dlog
 
     def test_debugger_checksum(self):
         loop = """
diff --git a/pypy/jit/backend/x86/test/test_zrpy_platform.py b/pypy/jit/backend/x86/test/test_zrpy_platform.py
--- a/pypy/jit/backend/x86/test/test_zrpy_platform.py
+++ b/pypy/jit/backend/x86/test/test_zrpy_platform.py
@@ -74,8 +74,8 @@
     myjitdriver = jit.JitDriver(greens = [], reds = ['n'])
 
     def entrypoint(argv):
-        myjitdriver.set_param('threshold', 2)
-        myjitdriver.set_param('trace_eagerness', 0)
+        jit.set_param(myjitdriver, 'threshold', 2)
+        jit.set_param(myjitdriver, 'trace_eagerness', 0)
         n = 16
         while n > 0:
             myjitdriver.can_enter_jit(n=n)
diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py
--- a/pypy/jit/codewriter/call.py
+++ b/pypy/jit/codewriter/call.py
@@ -42,8 +42,7 @@
         except AttributeError:
             pass
 
-        def is_candidate(graph):
-            return policy.look_inside_graph(graph)
+        is_candidate = policy.look_inside_graph
 
         assert len(self.jitdrivers_sd) > 0
         todo = [jd.portal_graph for jd in self.jitdrivers_sd]
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -162,7 +162,6 @@
 _ll_4_list_setslice = rlist.ll_listsetslice
 _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
 _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
-_ll_1_list_list2fixed = lltypesystem_rlist.ll_list2fixed
 _ll_2_list_inplace_mul = rlist.ll_inplace_mul
 
 _ll_2_list_getitem_foldable = _ll_2_list_getitem
diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py
--- a/pypy/jit/metainterp/compile.py
+++ b/pypy/jit/metainterp/compile.py
@@ -38,7 +38,8 @@
         else:
             extraprocedures = [procedure]
         metainterp_sd.stats.view(errmsg=errmsg,
-                                 extraprocedures=extraprocedures)
+                                 extraprocedures=extraprocedures,
+                                 metainterp_sd=metainterp_sd)
 
 def create_empty_loop(metainterp, name_prefix=''):
     name = metainterp.staticdata.stats.name_for_new_loop()
@@ -111,33 +112,26 @@
     """
     from pypy.jit.metainterp.optimizeopt import optimize_trace
 
-    history = metainterp.history
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
+    history = metainterp.history
 
-    if False:
-        part = partial_trace
-        assert False
-        procedur_token = metainterp.get_procedure_token(greenkey)
-        assert procedure_token
-        all_target_tokens = []
-    else:
-        jitcell_token = make_jitcell_token(jitdriver_sd)
-        part = create_empty_loop(metainterp)
-        part.inputargs = inputargs[:]
-        h_ops = history.operations
-        part.resume_at_jump_descr = resume_at_jump_descr
-        part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \
-                          [h_ops[i].clone() for i in range(start, len(h_ops))] + \
-                          [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
+    jitcell_token = make_jitcell_token(jitdriver_sd)
+    part = create_empty_loop(metainterp)
+    part.inputargs = inputargs[:]
+    h_ops = history.operations
+    part.resume_at_jump_descr = resume_at_jump_descr
+    part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \
+                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
+                      [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
-        try:
-            optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
-        except InvalidLoop:
-            return None
-        target_token = part.operations[0].getdescr()
-        assert isinstance(target_token, TargetToken)
-        all_target_tokens = [target_token]
+    try:
+        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
+    except InvalidLoop:
+        return None
+    target_token = part.operations[0].getdescr()
+    assert isinstance(target_token, TargetToken)
+    all_target_tokens = [target_token]
 
     loop = create_empty_loop(metainterp)        
     loop.inputargs = part.inputargs
@@ -175,10 +169,10 @@
     loop.original_jitcell_token = jitcell_token
     for label in all_target_tokens:
         assert isinstance(label, TargetToken)
-        label.original_jitcell_token = jitcell_token
         if label.virtual_state and label.short_preamble:
             metainterp_sd.logger_ops.log_short_preamble([], label.short_preamble)
     jitcell_token.target_tokens = all_target_tokens
+    propagate_original_jitcell_token(loop)
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
     return all_target_tokens[0]
@@ -246,11 +240,11 @@
     for box in loop.inputargs:
         assert isinstance(box, Box)
 
-    target_token = loop.operations[-1].getdescr()
+    target_token = loop.operations[-1].getdescr()    
     resumekey.compile_and_attach(metainterp, loop)
+    
     target_token = label.getdescr()
     assert isinstance(target_token, TargetToken)
-    target_token.original_jitcell_token = loop.original_jitcell_token
     record_loop_or_bridge(metainterp_sd, loop)
     return target_token
 
@@ -287,6 +281,15 @@
     assert i == len(inputargs)
     loop.operations = extra_ops + loop.operations
 
+def propagate_original_jitcell_token(trace):
+    for op in trace.operations:
+        if op.getopnum() == rop.LABEL:
+            token = op.getdescr()
+            assert isinstance(token, TargetToken)
+            assert token.original_jitcell_token is None
+            token.original_jitcell_token = trace.original_jitcell_token
+            
+    
 def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
     vinfo = jitdriver_sd.virtualizable_info
     if vinfo is not None:
@@ -318,7 +321,10 @@
         metainterp_sd.stats.compiled()
     metainterp_sd.log("compiled new " + type)
     #
-    metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type, ops_offset)
+    loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
+    metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n,
+                                      type, ops_offset,
+                                      name=loopname)
     #
     if metainterp_sd.warmrunnerdesc is not None:    # for tests
         metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(original_jitcell_token)
@@ -557,6 +563,7 @@
         inputargs = metainterp.history.inputargs
         if not we_are_translated():
             self._debug_suboperations = new_loop.operations
+        propagate_original_jitcell_token(new_loop)
         send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata,
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
@@ -743,6 +750,7 @@
         jitdriver_sd = metainterp.jitdriver_sd
         redargs = new_loop.inputargs
         new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd)
+        propagate_original_jitcell_token(new_loop)
         send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd,
                              metainterp_sd, new_loop, "entry bridge")
         # send the new_loop to warmspot.py, to be called directly the next time
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,7 +12,7 @@
     def get_display_text(self):
         return None
 
-def display_procedures(procedures, errmsg=None, highlight_procedures={}):
+def display_procedures(procedures, errmsg=None, highlight_procedures={}, metainterp_sd=None):
     graphs = [(procedure, highlight_procedures.get(procedure, 0))
               for procedure in procedures]
     for graph, highlight in graphs:
@@ -20,7 +20,7 @@
             if is_interesting_guard(op):
                 graphs.append((SubGraph(op.getdescr()._debug_suboperations),
                                highlight))
-    graphpage = ResOpGraphPage(graphs, errmsg)
+    graphpage = ResOpGraphPage(graphs, errmsg, metainterp_sd)
     graphpage.display()
 
 def is_interesting_guard(op):
@@ -36,8 +36,8 @@
 
 class ResOpGraphPage(GraphPage):
 
-    def compute(self, graphs, errmsg=None):
-        resopgen = ResOpGen()
+    def compute(self, graphs, errmsg=None, metainterp_sd=None):
+        resopgen = ResOpGen(metainterp_sd)
         for graph, highlight in graphs:
             resopgen.add_graph(graph, highlight)
         if errmsg:
@@ -50,13 +50,14 @@
     CLUSTERING = True
     BOX_COLOR = (128, 0, 96)
 
-    def __init__(self):
+    def __init__(self, metainterp_sd=None):
         self.graphs = []
         self.highlight_graphs = {}
         self.block_starters = {}    # {graphindex: {set-of-operation-indices}}
         self.all_operations = {}
         self.errmsg = None
         self.target_tokens = {}
+        self.metainterp_sd = metainterp_sd
 
     def op_name(self, graphindex, opindex):
         return 'g%dop%d' % (graphindex, opindex)
@@ -164,7 +165,14 @@
         opindex = opstartindex
         while True:
             op = operations[opindex]
-            lines.append(op.repr(graytext=True))
+            op_repr = op.repr(graytext=True)
+            if op.getopnum() == rop.DEBUG_MERGE_POINT:
+                jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
+                if jd_sd._get_printable_location_ptr:
+                    s = jd_sd.warmstate.get_location_str(op.getarglist()[2:])
+                    s = s.replace(',', '.') # we use comma for argument splitting
+                    op_repr = "debug_merge_point(%d, '%s')" % (op.getarg(1).getint(), s)
+            lines.append(op_repr)
             if is_interesting_guard(op):
                 tgt = op.getdescr()._debug_suboperations[0]
                 tgt_g, tgt_i = self.all_operations[tgt]
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -79,9 +79,9 @@
             opnum == rop.COPYSTRCONTENT or
             opnum == rop.COPYUNICODECONTENT):
             return
-        if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
-            return
-        if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
+        if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST or
+            rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST or
+            rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST):
             return
         if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
             effectinfo = descr.get_extra_info()
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -1008,25 +1008,6 @@
         # a jump back to itself and possibly a few bridges ending with finnish.
         # Only the operations within the loop formed by that single jump will
         # be counted.
-
-        # XXX hacked version, ignore and remove me when jit-targets is merged.
-        loops = self.get_all_loops()
-        loops = [loop for loop in loops if 'Preamble' not in repr(loop)] #XXX
-        assert len(loops) == 1
-        loop, = loops
-        jumpop = loop.operations[-1]
-        assert jumpop.getopnum() == rop.JUMP
-        insns = {}
-        for op in loop.operations:
-            opname = op.getopname()
-            insns[opname] = insns.get(opname, 0) + 1
-        return self._check_insns(insns, expected, check)
-
-    def check_simple_loop(self, expected=None, **check):
-        # Usefull in the simplest case when we have only one trace ending with
-        # a jump back to itself and possibly a few bridges ending with finnish.
-        # Only the operations within the loop formed by that single jump will
-        # be counted.
         loops = self.get_all_loops()
         assert len(loops) == 1
         loop = loops[0]
@@ -1085,7 +1066,7 @@
         if option.view:
             self.view()
 
-    def view(self, errmsg=None, extraprocedures=[]):
+    def view(self, errmsg=None, extraprocedures=[], metainterp_sd=None):
         from pypy.jit.metainterp.graphpage import display_procedures
         procedures = self.get_all_loops()[:]
         for procedure in extraprocedures:
@@ -1097,7 +1078,7 @@
             if hasattr(procedure, '_looptoken_number') and (
                procedure._looptoken_number in self.invalidated_token_numbers):
                 highlight_procedures.setdefault(procedure, 2)
-        display_procedures(procedures, errmsg, highlight_procedures)
+        display_procedures(procedures, errmsg, highlight_procedures, metainterp_sd)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py
--- a/pypy/jit/metainterp/logger.py
+++ b/pypy/jit/metainterp/logger.py
@@ -13,14 +13,14 @@
         self.metainterp_sd = metainterp_sd
         self.guard_number = guard_number
 
-    def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None):
+    def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''):
         if type is None:
             debug_start("jit-log-noopt-loop")
             logops = self._log_operations(inputargs, operations, ops_offset)
             debug_stop("jit-log-noopt-loop")
         else:
             debug_start("jit-log-opt-loop")
-            debug_print("# Loop", number, ":", type,
+            debug_print("# Loop", number, '(%s)' % name , ":", type,
                         "with", len(operations), "ops")
             logops = self._log_operations(inputargs, operations, ops_offset)
             debug_stop("jit-log-opt-loop")
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -1,10 +1,13 @@
 from __future__ import with_statement
 from pypy.jit.metainterp.optimizeopt.test.test_util import (
-    LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot)
+    LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot,
+    FakeMetaInterpStaticData)
 from pypy.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
 from pypy.jit.metainterp.optimize import InvalidLoop
 from py.test import raises
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method
 
 class BaseTestMultiLabel(BaseTest):
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
@@ -84,6 +87,8 @@
         
         return optimized
 
+class OptimizeoptTestMultiLabel(BaseTestMultiLabel):
+
     def test_simple(self):
         ops = """
         [i1]
@@ -381,6 +386,82 @@
         """
         self.optimize_loop(ops, expected)
 
-class TestLLtype(BaseTestMultiLabel, LLtypeMixin):
+    def test_virtual_as_field_of_forced_box(self):
+        ops = """
+        [p0]
+        pv1 = new_with_vtable(ConstClass(node_vtable))
+        label(pv1, p0)
+        pv2 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(pv2, pv1, descr=valuedescr)
+        jump(pv1, pv2)
+        """
+        with raises(InvalidLoop):
+            self.optimize_loop(ops, ops)
+
+class OptRenameStrlen(Optimization):
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_STRLEN(self, op):
+        newop = op.clone()
+        newop.result = op.result.clonebox()
+        self.emit_operation(newop)
+        self.make_equal_to(op.result, self.getvalue(newop.result))
+    
+dispatch_opt = make_dispatcher_method(OptRenameStrlen, 'optimize_',
+                                      default=OptRenameStrlen.emit_operation)
+
+class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
+
+    def _do_optimize_loop(self, loop, call_pure_results):
+        from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll
+        from pypy.jit.metainterp.optimizeopt.util import args_dict
+        from pypy.jit.metainterp.optimizeopt.pure import OptPure
+
+        self.loop = loop
+        loop.call_pure_results = args_dict()
+        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+        optimize_unroll(metainterp_sd, loop, [OptRenameStrlen(), OptPure()], True)
+
+    def test_optimizer_renaming_boxes(self):
+        ops = """
+        [p1]
+        i1 = strlen(p1)
+        label(p1)
+        i2 = strlen(p1)
+        i3 = int_add(i2, 7)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        i1 = strlen(p1)
+        label(p1, i1)
+        i11 = same_as(i1)
+        i2 = int_add(i11, 7)
+        jump(p1, i11)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_optimizer_renaming_boxes_not_imported(self):
+        ops = """
+        [p1]
+        i1 = strlen(p1)
+        label(p1)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        i1 = strlen(p1)
+        label(p1, i1)
+        i11 = same_as(i1)
+        jump(p1, i11)
+        """
+        self.optimize_loop(ops, expected)
+
+        
+
+class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
     pass
 
+class TestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin):
+    pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7759,7 +7759,7 @@
         jump(i0, p0, i2)
         """
         self.optimize_loop(ops, expected)
-
+        
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -265,7 +265,16 @@
                     self.optimizer.importable_values[value] = imp
                 newvalue = self.optimizer.getvalue(op.result)
                 newresult = newvalue.get_key_box()
-                assert newresult is op.result or newvalue.is_constant()
+                # note that emitting here SAME_AS should not happen, but
+                # in case it does, we would prefer to be suboptimal in asm
+                # to a fatal RPython exception.
+                if newresult is not op.result and not newvalue.is_constant():
+                    op = ResOperation(rop.SAME_AS, [op.result], newresult)
+                    self.optimizer._newoperations.append(op)
+                    if self.optimizer.loop.logops:
+                        debug_print('  Falling back to add extra: ' +
+                                    self.optimizer.loop.logops.repr_of_resop(op))
+                    
         self.optimizer.flush()
         self.optimizer.emitting_dissabled = False
 
@@ -430,7 +439,13 @@
             return
         for a in op.getarglist():
             if not isinstance(a, Const) and a not in seen:
-                self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer, seen)
+                self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer,
+                                             seen)
+
+        if self.optimizer.loop.logops:
+            debug_print('  Emitting short op: ' +
+                        self.optimizer.loop.logops.repr_of_resop(op))
+
         optimizer.send_extra_operation(op)
         seen[op.result] = True
         if op.is_ovf():
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -409,7 +409,13 @@
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals
-        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
+        if optimizer:
+            box = value.force_box(optimizer)
+        else:
+            if value.is_virtual():
+                raise BadVirtualState
+            box = value.get_key_box()
+        boxes[self.position_in_notvirtuals] = box
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -471,8 +477,14 @@
             optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
+
+        # We try twice. The first time around we allow boxes to be forced
+        # which might change the virtual state if the box appear in more
+        # than one place among the inputargs.
         for i in range(len(values)):
             self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
+        for i in range(len(values)):
+            self.state[i].enum_forced_boxes(inputargs, values[i], None)
 
         if keyboxes:
             for i in range(len(values)):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -976,10 +976,13 @@
         self.verify_green_args(jitdriver_sd, greenboxes)
         self.debug_merge_point(jitdriver_sd, jdindex, self.metainterp.in_recursion,
                                greenboxes)
-
+        
         if self.metainterp.seen_loop_header_for_jdindex < 0:
-            if not jitdriver_sd.no_loop_header or not any_operation:
+            if not any_operation:
                 return
+            if self.metainterp.in_recursion or not self.metainterp.get_procedure_token(greenboxes, True):
+                if not jitdriver_sd.no_loop_header:
+                    return
             # automatically add a loop_header if there is none
             self.metainterp.seen_loop_header_for_jdindex = jdindex
         #
@@ -2053,9 +2056,15 @@
             from pypy.jit.metainterp.resoperation import opname
             raise NotImplementedError(opname[opnum])
 
-    def get_procedure_token(self, greenkey):
+    def get_procedure_token(self, greenkey, with_compiled_targets=False):
         cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey)
-        return cell.get_procedure_token()
+        token = cell.get_procedure_token()
+        if with_compiled_targets:
+            if not token:
+                return None
+            if not token.target_tokens:
+                return None
+        return token
         
     def compile_loop(self, original_boxes, live_arg_boxes, start, resume_at_jump_descr):
         num_green_args = self.jitdriver_sd.num_green_args
@@ -2088,11 +2097,9 @@
     def compile_trace(self, live_arg_boxes, resume_at_jump_descr):
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = live_arg_boxes[:num_green_args]
-        target_jitcell_token = self.get_procedure_token(greenkey)
+        target_jitcell_token = self.get_procedure_token(greenkey, True)
         if not target_jitcell_token:
             return
-        if not target_jitcell_token.target_tokens:
-            return
 
         self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None,
                             descr=target_jitcell_token)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2697,7 +2697,7 @@
         # bridge back to the preamble of the first loop is produced. A guard in
         # this bridge is later traced resulting in a failed attempt of retracing
         # the second loop.
-        self.check_trace_count(8)
+        self.check_trace_count(9)
 
         # FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
 
diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py
--- a/pypy/jit/metainterp/test/test_compile.py
+++ b/pypy/jit/metainterp/test/test_compile.py
@@ -18,7 +18,7 @@
         self.seen.append((inputargs, operations, token))
 
 class FakeLogger(object):
-    def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None):
+    def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''):
         pass
 
     def repr_of_resop(self, op):
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -255,6 +255,11 @@
         assert h.getarrayitem(box1, descr1, index1) is box2
         assert h.getarrayitem(box1, descr1, index2) is box4
 
+        h.invalidate_caches(rop.GUARD_TRUE, None, [])
+        assert h.getfield(box1, descr1) is box2
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr1, index2) is box4
+
         h.invalidate_caches(
             rop.CALL_LOOPINVARIANT, FakeCallDescr(FakeEffektinfo.EF_LOOPINVARIANT), [])
 
diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py
--- a/pypy/jit/metainterp/test/test_logger.py
+++ b/pypy/jit/metainterp/test/test_logger.py
@@ -180,7 +180,7 @@
     def test_intro_loop(self):
         bare_logger = logger.Logger(self.make_metainterp_sd())
         output = capturing(bare_logger.log_loop, [], [], 1, "foo")
-        assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops"
+        assert output.splitlines()[0] == "# Loop 1 () : foo with 0 ops"
         pure_parse(output)
 
     def test_intro_bridge(self):
diff --git a/pypy/jit/metainterp/test/test_loop.py b/pypy/jit/metainterp/test/test_loop.py
--- a/pypy/jit/metainterp/test/test_loop.py
+++ b/pypy/jit/metainterp/test/test_loop.py
@@ -756,7 +756,7 @@
         res = self.meta_interp(interpret, [1])
         assert res == interpret(1)
         # XXX it's unsure how many loops should be there
-        self.check_trace_count(3)
+        self.check_trace_count(2)
 
     def test_path_with_operations_not_from_start(self):
         jitdriver = JitDriver(greens = ['k'], reds = ['n', 'z'])
diff --git a/pypy/jit/metainterp/test/test_virtual.py b/pypy/jit/metainterp/test/test_virtual.py
--- a/pypy/jit/metainterp/test/test_virtual.py
+++ b/pypy/jit/metainterp/test/test_virtual.py
@@ -860,12 +860,15 @@
         assert res == f(10)
         self.check_resops(jump=2)
 
-    def test_nested_loops(self):
+    def test_nested_loops_1(self):
         class Int(object):
             def __init__(self, val):
                 self.val = val
-        myjitdriver = JitDriver(greens = ['pc'], reds = ['n', 'sa', 'i', 'j'])
         bytecode = "iajb+JI"
+        def get_printable_location(i):
+            return "%d: %s" % (i, bytecode[i])
+        myjitdriver = JitDriver(greens = ['pc'], reds = ['n', 'sa', 'i', 'j'],
+                                get_printable_location=get_printable_location)
         def f(n):
             pc = sa = 0
             i = j = Int(0)
@@ -877,7 +880,7 @@
                 elif op == 'j':
                     j = Int(0)
                 elif op == '+':
-                    sa += i.val * j.val
+                    sa += (i.val + 2) * (j.val + 2)
                 elif op == 'a':
                     i = Int(i.val + 1)
                 elif op == 'b':
@@ -899,6 +902,7 @@
         assert res == f(10)
         self.check_aborted_count(0)
         self.check_target_token_count(3)
+        self.check_resops(int_mul=2)
 
     def test_nested_loops_bridge(self):
         class Int(object):
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -6,7 +6,7 @@
     OperationError, wrap_oserror, operationerrfmt)
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rlib.rarithmetic import intmask
-from pypy.rlib import rpoll
+from pypy.rlib import rpoll, rsocket
 import sys
 
 READABLE = 1
@@ -252,7 +252,9 @@
         # "header" and the "body" of the message and send them at once.
         message = lltype.malloc(rffi.CCHARP.TO, size + 4, flavor='raw')
         try:
-            rffi.cast(rffi.UINTP, message)[0] = rffi.r_uint(size) # XXX htonl!
+            length = rffi.r_uint(rsocket.htonl(
+                    rffi.cast(lltype.Unsigned, size)))
+            rffi.cast(rffi.UINTP, message)[0] = length
             i = size - 1
             while i >= 0:
                 message[4 + i] = buffer[offset + i]
@@ -264,7 +266,8 @@
     def do_recv_string(self, space, buflength, maxlength):
         with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1) as length_ptr:
             self._recvall(space, rffi.cast(rffi.CCHARP, length_ptr), 4)
-            length = intmask(length_ptr[0])
+            length = intmask(rsocket.ntohl(
+                    rffi.cast(lltype.Unsigned, length_ptr[0])))
         if length > maxlength: # bad message, close connection
             self.flags &= ~READABLE
             if self.flags == 0:
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -37,6 +37,9 @@
     def test_connection(self):
         rhandle, whandle = self.make_pair()
 
+        whandle.send_bytes("abc")
+        assert rhandle.recv_bytes(100) == "abc"
+
         obj = [1, 2.0, "hello"]
         whandle.send(obj)
         obj2 = rhandle.recv()
@@ -150,4 +153,20 @@
         import _multiprocessing
 
         raises(IOError, _multiprocessing.Connection, -1)
-        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
+        raises(IOError, _multiprocessing.Connection, -15)
+
+    def test_byte_order(self):
+        # The exact format of net strings (length in network byte
+        # order) is important for interoperation with others
+        # implementations.
+        rhandle, whandle = self.make_pair()
+        whandle.send_bytes("abc")
+        whandle.send_bytes("defg")
+        import socket
+        sock = socket.fromfd(rhandle.fileno(),
+                             socket.AF_INET, socket.SOCK_STREAM)
+        data1 = sock.recv(7)
+        assert data1 == '\x00\x00\x00\x03abc'
+        data2 = sock.recv(8)
+        assert data2 == '\x00\x00\x00\x04defg'
+
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -58,6 +58,7 @@
 class W_PyCFunctionObject(Wrappable):
     def __init__(self, space, ml, w_self, w_module=None):
         self.ml = ml
+        self.name = rffi.charp2str(self.ml.c_ml_name)
         self.w_self = w_self
         self.w_module = w_module
 
@@ -69,7 +70,7 @@
         flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST)
         if space.is_true(w_kw) and not flags & METH_KEYWORDS:
             raise OperationError(space.w_TypeError, space.wrap(
-                rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword arguments"))
+                self.name + "() takes no keyword arguments"))
 
         func = rffi.cast(PyCFunction, self.ml.c_ml_meth)
         length = space.int_w(space.len(w_args))
@@ -80,13 +81,12 @@
             if length == 0:
                 return generic_cpy_call(space, func, w_self, None)
             raise OperationError(space.w_TypeError, space.wrap(
-                rffi.charp2str(self.ml.c_ml_name) + "() takes no arguments"))
+                self.name + "() takes no arguments"))
         elif flags & METH_O:
             if length != 1:
                 raise OperationError(space.w_TypeError,
                         space.wrap("%s() takes exactly one argument (%d given)" %  (
-                        rffi.charp2str(self.ml.c_ml_name), 
-                        length)))
+                        self.name, length)))
             w_arg = space.getitem(w_args, space.wrap(0))
             return generic_cpy_call(space, func, w_self, w_arg)
         elif flags & METH_VARARGS:
@@ -199,6 +199,7 @@
     __call__ = interp2app(cfunction_descr_call),
     __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc),
     __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject),
+    __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject),
     )
 W_PyCFunctionObject.typedef.acceptable_as_base_class = False
 
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -63,6 +63,7 @@
              ),
             ])
         assert mod.getarg_O(1) == 1
+        assert mod.getarg_O.__name__ == "getarg_O"
         raises(TypeError, mod.getarg_O)
         raises(TypeError, mod.getarg_O, 1, 1)
 
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -4,6 +4,7 @@
 class PyPyModule(MixedModule):
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
+        'remove_invalidates': 'interp_extras.remove_invalidates',
     }
     appleveldefs = {}
 
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -5,3 +5,11 @@
 @unwrap_spec(array=BaseArray)
 def debug_repr(space, array):
     return space.wrap(array.find_sig().debug_repr())
+
+ at unwrap_spec(array=BaseArray)
+def remove_invalidates(space, array):
+    """ Array modification will no longer invalidate any of it's
+    potential children. Use only for performance debugging
+    """
+    del array.invalidates[:]
+    return space.w_None
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -102,3 +102,27 @@
 class ConstantIterator(BaseIterator):
     def next(self, shapelen):
         return self
+
+# ------ other iterators that are not part of the computation frame ----------
+
+class AxisIterator(object):
+    """ This object will return offsets of each start of the last stride
+    """
+    def __init__(self, arr):
+        self.arr = arr
+        self.indices = [0] * (len(arr.shape) - 1)
+        self.done = False
+        self.offset = arr.start
+
+    def next(self):
+        for i in range(len(self.arr.shape) - 2, -1, -1):
+            if self.indices[i] < self.arr.shape[i] - 1:
+                self.indices[i] += 1
+                self.offset += self.arr.strides[i]
+                break
+            else:
+                self.indices[i] = 0
+                self.offset -= self.arr.backstrides[i]
+        else:
+            self.done = True
+        
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
+from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
 from pypy.module.micronumpy.strides import calculate_slice_strides
@@ -9,27 +9,31 @@
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator,\
-     view_iter_from_arr, OneDimIterator
+     view_iter_from_arr, OneDimIterator, AxisIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
-    reds=['result_size', 'frame', 'ri', 'self', 'result']
+    reds=['result_size', 'frame', 'ri', 'self', 'result'],
+    get_printable_location=signature.new_printable_location('numpy'),
 )
 all_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
-    reds=['frame', 'self', 'dtype']
+    reds=['frame', 'self', 'dtype'],
+    get_printable_location=signature.new_printable_location('all'),
 )
 any_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
-    reds=['frame', 'self', 'dtype']
+    reds=['frame', 'self', 'dtype'],
+    get_printable_location=signature.new_printable_location('any'),
 )
 slice_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
-    reds=['self', 'frame', 'source', 'res_iter']
+    reds=['self', 'frame', 'source', 'res_iter'],
+    get_printable_location=signature.new_printable_location('slice'),
 )
 
 def _find_shape_and_elems(space, w_iterable):
@@ -291,7 +295,8 @@
     def _reduce_argmax_argmin_impl(op_name):
         reduce_driver = jit.JitDriver(
             greens=['shapelen', 'sig'],
-            reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype']
+            reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype'],
+            get_printable_location=signature.new_printable_location(op_name),
         )
         def loop(self):
             sig = self.find_sig()
@@ -375,6 +380,9 @@
     def descr_get_dtype(self, space):
         return space.wrap(self.find_dtype())
 
+    def descr_get_ndim(self, space):
+        return space.wrap(len(self.shape))
+
     @jit.unroll_safe
     def descr_get_shape(self, space):
         return space.newtuple([space.wrap(i) for i in self.shape])
@@ -390,10 +398,10 @@
         return space.wrap(self.size)
 
     def descr_copy(self, space):
-        return self.copy()
+        return self.copy(space)
 
-    def copy(self):
-        return self.get_concrete().copy()
+    def copy(self, space):
+        return self.get_concrete().copy(space)
 
     def descr_len(self, space):
         if len(self.shape):
@@ -404,7 +412,7 @@
     def descr_repr(self, space):
         res = StringBuilder()
         res.append("array(")
-        concrete = self.get_concrete()
+        concrete = self.get_concrete_or_scalar()
         dtype = concrete.find_dtype()
         if not concrete.size:
             res.append('[]')
@@ -417,8 +425,9 @@
         else:
             concrete.to_str(space, 1, res, indent='       ')
         if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and
-            dtype is not interp_dtype.get_dtype_cache(space).w_int64dtype) or \
-            not self.size:
+            not (dtype.kind == interp_dtype.SIGNEDLTR and
+            dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or
+            not self.size):
             res.append(", dtype=" + dtype.name)
         res.append(")")
         return space.wrap(res.build())
@@ -536,7 +545,7 @@
                               new_shape, self)
         else:
             # Create copy with contiguous data
-            arr = concrete.copy()
+            arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
 
@@ -578,8 +587,8 @@
             strides.append(concrete.strides[i])
             backstrides.append(concrete.backstrides[i])
             shape.append(concrete.shape[i])
-        return space.wrap(W_NDimSlice(concrete.start, strides[:],
-                                      backstrides[:], shape[:], concrete))
+        return space.wrap(W_NDimSlice(concrete.start, strides,
+                                      backstrides, shape, concrete))
 
     def descr_get_flatiter(self, space):
         return space.wrap(W_FlatIterator(self))
@@ -606,6 +615,9 @@
                                                        space.w_False]))
         return w_d
 
+    def supports_fast_slicing(self):
+        return False
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -639,7 +651,7 @@
     def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False):
         builder.append(self.dtype.itemtype.str_format(self.value))
 
-    def copy(self):
+    def copy(self, space):
         return Scalar(self.dtype, self.value)
 
     def create_sig(self, res_shape):
@@ -790,6 +802,9 @@
     def get_concrete(self):
         return self
 
+    def supports_fast_slicing(self):
+        return self.order == 'C' and self.strides[-1] == 1
+
     def find_dtype(self):
         return self.dtype
 
@@ -814,8 +829,8 @@
         if self.order == 'C':
             strides.reverse()
             backstrides.reverse()
-        self.strides = strides[:]
-        self.backstrides = backstrides[:]
+        self.strides = strides
+        self.backstrides = backstrides
 
     def array_sig(self, res_shape):
         if res_shape is not None and self.shape != res_shape:
@@ -829,80 +844,80 @@
         each line will begin with indent.
         '''
         size = self.size
+        ccomma = ',' * comma
+        ncomma = ',' * (1 - comma)
+        dtype = self.find_dtype()
         if size < 1:
             builder.append('[]')
             return
+        elif size == 1:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
-        dtype = self.find_dtype()
         ndims = len(self.shape)
         i = 0
-        start = True
         builder.append('[')
         if ndims > 1:
             if use_ellipsis:
-                for i in range(3):
-                    if start:
-                        start = False
-                    else:
-                        builder.append(',' * comma + '\n')
-                        if ndims == 3:
+                for i in range(min(3, self.shape[0])):
+                    if i > 0:
+                        builder.append(ccomma + '\n')
+                        if ndims >= 3:
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    # create_slice requires len(chunks) > 1 in order to reduce
-                    # shape
-                    view = self.create_slice([(i, 0, 0, 1), (0, self.shape[1], 1, self.shape[1])]).get_concrete()
-                    view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis)
-                builder.append('\n' + indent + '..., ')
-                i = self.shape[0] - 3
+                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view.to_str(space, comma, builder, indent=indent + ' ',
+                                                    use_ellipsis=use_ellipsis)
+                if i < self.shape[0] - 1:
+                    builder.append(ccomma +'\n' + indent + '...' + ncomma)
+                    i = self.shape[0] - 3
+                else:
+                    i += 1
             while i < self.shape[0]:
-                if start:
-                    start = False
-                else:
-                    builder.append(',' * comma + '\n')
-                    if ndims == 3:
+                if i > 0:
+                    builder.append(ccomma + '\n')
+                    if ndims >= 3:
                         builder.append('\n' + indent)
                     else:
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1), (0, self.shape[1], 1, self.shape[1])]).get_concrete()
-                view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis)
+                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view.to_str(space, comma, builder, indent=indent + ' ',
+                                                    use_ellipsis=use_ellipsis)
                 i += 1
         elif ndims == 1:
-            spacer = ',' * comma + ' '
+            spacer = ccomma + ' '
             item = self.start
             # An iterator would be a nicer way to walk along the 1d array, but
             # how do I reset it if printing ellipsis? iterators have no
             # "set_offset()"
             i = 0
             if use_ellipsis:
-                for i in range(3):
-                    if start:
-                        start = False
-                    else:
+                for i in range(min(3, self.shape[0])):
+                    if i > 0:
                         builder.append(spacer)
                     builder.append(dtype.itemtype.str_format(self.getitem(item)))
                     item += self.strides[0]
-                # Add a comma only if comma is False - this prevents adding two
-                # commas
-                builder.append(spacer + '...' + ',' * (1 - comma))
-                # Ugly, but can this be done with an iterator?
-                item = self.start + self.backstrides[0] - 2 * self.strides[0]
-                i = self.shape[0] - 3
+                if i < self.shape[0] - 1:
+                    # Add a comma only if comma is False - this prevents adding
+                    # two commas
+                    builder.append(spacer + '...' + ncomma)
+                    # Ugly, but can this be done with an iterator?
+                    item = self.start + self.backstrides[0] - 2 * self.strides[0]
+                    i = self.shape[0] - 3
+                else:
+                    i += 1
             while i < self.shape[0]:
-                if start:
-                    start = False
-                else:
+                if i > 0:
                     builder.append(spacer)
                 builder.append(dtype.itemtype.str_format(self.getitem(item)))
                 item += self.strides[0]
                 i += 1
-        else:
-            builder.append('[')
         builder.append(']')
 
     @jit.unroll_safe
@@ -929,39 +944,35 @@
             item += v * self.strides[i]
         return item
 
-
-class ViewArray(ConcreteArray):
-    def copy(self):
-        array = W_NDimArray(self.size, self.shape[:], self.find_dtype())
-        iter = view_iter_from_arr(self)
-        a_iter = ArrayIterator(array.size)
-        while not iter.done():
-            array.setitem(a_iter.offset, self.getitem(iter.offset))
-            iter = iter.next(len(self.shape))
-            a_iter = a_iter.next(len(array.shape))
-        return array
-
-    def create_sig(self, res_shape):
-        return signature.ViewSignature(self.dtype)
-
-
-class W_NDimSlice(ViewArray):
-    def __init__(self, start, strides, backstrides, shape, parent):
-        assert isinstance(parent, ConcreteArray)
-        if isinstance(parent, W_NDimSlice):
-            parent = parent.parent
-        size = 1
-        for sh in shape:
-            size *= sh
-        self.strides = strides
-        self.backstrides = backstrides
-        ViewArray.__init__(self, size, shape, parent.dtype, parent.order,
-                               parent)
-        self.start = start
-
     def setslice(self, space, w_value):
         res_shape = shape_agreement(space, self.shape, w_value.shape)
-        self._sliceloop(w_value, res_shape)
+        if (res_shape == w_value.shape and self.supports_fast_slicing() and
+            w_value.supports_fast_slicing() and
+            self.dtype is w_value.find_dtype()):
+            self._fast_setslice(space, w_value)
+        else:
+            self._sliceloop(w_value, res_shape)
+
+    def _fast_setslice(self, space, w_value):
+        assert isinstance(w_value, ConcreteArray)
+        itemsize = self.dtype.itemtype.get_element_size()
+        if len(self.shape) == 1:
+            rffi.c_memcpy(
+                rffi.ptradd(self.storage, self.start * itemsize),
+                rffi.ptradd(w_value.storage, w_value.start * itemsize),
+                self.size * itemsize
+            )
+        else:
+            dest = AxisIterator(self)
+            source = AxisIterator(w_value)
+            while not dest.done:
+                rffi.c_memcpy(
+                    rffi.ptradd(self.storage, dest.offset * itemsize),
+                    rffi.ptradd(w_value.storage, source.offset * itemsize),
+                    self.shape[-1] * itemsize
+                )
+                source.next()
+                dest.next()
 
     def _sliceloop(self, source, res_shape):
         sig = source.find_sig(res_shape)
@@ -979,6 +990,31 @@
             frame.next(shapelen)
             res_iter = res_iter.next(shapelen)
 
+    def copy(self, space):
+        array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
+        array.setslice(space, self)
+        return array
+
+
+class ViewArray(ConcreteArray):
+    def create_sig(self, res_shape):
+        return signature.ViewSignature(self.dtype)
+
+
+class W_NDimSlice(ViewArray):
+    def __init__(self, start, strides, backstrides, shape, parent):
+        assert isinstance(parent, ConcreteArray)
+        if isinstance(parent, W_NDimSlice):
+            parent = parent.parent
+        size = 1
+        for sh in shape:
+            size *= sh
+        self.strides = strides
+        self.backstrides = backstrides
+        ViewArray.__init__(self, size, shape, parent.dtype, parent.order,
+                               parent)
+        self.start = start
+
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
             return
@@ -998,9 +1034,9 @@
                 strides.reverse()
                 backstrides.reverse()
                 new_shape.reverse()
-            self.strides = strides[:]
-            self.backstrides = backstrides[:]
-            self.shape = new_shape[:]
+            self.strides = strides
+            self.backstrides = backstrides
+            self.shape = new_shape
             return
         new_strides = calc_new_strides(new_shape, self.shape, self.strides)
         if new_strides is None:
@@ -1010,22 +1046,13 @@
         for nd in range(len(new_shape)):
             new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
         self.strides = new_strides[:]
-        self.backstrides = new_backstrides[:]
+        self.backstrides = new_backstrides
         self.shape = new_shape[:]
 
 class W_NDimArray(ConcreteArray):
     """ A class representing contiguous array. We know that each iteration
     by say ufunc will increase the data index by one
     """
-    def copy(self):
-        array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
-        rffi.c_memcpy(
-            array.storage,
-            self.storage,
-            self.size * self.dtype.itemtype.get_element_size()
-        )
-        return array
-
     def setitem(self, item, value):
         self.invalidated()
         self.dtype.setitem(self.storage, item, value)
@@ -1162,6 +1189,7 @@
     shape = GetSetProperty(BaseArray.descr_get_shape,
                            BaseArray.descr_set_shape),
     size = GetSetProperty(BaseArray.descr_get_size),
+    ndim = GetSetProperty(BaseArray.descr_get_ndim),
 
     T = GetSetProperty(BaseArray.descr_get_transpose),
     flat = GetSetProperty(BaseArray.descr_get_flatiter),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -1,9 +1,10 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
-from pypy.module.micronumpy import interp_boxes, interp_dtype, types
-from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature, find_sig
+from pypy.module.micronumpy import interp_boxes, interp_dtype
+from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature,\
+     find_sig, new_printable_location
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
@@ -11,7 +12,8 @@
 reduce_driver = jit.JitDriver(
     greens = ['shapelen', "sig"],
     virtualizables = ["frame"],
-    reds = ["frame", "self", "dtype", "value", "obj"]
+    reds = ["frame", "self", "dtype", "value", "obj"],
+    get_printable_location=new_printable_location('reduce'),
 )
 
 class W_Ufunc(Wrappable):
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -5,6 +5,11 @@
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib.jit import hint, unroll_safe, promote
 
+def new_printable_location(driver_name):
+    def get_printable_location(shapelen, sig):
+        return 'numpy ' + sig.debug_repr() + ' [%d dims,%s]' % (shapelen, driver_name)
+    return get_printable_location
+
 def sigeq(one, two):
     return one.eq(two)
 
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,4 +1,9 @@
+from pypy.rlib import jit
 
+
+ at jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
+    jit.isconstant(len(chunks))
+)
 def calculate_slice_strides(shape, start, strides, backstrides, chunks):
     rstrides = []
     rbackstrides = []
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -158,6 +158,7 @@
         assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None
         assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2]
 
+
 class AppTestNumArray(BaseNumpyAppTest):
     def test_ndarray(self):
         from numpypy import ndarray, array, dtype
@@ -179,6 +180,20 @@
         ar = array(range(5))
         assert type(ar) is type(ar + ar)
 
+    def test_ndim(self):
+        from numpypy import array
+        x = array(0.2)
+        assert x.ndim == 0
+        x = array([1, 2])
+        assert x.ndim == 1
+        x = array([[1, 2], [3, 4]])
+        assert x.ndim == 2
+        x = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        assert x.ndim == 3
+        # numpy actually raises an AttributeError, but numpypy raises an
+        # TypeError
+        raises(TypeError, 'x.ndim = 3')
+
     def test_init(self):
         from numpypy import zeros
         a = zeros(15)
@@ -725,19 +740,19 @@
         a = identity(0)
         assert len(a) == 0
         assert a.dtype == dtype('float64')
-        assert a.shape == (0,0)
+        assert a.shape == (0, 0)
         b = identity(1, dtype=int32)
         assert len(b) == 1
         assert b[0][0] == 1
-        assert b.shape == (1,1)
+        assert b.shape == (1, 1)
         assert b.dtype == dtype('int32')
         c = identity(2)
-        assert c.shape == (2,2)
-        assert (c == [[1,0],[0,1]]).all()
+        assert c.shape == (2, 2)
+        assert (c == [[1, 0], [0, 1]]).all()
         d = identity(3, dtype='int32')
-        assert d.shape == (3,3)
+        assert d.shape == (3, 3)
         assert d.dtype == dtype('int32')
-        assert (d == [[1,0,0],[0,1,0],[0,0,1]]).all()
+        assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all()
 
     def test_prod(self):
         from numpypy import array
@@ -898,6 +913,15 @@
         b[0] = 3
         assert debug_repr(b) == 'Array'
 
+    def test_remove_invalidates(self):
+        from numpypy import array
+        from numpypy.pypy import remove_invalidates
+        a = array([1, 2, 3])
+        b = a + a
+        remove_invalidates(a)
+        a[0] = 14
+        assert b[0] == 28
+
     def test_virtual_views(self):
         from numpypy import arange
         a = arange(15)
@@ -945,13 +969,13 @@
 
     def test_tolist_view(self):
         from numpypy import array
-        a = array([[1,2],[3,4]])
+        a = array([[1, 2], [3, 4]])
         assert (a + a).tolist() == [[2, 4], [6, 8]]
 
     def test_tolist_slice(self):
         from numpypy import array
         a = array([[17.1, 27.2], [40.3, 50.3]])
-        assert a[:,0].tolist() == [17.1, 40.3]
+        assert a[:, 0].tolist() == [17.1, 40.3]
         assert a[0].tolist() == [17.1, 27.2]
 
 
@@ -1077,6 +1101,17 @@
         a = ones((1, 2, 3))
         assert a[0, 1, 2] == 1.0
 
+    def test_multidim_setslice(self):
+        from numpypy import zeros, ones
+        a = zeros((3, 3))
+        b = ones((3, 3))
+        a[:, 1:3] = b[:, 1:3]
+        assert (a == [[0, 1, 1], [0, 1, 1], [0, 1, 1]]).all()
+        a = zeros((3, 3))
+        b = ones((3, 3))
+        a[:, ::2] = b[:, ::2]
+        assert (a == [[1, 0, 1], [1, 0, 1], [1, 0, 1]]).all()
+
     def test_broadcast_ufunc(self):
         from numpypy import array
         a = array([[1, 2], [3, 4], [5, 6]])
@@ -1213,6 +1248,7 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
         import struct
@@ -1255,17 +1291,17 @@
         assert g[1] == 2
         assert g[2] == 3
         h = fromstring("1, , 2, 3", dtype=uint8, sep=",")
-        assert (h == [1,0,2,3]).all()
+        assert (h == [1, 0, 2, 3]).all()
         i = fromstring("1    2 3", dtype=uint8, sep=" ")
-        assert (i == [1,2,3]).all()
+        assert (i == [1, 2, 3]).all()
         j = fromstring("1\t\t\t\t2\t3", dtype=uint8, sep="\t")
-        assert (j == [1,2,3]).all()
+        assert (j == [1, 2, 3]).all()
         k = fromstring("1,x,2,3", dtype=uint8, sep=",")
-        assert (k == [1,0]).all()
+        assert (k == [1, 0]).all()
         l = fromstring("1,x,2,3", dtype='float32', sep=",")
-        assert (l == [1.0,-1.0]).all()
+        assert (l == [1.0, -1.0]).all()
         m = fromstring("1,,2,3", sep=",")
-        assert (m == [1.0,-1.0,2.0,3.0]).all()
+        assert (m == [1.0, -1.0, 2.0, 3.0]).all()
         n = fromstring("3.4 2.0 3.8 2.2", dtype=int32, sep=" ")
         assert (n == [3]).all()
         o = fromstring("1.0 2f.0f 3.8 2.2", dtype=float32, sep=" ")
@@ -1313,7 +1349,6 @@
         j = fromstring(self.ulongval, dtype='L')
         assert j[0] == 12
 
-
     def test_fromstring_invalid(self):
         from numpypy import fromstring, uint16, uint8, int32
         #default dtype is 64-bit float, so 3 bytes should fail
@@ -1327,6 +1362,7 @@
 class AppTestRepr(BaseNumpyAppTest):
     def test_repr(self):
         from numpypy import array, zeros
+        int_size = array(5).dtype.itemsize
         a = array(range(5), float)
         assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])"
         a = array([], float)
@@ -1334,14 +1370,26 @@
         a = zeros(1001)
         assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])"
         a = array(range(5), long)
-        assert repr(a) == "array([0, 1, 2, 3, 4])"
+        if a.dtype.itemsize == int_size:
+            assert repr(a) == "array([0, 1, 2, 3, 4])"
+        else:
+            assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)"
+        a = array(range(5), 'int32')
+        if a.dtype.itemsize == int_size:
+            assert repr(a) == "array([0, 1, 2, 3, 4])"
+        else:
+            assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)"
         a = array([], long)
         assert repr(a) == "array([], dtype=int64)"
         a = array([True, False, True, False], "?")
         assert repr(a) == "array([True, False, True, False], dtype=bool)"
+        a = zeros([])
+        assert repr(a) == "array(0.0)"
+        a = array(0.2)
+        assert repr(a) == "array(0.2)"
 
     def test_repr_multi(self):
-        from numpypy import array, zeros
+        from numpypy import arange, zeros
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1354,6 +1402,16 @@
        [[0.0, 0.0, 0.0, 0.0],
         [0.0, 0.0, 0.0, 0.0],
         [0.0, 0.0, 0.0, 0.0]]])'''
+        a = arange(1002).reshape((2, 501))
+        assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500],
+       [501, 502, 503, ..., 999, 1000, 1001]])'''
+        assert repr(a.T) == '''array([[0, 501],
+       [1, 502],
+       [2, 503],
+       ...,
+       [498, 999],
+       [499, 1000],
+       [500, 1001]])'''
 
     def test_repr_slice(self):
         from numpypy import array, zeros
@@ -1397,7 +1455,7 @@
 
         a = zeros((400, 400), dtype=int)
         assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \
-           " [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n" \
+           " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \
            " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]"
         a = zeros((2, 2, 2))
         r = str(a)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -8,10 +8,12 @@
 from pypy.tool import logparser
 from pypy.jit.tool.jitoutput import parse_prof
 from pypy.module.pypyjit.test_pypy_c.model import (Log, find_ids_range,
-                                                   find_ids, TraceWithIds,
+                                                   find_ids,
                                                    OpMatcher, InvalidMatch)
 
 class BaseTestPyPyC(object):
+    log_string = 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary'
+    
     def setup_class(cls):
         if '__pypy__' not in sys.builtin_module_names:
             py.test.skip("must run this test with pypy")
@@ -52,8 +54,7 @@
             cmdline += ['--jit', ','.join(jitcmdline)]
         cmdline.append(str(self.filepath))
         #
-        print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary:' + str(logfile)}
+        env={'PYPYLOG': self.log_string + ':' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test__ffi.py b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test__ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
@@ -98,7 +98,8 @@
             end = time.time()
             return end - start
         #
-        log = self.run(main, [get_libc_name(), 200], threshold=150)
+        log = self.run(main, [get_libc_name(), 200], threshold=150,
+                       import_site=True)
         assert 1 <= log.result <= 1.5 # at most 0.5 seconds of overhead
         loops = log.loops_by_id('sleep')
         assert len(loops) == 1 # make sure that we actually JITted the loop
@@ -121,7 +122,7 @@
             return fabs._ptr.getaddr(), x
 
         libm_name = get_libm_name(sys.platform)
-        log = self.run(main, [libm_name])
+        log = self.run(main, [libm_name], import_site=True)
         fabs_addr, res = log.result
         assert res == -4.0
         loop, = log.loops_by_filename(self.filepath)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -15,7 +15,7 @@
                 i += letters[i % len(letters)] == uletters[i % len(letters)]
             return i
 
-        log = self.run(main, [300])
+        log = self.run(main, [300], import_site=True)
         assert log.result == 300
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
@@ -55,7 +55,7 @@
                 i += int(long(string.digits[i % len(string.digits)], 16))
             return i
 
-        log = self.run(main, [1100])
+        log = self.run(main, [1100], import_site=True)
         assert log.result == main(1100)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -66,11 +66,11 @@
     return None
 
 copyright_str = """
-Copyright 2003-2011 PyPy development team.
+Copyright 2003-2012 PyPy development team.
 All Rights Reserved.
 For further information, see <http://pypy.org>
 
-Portions Copyright (c) 2001-2008 Python Software Foundation.
+Portions Copyright (c) 2001-2012 Python Software Foundation.
 All Rights Reserved.
 
 Portions Copyright (c) 2000 BeOpen.com.
diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -185,7 +185,7 @@
 class FlowExecutionContext(ExecutionContext):
 
     def __init__(self, space, code, globals, constargs={}, outer_func=None,
-                 name=None):
+                 name=None, is_generator=False):
         ExecutionContext.__init__(self, space)
         self.code = code
 
@@ -208,6 +208,7 @@
         initialblock = SpamBlock(FrameState(frame).copy())
         self.pendingblocks = collections.deque([initialblock])
         self.graph = FunctionGraph(name or code.co_name, initialblock)
+        self.is_generator = is_generator
 
     make_link = Link # overridable for transition tracking
 
@@ -247,6 +248,8 @@
         return outcome, w_exc_cls, w_exc_value
 
     def build_flow(self):
+        if self.is_generator:
+            self.produce_generator_mark()
         while self.pendingblocks:
             block = self.pendingblocks.popleft()
             frame = self.create_frame()
@@ -259,9 +262,15 @@
                 self.topframeref = jit.non_virtual_ref(frame)
                 self.crnt_frame = frame
                 try:
-                    w_result = frame.dispatch(frame.pycode,
-                                              frame.last_instr,
-                                              self)
+                    frame.frame_finished_execution = False
+                    while True:
+                        w_result = frame.dispatch(frame.pycode,
+                                                  frame.last_instr,
+                                                  self)
+                        if frame.frame_finished_execution:
+                            break
+                        else:
+                            self.generate_yield(frame, w_result)
                 finally:
                     self.crnt_frame = None
                     self.topframeref = old_frameref
@@ -307,6 +316,21 @@
             del self.recorder
         self.fixeggblocks()
 
+    def produce_generator_mark(self):
+        [initialblock] = self.pendingblocks
+        initialblock.operations.append(
+            SpaceOperation('generator_mark', [], Variable()))
+
+    def generate_yield(self, frame, w_result):
+        assert self.is_generator
+        self.recorder.crnt_block.operations.append(
+            SpaceOperation('yield', [w_result], Variable()))
+        # we must push a dummy value that will be POPped: it's the .send()
+        # passed into the generator (2.5 feature)
+        assert sys.version_info >= (2, 5)
+        frame.pushvalue(None)
+        frame.last_instr += 1
+
     def fixeggblocks(self):
         # EggBlocks reuse the variables of their previous block,
         # which is deemed not acceptable for simplicity of the operations
diff --git a/pypy/objspace/flow/objspace.py b/pypy/objspace/flow/objspace.py
--- a/pypy/objspace/flow/objspace.py
+++ b/pypy/objspace/flow/objspace.py
@@ -8,6 +8,7 @@
 from pypy.interpreter.pycode import PyCode, cpython_code_signature
 from pypy.interpreter.module import Module
 from pypy.interpreter.error import OperationError
+from pypy.interpreter.astcompiler.consts import CO_GENERATOR
 from pypy.interpreter import pyframe, argument
 from pypy.objspace.flow.model import *
 from pypy.objspace.flow import flowcontext, operation, specialcase
@@ -248,15 +249,13 @@
                 return ecls
         return None
 
-    def build_flow(self, func, constargs={}):
+    def build_flow(self, func, constargs={}, tweak_for_generator=True):
         """
         """
         if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
             raise Exception, "%r is tagged as NOT_RPYTHON" % (func,)
         code = func.func_code
-        if code.co_flags & 32:
-            # generator
-            raise TypeError("%r is a generator" % (func,))
+        is_generator = bool(code.co_flags & CO_GENERATOR)
         code = PyCode._from_code(self, code)
         if func.func_closure is None:
             cl = None
@@ -272,7 +271,8 @@
         class outerfunc: # hack
             closure = cl
         ec = flowcontext.FlowExecutionContext(self, code, func.func_globals,
-                                              constargs, outerfunc, name)
+                                              constargs, outerfunc, name,
+                                              is_generator)
         graph = ec.graph
         graph.func = func
         # attach a signature and defaults to the graph
@@ -292,6 +292,11 @@
             e = error.FlowingError(formated)
             raise error.FlowingError, e, tb
         checkgraph(graph)
+        #
+        if is_generator and tweak_for_generator:
+            from pypy.translator.generator import tweak_generator_graph
+            tweak_generator_graph(graph)
+        #
         return graph
 
     def fixedview(self, w_tuple, expected_length=None):
diff --git a/pypy/objspace/flow/test/test_generator.py b/pypy/objspace/flow/test/test_generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/flow/test/test_generator.py
@@ -0,0 +1,18 @@
+from pypy.objspace.flow.test.test_objspace import Base
+
+
+class TestGenerator(Base):
+
+    def test_simple_generator(self):
+        def f(n):
+            i = 0
+            while i < n:
+                yield i
+                yield i
+                i += 1
+        graph = self.codetest(f, tweak_for_generator=False)
+        ops = self.all_operations(graph)
+        assert ops == {'generator_mark': 1,
+                       'lt': 1, 'is_true': 1,
+                       'yield': 2,
+                       'inplace_add': 1}
diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py
--- a/pypy/objspace/flow/test/test_objspace.py
+++ b/pypy/objspace/flow/test/test_objspace.py
@@ -16,14 +16,14 @@
 is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2
 
 class Base:
-    def codetest(self, func):
+    def codetest(self, func, **kwds):
         import inspect
         try:
             func = func.im_func
         except AttributeError:
             pass
         #name = func.func_name
-        graph = self.space.build_flow(func)
+        graph = self.space.build_flow(func, **kwds)
         graph.source = inspect.getsource(func)
         self.show(graph)
         return graph
@@ -882,12 +882,6 @@
                 num = bytecode_spec.opmap[name]
                 flow_meth_names[num] = locals()['old_' + name]
 
-    def test_generator(self):
-        def f():
-            yield 3
-
-        py.test.raises(TypeError, "self.codetest(f)")
-
     def test_dont_capture_RuntimeError(self):
         class Foo:
             def __hash__(self):
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -519,7 +519,7 @@
         if maxsplit > 0 and maxsplit < upper + 2:
             upper = maxsplit - 1
             assert upper >= 0
-        
+
         try:
             result_size = ovfcheck(upper * len(by))
             result_size = ovfcheck(result_size + upper)
@@ -537,25 +537,32 @@
         builder.append(by)
         builder.append_slice(input, upper, len(input))
     else:
-        # An ok guess for the result size
-        builder = StringBuilder(len(input))
+        # First compute the exact result size
+        count = input.count(sub)
+        if count > maxsplit and maxsplit > 0:
+            count = maxsplit
+        diff_len = len(by) - len(sub)
+        try:
+            result_size = ovfcheck(diff_len * count)
+            result_size = ovfcheck(result_size + len(input))
+        except OverflowError:
+            raise OperationError(space.w_OverflowError,
+                space.wrap("replace string is too long")
+            )
+
+        builder = StringBuilder(result_size)
         start = 0
         sublen = len(sub)
-        first = True
 
         while maxsplit != 0:
             next = input.find(sub, start)
             if next < 0:
                 break
-            if not first:
-                builder.append(by)
-            first = False
             builder.append_slice(input, start, next)
+            builder.append(by)
             start = next + sublen
             maxsplit -= 1   # NB. if it's already < 0, it stays < 0
 
-        if not first:
-            builder.append(by)
         builder.append_slice(input, start, len(input))
 
     return space.wrap(builder.build())
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -528,6 +528,9 @@
                         set_param(driver, name1, int(value))
                     except ValueError:
                         raise
+                    break
+            else:
+                raise ValueError
 set_user_param._annspecialcase_ = 'specialize:arg(0)'
 
 
diff --git a/pypy/rlib/rsre/rsre_jit.py b/pypy/rlib/rsre/rsre_jit.py
--- a/pypy/rlib/rsre/rsre_jit.py
+++ b/pypy/rlib/rsre/rsre_jit.py
@@ -22,7 +22,7 @@
                     info = '%s/%d' % (info, args[debugprint[2]])
             else:
                 info = ''
-            return '%s%s %s' % (name, info, s)
+            return 're %s%s %s' % (name, info, s)
         #
         self.get_printable_location = get_printable_location
 
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -375,7 +375,6 @@
         newitems = malloc(LIST.items.TO, n)
         rgc.ll_arraycopy(olditems, newitems, 0, 0, n)
         return newitems
-ll_list2fixed.oopspec = 'list.list2fixed(l)'
 
 def ll_list2fixed_exact(l):
     ll_assert(l.length == len(l.items), "ll_list2fixed_exact: bad length")
diff --git a/pypy/rpython/test/test_generator.py b/pypy/rpython/test/test_generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/test/test_generator.py
@@ -0,0 +1,82 @@
+from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
+
+
+class BaseTestGenerator(BaseRtypingTest):
+
+    def test_simple_explicit(self):
+        def g(a, b, c):
+            yield a
+            yield b
+            yield c
+        def f():
+            gen = g(3, 5, 8)
+            x = gen.next() * 100
+            x += gen.next() * 10
+            x += gen.next()
+            return x
+        res = self.interpret(f, [])
+        assert res == 358
+
+    def test_cannot_merge(self):
+        # merging two different generators is not supported
+        # right now, but we can use workarounds like here
+        class MyGen:
+            _immutable_ = True
+            def next(self):
+                raise NotImplementedError
+        class MyG1(MyGen):
+            _immutable_ = True
+            def __init__(self, a):
+                self._gen = self.g1(a)
+            def next(self):
+                return self._gen.next()
+            @staticmethod
+            def g1(a):
+                yield a + 1
+                yield a + 2
+        class MyG2(MyGen):
+            _immutable_ = True
+            def __init__(self):
+                self._gen = self.g2()
+            def next(self):
+                return self._gen.next()
+            @staticmethod
+            def g2():
+                yield 42
+        def f(n):
+            if n > 0:
+                gen = MyG1(n)
+            else:
+                gen = MyG2()
+            return gen.next()
+        res = self.interpret(f, [10])
+        assert res == 11
+        res = self.interpret(f, [0])
+        assert res == 42
+
+    def test_except_block(self):
+        def foo():
+            raise ValueError
+        def g(a, b, c):
+            yield a
+            yield b
+            try:
+                foo()
+            except ValueError:
+                pass
+            yield c
+        def f():
+            gen = g(3, 5, 8)
+            x = gen.next() * 100
+            x += gen.next() * 10
+            x += gen.next()
+            return x
+        res = self.interpret(f, [])
+        assert res == 358
+
+
+class TestLLtype(BaseTestGenerator, LLRtypeMixin):
+    pass
+
+class TestOOtype(BaseTestGenerator, OORtypeMixin):
+    pass
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -3,6 +3,7 @@
 from pypy.jit.metainterp.resoperation import opname
 from pypy.jit.tool.oparser import OpParser
 from pypy.tool.logparser import parse_log_file, extract_category
+from copy import copy
 
 class Op(object):
     bridge = None
@@ -23,27 +24,24 @@
         self.failargs = failargs
 
     def getarg(self, i):
-        return self._getvar(self.args[i])
+        return self.args[i]
 
     def getargs(self):
-        return [self._getvar(v) for v in self.args]
+        return self.args[:]
 
     def getres(self):
-        return self._getvar(self.res)
+        return self.res
 
     def getdescr(self):
         return self.descr
 
-    def _getvar(self, v):
-        return v
-
     def is_guard(self):
         return self._is_guard
 
     def repr(self):
         args = self.getargs()
         if self.descr is not None:
-            args.append('descr=%s' % self.getdescr())
+            args.append('descr=%s' % self.descr)
         arglist = ', '.join(args)
         if self.res is not None:
             return '%s = %s(%s)' % (self.getres(), self.name, arglist)
@@ -52,8 +50,6 @@
 
     def __repr__(self):
         return self.repr()
-        ## return '<%s (%s)>' % (self.name, ', '.join([repr(a)
-        ##                                             for a in self.args]))
 
 class SimpleParser(OpParser):
 
@@ -145,18 +141,27 @@
     is_bytecode = True
     inline_level = None
 
-    def __init__(self, operations, storage):
-        if operations[0].name == 'debug_merge_point':
-            self.inline_level = int(operations[0].args[0])
-            m = re.search('<code object ([<>\w]+)\. file \'(.+?)\'\. line (\d+)> #(\d+) (\w+)',
-                         operations[0].args[1])
-            if m is None:
-                # a non-code loop, like StrLiteralSearch or something
-                self.bytecode_name = operations[0].args[1][1:-1]
-            else:
-                self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
-                self.startlineno = int(lineno)
-                self.bytecode_no = int(bytecode_no)
+    def parse_code_data(self, arg):
+        m = re.search('<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
+                      arg)
+        if m is None:
+            # a non-code loop, like StrLiteralSearch or something
+            self.bytecode_name = arg
+        else:
+            self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
+            self.startlineno = int(lineno)
+            self.bytecode_no = int(bytecode_no)
+
+
+    def __init__(self, operations, storage, loopname):
+        for op in operations:
+            if op.name == 'debug_merge_point':
+                self.inline_level = int(op.args[0])
+                self.parse_code_data(op.args[1][1:-1])
+                break
+        else:
+            self.inline_level = 0
+            self.parse_code_data(loopname)
         self.operations = operations
         self.storage = storage
         self.code = storage.disassemble_code(self.filename, self.startlineno,
@@ -164,7 +169,7 @@
 
     def repr(self):
         if self.filename is None:
-            return "Unknown"
+            return self.bytecode_name
         return "%s, file '%s', line %d" % (self.name, self.filename,
                                            self.startlineno)
 
@@ -180,7 +185,10 @@
         return self.code.map[self.bytecode_no]
 
     def getlineno(self):
-        return self.getopcode().lineno
+        code = self.getopcode()
+        if code is None:
+            return None
+        return code.lineno
     lineno = property(getlineno)
 
     def getline_starts_here(self):
@@ -219,7 +227,8 @@
         self.storage = storage
 
     @classmethod
-    def from_operations(cls, operations, storage, limit=None, inputargs=''):
+    def from_operations(cls, operations, storage, limit=None, inputargs='',
+                        loopname=''):
         """ Slice given operation list into a chain of TraceForOpcode chunks.
         Also detect inlined functions and make them Function
         """
@@ -245,13 +254,13 @@
         for op in operations:
             if op.name == 'debug_merge_point':
                 if so_far:
-                    append_to_res(cls.TraceForOpcode(so_far, storage))
+                    append_to_res(cls.TraceForOpcode(so_far, storage, loopname))
                     if limit:
                         break
                     so_far = []
             so_far.append(op)
         if so_far:
-            append_to_res(cls.TraceForOpcode(so_far, storage))
+            append_to_res(cls.TraceForOpcode(so_far, storage, loopname))
         # wrap stack back up
         if not stack:
             # no ops whatsoever
@@ -299,7 +308,7 @@
 
     def repr(self):
         if self.filename is None:
-            return "Unknown"
+            return self.chunks[0].bytecode_name
         return "%s, file '%s', line %d" % (self.name, self.filename,
                                            self.startlineno)
 
@@ -384,9 +393,30 @@
                               parser.postprocess(loop, backend_tp=bname,
                                                  backend_dump=dump,
                                                  dump_start=start_ofs))
-        loops.append(loop)
+        loops += split_trace(loop)
     return log, loops
 
+def split_trace(trace):
+    labels = [0]
+    if trace.comment and 'Guard' in trace.comment:
+        descrs = ['bridge ' + re.search('Guard (\d+)', trace.comment).group(1)]
+    else:
+        descrs = ['entry ' + re.search('Loop (\d+)', trace.comment).group(1)]
+    for i, op in enumerate(trace.operations):
+        if op.name == 'label':
+            labels.append(i)
+            descrs.append(op.descr)
+    labels.append(len(trace.operations) - 1)
+    parts = []
+    for i in range(len(labels) - 1):
+        start, stop = labels[i], labels[i+1]
+        part = copy(trace)
+        part.operations = trace.operations[start : stop + 1]
+        part.descr = descrs[i]
+        part.comment = trace.comment
+        parts.append(part)
+    
+    return parts
 
 def parse_log_counts(input, loops):
     if not input:
@@ -394,11 +424,7 @@
     lines = input[-1].splitlines()
     mapping = {}
     for loop in loops:
-        com = loop.comment
-        if 'Loop' in com:
-            mapping['loop ' + re.search('Loop (\d+)', com).group(1)] = loop
-        else:
-            mapping['bridge ' + re.search('Guard (\d+)', com).group(1)] = loop
+        mapping[loop.descr] = loop
     for line in lines:
         if line:
             num, count = line.split(':', 2)
diff --git a/pypy/tool/jitlogparser/storage.py b/pypy/tool/jitlogparser/storage.py
--- a/pypy/tool/jitlogparser/storage.py
+++ b/pypy/tool/jitlogparser/storage.py
@@ -6,7 +6,6 @@
 import py
 import os
 from lib_pypy.disassembler import dis
-from pypy.tool.jitlogparser.parser import Function
 from pypy.tool.jitlogparser.module_finder import gather_all_code_objs
 
 class LoopStorage(object):
diff --git a/pypy/tool/jitlogparser/test/test_modulefinder.py b/pypy/tool/jitlogparser/test/test_modulefinder.py
--- a/pypy/tool/jitlogparser/test/test_modulefinder.py
+++ b/pypy/tool/jitlogparser/test/test_modulefinder.py
@@ -7,12 +7,14 @@
         py.test.skip("Specific python 2.6 tests")
 
 def test_gather_code_py():
+    py.test.skip("XXX broken, fix me")
     fname = re.__file__
     codes = gather_all_code_objs(fname)
     assert len(codes) == 21
     assert sorted(codes.keys()) == [102, 134, 139, 144, 153, 164, 169, 181, 188, 192, 197, 206, 229, 251, 266, 271, 277, 285, 293, 294, 308]
 
 def test_load_code():
+    py.test.skip("XXX broken, fix me")
     fname = re.__file__
     code = gather_all_code_objs(fname)[144]
     assert code.co_name == 'sub'
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -1,6 +1,7 @@
 from pypy.tool.jitlogparser.parser import (SimpleParser, TraceForOpcode,
                                            Function, adjust_bridges,
-                                           import_log, Op)
+                                           import_log, split_trace, Op,
+                                           parse_log_counts)
 from pypy.tool.jitlogparser.storage import LoopStorage
 import py, sys
 
@@ -32,23 +33,26 @@
     ''')
     res = Function.from_operations(ops.operations, LoopStorage())
     assert len(res.chunks) == 1
-    assert res.chunks[0].repr()
+    assert 'SomeRandomStuff' in res.chunks[0].repr()
 
 def test_split():
     ops = parse('''
     [i0]
+    label()
     debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
     debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
     i1 = int_add(i0, 1)
     debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
     i2 = int_add(i1, 1)
     ''')
-    res = Function.from_operations(ops.operations, LoopStorage())
-    assert len(res.chunks) == 3
+    res = Function.from_operations(ops.operations, LoopStorage(), loopname='<loopname>')
+    assert len(res.chunks) == 4
     assert len(res.chunks[0].operations) == 1
-    assert len(res.chunks[1].operations) == 2
+    assert len(res.chunks[1].operations) == 1
     assert len(res.chunks[2].operations) == 2
-    assert res.chunks[2].bytecode_no == 11
+    assert len(res.chunks[3].operations) == 2
+    assert res.chunks[3].bytecode_no == 11
+    assert res.chunks[0].bytecode_name == '<loopname>'
 
 def test_inlined_call():
     ops = parse("""
@@ -231,3 +235,51 @@
     myrepr = 'c = foobar(a, b, descr=mydescr)'
     assert op.repr() == myrepr
     assert op.repr() == myrepr # do it twice
+
+def test_split_trace():
+    loop = parse('''
+    [i7]
+    i9 = int_lt(i7, 1003)
+    label(i9, descr=grrr)
+    guard_true(i9, descr=<Guard2>) []
+    i13 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+    label(i13, descr=asb)
+    i19 = int_lt(i13, 1003)
+    guard_true(i19, descr=<Guard2>) []
+    i113 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+    ''')
+    loop.comment = 'Loop 0'
+    parts = split_trace(loop)
+    assert len(parts) == 3
+    assert len(parts[0].operations) == 2
+    assert len(parts[1].operations) == 4
+    assert len(parts[2].operations) == 4
+    assert parts[1].descr == 'grrr'
+    assert parts[2].descr == 'asb'
+
+def test_parse_log_counts():
+    loop = parse('''
+    [i7]
+    i9 = int_lt(i7, 1003)
+    label(i9, descr=grrr)
+    guard_true(i9, descr=<Guard2>) []
+    i13 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+    label(i13, descr=asb)
+    i19 = int_lt(i13, 1003)
+    guard_true(i19, descr=<Guard3>) []
+    i113 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+    ''')
+    bridge = parse('''
+    # bridge out of Guard 2 with 1 ops
+    []
+    i0 = int_lt(1, 2)
+    finish(i0)
+    ''')
+    bridge.comment = 'bridge out of Guard 2 with 1 ops'
+    loop.comment = 'Loop 0'
+    loops = split_trace(loop) + split_trace(bridge)
+    input = ['grrr:123\nasb:12\nbridge 2:1234']
+    parse_log_counts(input, loops)
+    assert loops[-1].count == 1234
+    assert loops[1].count == 123
+    assert loops[2].count == 12
diff --git a/pypy/translator/generator.py b/pypy/translator/generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/generator.py
@@ -0,0 +1,179 @@
+from pypy.objspace.flow.model import Block, Link, SpaceOperation, checkgraph
+from pypy.objspace.flow.model import Variable, Constant, FunctionGraph
+from pypy.translator.unsimplify import insert_empty_startblock
+from pypy.translator.unsimplify import split_block
+from pypy.translator.simplify import eliminate_empty_blocks, simplify_graph
+from pypy.tool.sourcetools import func_with_new_name
+from pypy.interpreter.argument import Signature
+
+
+class AbstractPosition(object):
+    _immutable_ = True
+    _attrs_ = ()
+
+
+def tweak_generator_graph(graph):
+    if not hasattr(graph.func, '_generator_next_method_of_'):
+        # This is the first copy of the graph.  We replace it with
+        # a small bootstrap graph.
+        GeneratorIterator = make_generatoriterator_class(graph)
+        replace_graph_with_bootstrap(GeneratorIterator, graph)
+        # We attach a 'next' method to the GeneratorIterator class
+        # that will invoke the real function, based on a second
+        # copy of the graph.
+        attach_next_method(GeneratorIterator, graph)
+    else:
+        # This is the second copy of the graph.  Tweak it.
+        GeneratorIterator = graph.func._generator_next_method_of_
+        tweak_generator_body_graph(GeneratorIterator.Entry, graph)
+
+
+def make_generatoriterator_class(graph):
+    class GeneratorIterator(object):
+        class Entry(AbstractPosition):
+            _immutable_ = True
+            varnames = get_variable_names(graph.startblock.inputargs)
+        def __init__(self, entry):
+            self.current = entry
+    return GeneratorIterator
+
+def replace_graph_with_bootstrap(GeneratorIterator, graph):
+    Entry = GeneratorIterator.Entry
+    newblock = Block(graph.startblock.inputargs)
+    v_generator = Variable('generator')
+    v_entry = Variable('entry')
+    newblock.operations.append(
+        SpaceOperation('simple_call', [Constant(Entry)], v_entry))
+    assert len(graph.startblock.inputargs) == len(Entry.varnames)
+    for v, name in zip(graph.startblock.inputargs, Entry.varnames):
+        newblock.operations.append(
+            SpaceOperation('setattr', [v_entry, Constant(name), v],
+                           Variable()))
+    newblock.operations.append(
+        SpaceOperation('simple_call', [Constant(GeneratorIterator), v_entry],
+                       v_generator))
+    newblock.closeblock(Link([v_generator], graph.returnblock))
+    graph.startblock = newblock
+
+def attach_next_method(GeneratorIterator, graph):
+    func = graph.func
+    func = func_with_new_name(func, '%s__next' % (func.func_name,))
+    func._generator_next_method_of_ = GeneratorIterator
+    func._always_inline_ = True
+    #
+    def next(self):
+        entry = self.current
+        self.current = None
+        assert entry is not None      # else, recursive generator invocation
+        (next_entry, return_value) = func(entry)
+        self.current = next_entry
+        return return_value
+    GeneratorIterator.next = next
+    return func   # for debugging
+
+def get_variable_names(variables):
+    seen = set()
+    result = []
+    for v in variables:
+        name = v._name.strip('_')
+        while name in seen:
+            name += '_'
+        result.append('g_' + name)
+        seen.add(name)
+    return result
+
+def _insert_reads(block, varnames):
+    assert len(varnames) == len(block.inputargs)
+    v_entry1 = Variable('entry')
+    for i, name in enumerate(varnames):
+        block.operations.insert(i,
+            SpaceOperation('getattr', [v_entry1, Constant(name)],
+                           block.inputargs[i]))
+    block.inputargs = [v_entry1]
+
+def tweak_generator_body_graph(Entry, graph):
+    # First, always run simplify_graph in order to reduce the number of
+    # variables passed around
+    simplify_graph(graph)
+    #
+    assert graph.startblock.operations[0].opname == 'generator_mark'
+    graph.startblock.operations.pop(0)
+    #
+    insert_empty_startblock(None, graph)
+    _insert_reads(graph.startblock, Entry.varnames)
+    Entry.block = graph.startblock
+    #
+    mappings = [Entry]
+    #
+    stopblock = Block([])
+    v0 = Variable(); v1 = Variable()
+    stopblock.operations = [
+        SpaceOperation('simple_call', [Constant(StopIteration)], v0),
+        SpaceOperation('type', [v0], v1),
+        ]
+    stopblock.closeblock(Link([v1, v0], graph.exceptblock))
+    #
+    for block in list(graph.iterblocks()):
+        for exit in block.exits:
+            if exit.target is graph.returnblock:
+                exit.args = []
+                exit.target = stopblock
+        assert block is not stopblock
+        for index in range(len(block.operations)-1, -1, -1):
+            op = block.operations[index]
+            if op.opname == 'yield':
+                [v_yielded_value] = op.args
+                del block.operations[index]
+                newlink = split_block(None, block, index)
+                newblock = newlink.target
+                #
+                class Resume(AbstractPosition):
+                    _immutable_ = True
+                    block = newblock
+                Resume.__name__ = 'Resume%d' % len(mappings)
+                mappings.append(Resume)
+                varnames = get_variable_names(newlink.args)
+                #
+                _insert_reads(newblock, varnames)
+                #
+                v_resume = Variable('resume')
+                block.operations.append(
+                    SpaceOperation('simple_call', [Constant(Resume)],
+                                   v_resume))
+                for i, name in enumerate(varnames):
+                    block.operations.append(
+                        SpaceOperation('setattr', [v_resume, Constant(name),
+                                                   newlink.args[i]],
+                                       Variable()))
+                v_pair = Variable('pair')
+                block.operations.append(
+                    SpaceOperation('newtuple', [v_resume, v_yielded_value],
+                                   v_pair))
+                newlink.args = [v_pair]
+                newlink.target = graph.returnblock
+    #
+    regular_entry_block = Block([Variable('entry')])
+    block = regular_entry_block
+    for Resume in mappings:
+        v_check = Variable()
+        block.operations.append(
+            SpaceOperation('simple_call', [Constant(isinstance),
+                                           block.inputargs[0],
+                                           Constant(Resume)],
+                           v_check))
+        block.exitswitch = v_check
+        link1 = Link([block.inputargs[0]], Resume.block)
+        link1.exitcase = True
+        nextblock = Block([Variable('entry')])
+        link2 = Link([block.inputargs[0]], nextblock)
+        link2.exitcase = False
+        block.closeblock(link1, link2)
+        block = nextblock
+    block.closeblock(Link([Constant(AssertionError),
+                           Constant(AssertionError("bad generator class"))],
+                          graph.exceptblock))
+    graph.startblock = regular_entry_block
+    graph.signature = Signature(['entry'])
+    graph.defaults = ()
+    checkgraph(graph)
+    eliminate_empty_blocks(graph)
diff --git a/pypy/translator/sandbox/pypy_interact.py b/pypy/translator/sandbox/pypy_interact.py
--- a/pypy/translator/sandbox/pypy_interact.py
+++ b/pypy/translator/sandbox/pypy_interact.py
@@ -13,7 +13,8 @@
                   ATM this only works with PyPy translated with Boehm or
                   the semispace or generation GCs.
     --timeout=N   limit execution time to N (real-time) seconds.
-    --log=FILE    log all user input into the FILE
+    --log=FILE    log all user input into the FILE.
+    --verbose     log all proxied system calls.
 
 Note that you can get readline-like behavior with a tool like 'ledit',
 provided you use enough -u options:
@@ -26,18 +27,19 @@
 from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
 from pypy.translator.sandbox.sandlib import VirtualizedSandboxedProc
 from pypy.translator.sandbox.vfs import Dir, RealDir, RealFile
-from pypy.tool.lib_pypy import LIB_ROOT
+import pypy
+LIB_ROOT = os.path.dirname(os.path.dirname(pypy.__file__))
 
 class PyPySandboxedProc(VirtualizedSandboxedProc, SimpleIOSandboxedProc):
-    debug = True
     argv0 = '/bin/pypy-c'
     virtual_cwd = '/tmp'
     virtual_env = {}
     virtual_console_isatty = True
 
-    def __init__(self, executable, arguments, tmpdir=None):
+    def __init__(self, executable, arguments, tmpdir=None, debug=True):
         self.executable = executable = os.path.abspath(executable)
         self.tmpdir = tmpdir
+        self.debug = debug
         super(PyPySandboxedProc, self).__init__([self.argv0] + arguments,
                                                 executable=executable)
 
@@ -67,12 +69,13 @@
 
 if __name__ == '__main__':
     from getopt import getopt      # and not gnu_getopt!
-    options, arguments = getopt(sys.argv[1:], 't:h', 
+    options, arguments = getopt(sys.argv[1:], 't:hv', 
                                 ['tmp=', 'heapsize=', 'timeout=', 'log=',
-                                 'help'])
+                                 'verbose', 'help'])
     tmpdir = None
     timeout = None
     logfile = None
+    debug = False
     extraoptions = []
 
     def help():
@@ -104,6 +107,8 @@
             timeout = int(value)
         elif option == '--log':
             logfile = value
+        elif option in ['-v', '--verbose']:
+            debug = True
         elif option in ['-h', '--help']:
             help()
         else:
@@ -113,7 +118,7 @@
         help()
 
     sandproc = PyPySandboxedProc(arguments[0], extraoptions + arguments[1:],
-                                 tmpdir=tmpdir)
+                                 tmpdir=tmpdir, debug=debug)
     if timeout is not None:
         sandproc.settimeout(timeout, interrupt_main=True)
     if logfile is not None:
diff --git a/pypy/translator/sandbox/sandlib.py b/pypy/translator/sandbox/sandlib.py
--- a/pypy/translator/sandbox/sandlib.py
+++ b/pypy/translator/sandbox/sandlib.py
@@ -4,25 +4,29 @@
 for the outer process, which can run CPython or PyPy.
 """
 
-import py
 import sys, os, posixpath, errno, stat, time
-from pypy.tool.ansi_print import AnsiLog
 import subprocess
 from pypy.tool.killsubprocess import killsubprocess
 from pypy.translator.sandbox.vfs import UID, GID
+import py
 
-class MyAnsiLog(AnsiLog):
-    KW_TO_COLOR = {
-        'call': ((34,), False),
-        'result': ((34,), False),
-        'exception': ((34,), False),
-        'vpath': ((35,), False),
-        'timeout': ((1, 31), True),
-        }
+def create_log():
+    """Make and return a log for the sandbox to use, if needed."""
+    # These imports are local to avoid importing pypy if we don't need to.
+    from pypy.tool.ansi_print import AnsiLog
 
-log = py.log.Producer("sandlib")
-py.log.setconsumer("sandlib", MyAnsiLog())
+    class MyAnsiLog(AnsiLog):
+        KW_TO_COLOR = {
+            'call': ((34,), False),
+            'result': ((34,), False),
+            'exception': ((34,), False),
+            'vpath': ((35,), False),
+            'timeout': ((1, 31), True),
+            }
 
+    log = py.log.Producer("sandlib")
+    py.log.setconsumer("sandlib", MyAnsiLog())
+    return log
 
 # Note: we use lib_pypy/marshal.py instead of the built-in marshal
 # for two reasons.  The built-in module could be made to segfault
@@ -30,8 +34,9 @@
 # load().  Also, marshal.load(f) blocks with the GIL held when
 # f is a pipe with no data immediately avaialble, preventing the
 # _waiting_thread to run.
-from pypy.tool.lib_pypy import import_from_lib_pypy
-marshal = import_from_lib_pypy('marshal')
+import pypy
+marshal = py.path.local(pypy.__file__).join('..', '..', 'lib_pypy',
+                                            'marshal.py').pyimport()
 
 # Non-marshal result types
 RESULTTYPE_STATRESULT = object()
@@ -126,6 +131,7 @@
     for the external functions xxx that you want to support.
     """
     debug = False
+    log = None
     os_level_sandboxing = False   # Linux only: /proc/PID/seccomp
 
     def __init__(self, args, executable=None):
@@ -142,6 +148,9 @@
         self.currenttimeout = None
         self.currentlyidlefrom = None
 
+        if self.debug:
+            self.log = create_log()
+
     def withlock(self, function, *args, **kwds):
         lock = self.popenlock
         if lock is not None:
@@ -169,7 +178,8 @@
                 if delay <= 0.0:
                     break   # expired!
                 time.sleep(min(delay*1.001, 1))
-            log.timeout("timeout!")
+            if self.log:
+                self.log.timeout("timeout!")
             self.kill()
             #if interrupt_main:
             #    if hasattr(os, 'kill'):
@@ -246,22 +256,22 @@
                 args   = read_message(child_stdout)
             except EOFError, e:
                 break
-            if self.debug and not self.is_spam(fnname, *args):
-                log.call('%s(%s)' % (fnname,
+            if self.log and not self.is_spam(fnname, *args):
+                self.log.call('%s(%s)' % (fnname,
                                      ', '.join([shortrepr(x) for x in args])))
             try:
                 answer, resulttype = self.handle_message(fnname, *args)
             except Exception, e:
                 tb = sys.exc_info()[2]
                 write_exception(child_stdin, e, tb)
-                if self.debug:
+                if self.log:
                     if str(e):
-                        log.exception('%s: %s' % (e.__class__.__name__, e))
+                        self.log.exception('%s: %s' % (e.__class__.__name__, e))
                     else:
-                        log.exception('%s' % (e.__class__.__name__,))
+                        self.log.exception('%s' % (e.__class__.__name__,))
             else:
-                if self.debug and not self.is_spam(fnname, *args):
-                    log.result(shortrepr(answer))
+                if self.log and not self.is_spam(fnname, *args):
+                    self.log.result(shortrepr(answer))
                 try:
                     write_message(child_stdin, 0)  # error code - 0 for ok
                     write_message(child_stdin, answer, resulttype)
@@ -440,7 +450,8 @@
             node = dirnode.join(name)
         else:
             node = dirnode
-        log.vpath('%r => %r' % (vpath, node))
+        if self.log:
+            self.log.vpath('%r => %r' % (vpath, node))
         return node
 
     def do_ll_os__ll_os_stat(self, vpathname):
diff --git a/pypy/translator/test/test_generator.py b/pypy/translator/test/test_generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/test/test_generator.py
@@ -0,0 +1,156 @@
+from pypy.conftest import option
+from pypy.objspace.flow.objspace import FlowObjSpace
+from pypy.objspace.flow.model import Variable
+from pypy.interpreter.argument import Signature
+from pypy.translator.translator import TranslationContext
+from pypy.translator.generator import make_generatoriterator_class
+from pypy.translator.generator import replace_graph_with_bootstrap
+from pypy.translator.generator import get_variable_names
+from pypy.translator.generator import tweak_generator_body_graph
+from pypy.translator.generator import attach_next_method
+from pypy.translator.simplify import join_blocks
+
+
+# ____________________________________________________________
+
+def f_gen(n):
+    i = 0
+    while i < n:
+        yield i
+        i += 1
+
+class GeneratorIterator(object):
+    def __init__(self, entry):
+        self.current = entry
+    def next(self):
+        e = self.current
+        self.current = None
+        if isinstance(e, Yield1):
+            n = e.n_0
+            i = e.i_0
+            i += 1
+        else:
+            n = e.n_0
+            i = 0
+        if i < n:
+            e = Yield1()
+            e.n_0 = n
+            e.i_0 = i
+            self.current = e
+            return i
+        raise StopIteration
+
+    def __iter__(self):
+        return self
+
+class AbstractPosition(object):
+    _immutable_ = True
+class Entry1(AbstractPosition):
+    _immutable_ = True
+class Yield1(AbstractPosition):
+    _immutable_ = True
+
+def f_explicit(n):
+    e = Entry1()
+    e.n_0 = n
+    return GeneratorIterator(e)
+
+def test_explicit():
+    assert list(f_gen(10)) == list(f_explicit(10))
+
+def test_get_variable_names():
+    lst = get_variable_names([Variable('a'), Variable('b_'), Variable('a')])
+    assert lst == ['g_a', 'g_b', 'g_a_']
+
+# ____________________________________________________________
+
+
+class TestGenerator:
+
+    def test_replace_graph_with_bootstrap(self):
+        def func(n, x, y, z):
+            yield n
+            yield n
+        #
+        space = FlowObjSpace()
+        graph = space.build_flow(func, tweak_for_generator=False)
+        assert graph.startblock.operations[0].opname == 'generator_mark'
+        GeneratorIterator = make_generatoriterator_class(graph)
+        replace_graph_with_bootstrap(GeneratorIterator, graph)
+        if option.view:
+            graph.show()
+        block = graph.startblock
+        ops = block.operations
+        assert ops[0].opname == 'simple_call' # e = Entry1()
+        assert ops[1].opname == 'setattr'     # e.g_n = n
+        assert ops[1].args[1].value == 'g_n'
+        assert ops[2].opname == 'setattr'     # e.g_x = x
+        assert ops[2].args[1].value == 'g_x'
+        assert ops[3].opname == 'setattr'     # e.g_y = y
+        assert ops[3].args[1].value == 'g_y'
+        assert ops[4].opname == 'setattr'     # e.g_z = z
+        assert ops[4].args[1].value == 'g_z'
+        assert ops[5].opname == 'simple_call' # g = GeneratorIterator(e)
+        assert ops[5].args[1] == ops[0].result
+        assert len(ops) == 6
+        assert len(block.exits) == 1
+        assert block.exits[0].target is graph.returnblock
+
+    def test_tweak_generator_body_graph(self):
+        def f(n, x, y, z=3):
+            z *= 10
+            yield n + 1
+            z -= 10
+        #
+        space = FlowObjSpace()
+        graph = space.build_flow(f, tweak_for_generator=False)
+        class Entry:
+            varnames = ['g_n', 'g_x', 'g_y', 'g_z']
+        tweak_generator_body_graph(Entry, graph)
+        if option.view:
+            graph.show()
+        # XXX how to test directly that the graph is correct?  :-(
+        assert len(graph.startblock.inputargs) == 1
+        assert graph.signature == Signature(['entry'])
+        assert graph.defaults == ()
+
+    def test_tweak_generator_graph(self):
+        def f(n, x, y, z):
+            z *= 10
+            yield n + 1
+            z -= 10
+        #
+        space = FlowObjSpace()
+        graph = space.build_flow(f, tweak_for_generator=False)
+        GeneratorIterator = make_generatoriterator_class(graph)
+        replace_graph_with_bootstrap(GeneratorIterator, graph)
+        func1 = attach_next_method(GeneratorIterator, graph)
+        if option.view:
+            graph.show()
+        #
+        assert func1._generator_next_method_of_ is GeneratorIterator
+        assert hasattr(GeneratorIterator, 'next')
+        #
+        graph_next = space.build_flow(GeneratorIterator.next.im_func)
+        join_blocks(graph_next)
+        if option.view:
+            graph_next.show()
+        #
+        graph1 = space.build_flow(func1, tweak_for_generator=False)
+        tweak_generator_body_graph(GeneratorIterator.Entry, graph1)
+        if option.view:
+            graph1.show()
+
+    def test_automatic(self):
+        def f(n, x, y, z):
+            z *= 10
+            yield n + 1
+            z -= 10
+        #
+        space = FlowObjSpace()
+        graph = space.build_flow(f)   # tweak_for_generator=True
+        if option.view:
+            graph.show()
+        block = graph.startblock
+        assert len(block.exits) == 1
+        assert block.exits[0].target is graph.returnblock


More information about the pypy-commit mailing list