[pypy-commit] pypy jit-targets: hg merge default

hakanardo noreply at buildbot.pypy.org
Sat Dec 10 12:53:02 CET 2011


Author: Hakan Ardo <hakan at debian.org>
Branch: jit-targets
Changeset: r50341:8474a7603bbe
Date: 2011-12-10 12:52 +0100
http://bitbucket.org/pypy/pypy/changeset/8474a7603bbe/

Log:	hg merge default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -487,6 +487,16 @@
         'parser', 'fcntl', '_codecs', 'binascii'
     ]
 
+    # These modules are treated like CPython treats built-in modules,
+    # i.e. they always shadow any xx.py.  The other modules are treated
+    # like CPython treats extension modules, and are loaded in sys.path
+    # order by the fake entry '.../lib_pypy/__extensions__'.
+    MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([
+        '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings',
+        '_weakref', 'errno', 'exceptions', 'gc', 'imp', 'marshal',
+        'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport',
+    ], None)
+
     def make_builtins(self):
         "NOT_RPYTHON: only for initializing the space."
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -16,28 +16,53 @@
     """ Manage frame positions
     """
     def __init__(self):
-        self.frame_bindings = {}
-        self.frame_depth    = 0
+        self.bindings = {}
+        self.used = []      # list of bools
+        self.hint_frame_locations = {}
+
+    frame_depth = property(lambda:xxx, lambda:xxx)   # XXX kill me
+
+    def get_frame_depth(self):
+        return len(self.used)
 
     def get(self, box):
-        return self.frame_bindings.get(box, None)
+        return self.bindings.get(box, None)
 
     def loc(self, box):
+        """Return or create the frame location associated with 'box'."""
+        # first check if it's already in the frame_manager
         try:
-            return self.frame_bindings[box]
+            return self.bindings[box]
         except KeyError:
-            return self.get_new_loc(box)
+            pass
+        # check if we have a hint for this box
+        if box in self.hint_frame_locations:
+            # if we do, try to reuse the location for this box
+            loc = self.hint_frame_locations[box]
+            if self.try_to_reuse_location(box, loc):
+                return loc
+        # no valid hint.  make up a new free location
+        return self.get_new_loc(box)
 
     def get_new_loc(self, box):
         size = self.frame_size(box.type)
-        self.frame_depth += ((-self.frame_depth) & (size-1))
-        # ^^^ frame_depth is rounded up to a multiple of 'size', assuming
+        # frame_depth is rounded up to a multiple of 'size', assuming
         # that 'size' is a power of two.  The reason for doing so is to
         # avoid obscure issues in jump.py with stack locations that try
         # to move from position (6,7) to position (7,8).
-        newloc = self.frame_pos(self.frame_depth, box.type)
-        self.frame_bindings[box] = newloc
-        self.frame_depth += size
+        while self.get_frame_depth() & (size - 1):
+            self.used.append(False)
+        #
+        index = self.get_frame_depth()
+        newloc = self.frame_pos(index, box.type)
+        for i in range(size):
+            self.used.append(True)
+        #
+        if not we_are_translated():    # extra testing
+            testindex = self.get_loc_index(newloc)
+            assert testindex == index
+        #
+        self.bindings[box] = newloc
         return newloc
 
     def forget_frame_allocation(self, box):
@@ -46,11 +71,52 @@
         except KeyError:
             pass
 
+    def set_binding(self, box, loc):
+        self.bindings[box] = loc
+        #
+        index = self.get_loc_index(loc)
+        endindex = index + self.frame_size(box.type)
+        while len(self.used) < endindex:
+            self.used.append(False)
+        while index < endindex:
+            self.used[index] = True
+            index += 1
+
     def reserve_location_in_frame(self, size):
-        frame_depth = self.frame_depth
-        self.frame_depth += size
+        frame_depth = self.get_frame_depth()
+        for i in range(size):
+            self.used.append(True)
         return frame_depth
 
+    def mark_as_free(self, box):
+        try:
+            loc = self.bindings[box]
+        except KeyError:
+            return    # already gone
+        del self.bindings[box]
+        #
+        size = self.frame_size(box.type)
+        baseindex = self.get_loc_index(loc)
+        for i in range(size):
+            index = baseindex + i
+            assert 0 <= index < len(self.used)
+            self.used[index] = False
+
+    def try_to_reuse_location(self, box, loc):
+        index = self.get_loc_index(loc)
+        assert index >= 0
+        size = self.frame_size(box.type)
+        for i in range(size):
+            while (index + i) >= len(self.used):
+                self.used.append(False)
+            if self.used[index + i]:
+                return False    # already in use
+        # good, we can reuse the location
+        for i in range(size):
+            self.used[index + i] = True
+        self.bindings[box] = loc
+        return True
+
     # abstract methods that need to be overwritten for specific assemblers
     @staticmethod
     def frame_pos(loc, type):
@@ -58,6 +124,10 @@
     @staticmethod
     def frame_size(type):
         return 1
+    @staticmethod
+    def get_loc_index(loc):
+        raise NotImplementedError("Purely abstract")
+
 
 class RegisterManager(object):
     """ Class that keeps track of register allocations
@@ -76,8 +146,6 @@
         self.position = -1
         self.frame_manager = frame_manager
         self.assembler = assembler
-        self.hint_frame_locations = {}     # {Box: StackLoc}
-        self.freed_frame_locations = {}    # {StackLoc: None}
 
     def is_still_alive(self, v):
         # Check if 'v' is alive at the current position.
@@ -109,9 +177,7 @@
                 self.free_regs.append(self.reg_bindings[v])
                 del self.reg_bindings[v]
             if self.frame_manager is not None:
-                if v in self.frame_manager.frame_bindings:
-                    loc = self.frame_manager.frame_bindings[v]
-                    self.freed_frame_locations[loc] = None
+                self.frame_manager.mark_as_free(v)
 
     def possibly_free_vars(self, vars):
         """ Same as 'possibly_free_var', but for all v in vars.
@@ -183,23 +249,6 @@
                 self.reg_bindings[v] = loc
                 return loc
 
-    def _frame_loc(self, v):
-        # first check if it's already in the frame_manager
-        try:
-            return self.frame_manager.frame_bindings[v]
-        except KeyError:
-            pass
-        # check if we have a hint for this box
-        if v in self.hint_frame_locations:
-            # if we do, check that the hinted location is known to be free
-            loc = self.hint_frame_locations[v]
-            if loc in self.freed_frame_locations:
-                del self.freed_frame_locations[loc]
-                self.frame_manager.frame_bindings[v] = loc
-                return loc
-        # no valid hint.  make up a new free location
-        return self.frame_manager.get_new_loc(v)
-
     def _spill_var(self, v, forbidden_vars, selected_reg,
                    need_lower_byte=False):
         v_to_spill = self._pick_variable_to_spill(v, forbidden_vars,
@@ -207,7 +256,7 @@
         loc = self.reg_bindings[v_to_spill]
         del self.reg_bindings[v_to_spill]
         if self.frame_manager.get(v_to_spill) is None:
-            newloc = self._frame_loc(v_to_spill)
+            newloc = self.frame_manager.loc(v_to_spill)
             self.assembler.regalloc_mov(loc, newloc)
         return loc
 
@@ -284,7 +333,7 @@
         except KeyError:
             if box in self.bindings_to_frame_reg:
                 return self.frame_reg
-            return self._frame_loc(box)
+            return self.frame_manager.loc(box)
 
     def return_constant(self, v, forbidden_vars=[], selected_reg=None):
         """ Return the location of the constant v.  If 'selected_reg' is
@@ -332,7 +381,7 @@
             self.reg_bindings[v] = loc
             self.assembler.regalloc_mov(prev_loc, loc)
         else:
-            loc = self._frame_loc(v)
+            loc = self.frame_manager.loc(v)
             self.assembler.regalloc_mov(prev_loc, loc)
 
     def force_result_in_reg(self, result_v, v, forbidden_vars=[]):
@@ -351,7 +400,7 @@
             self.reg_bindings[result_v] = loc
             return loc
         if v not in self.reg_bindings:
-            prev_loc = self._frame_loc(v)
+            prev_loc = self.frame_manager.loc(v)
             loc = self.force_allocate_reg(v, forbidden_vars)
             self.assembler.regalloc_mov(prev_loc, loc)
         assert v in self.reg_bindings
@@ -371,7 +420,7 @@
     def _sync_var(self, v):
         if not self.frame_manager.get(v):
             reg = self.reg_bindings[v]
-            to = self._frame_loc(v)
+            to = self.frame_manager.loc(v)
             self.assembler.regalloc_mov(reg, to)
         # otherwise it's clean
 
diff --git a/pypy/jit/backend/llsupport/test/test_regalloc.py b/pypy/jit/backend/llsupport/test/test_regalloc.py
--- a/pypy/jit/backend/llsupport/test/test_regalloc.py
+++ b/pypy/jit/backend/llsupport/test/test_regalloc.py
@@ -42,8 +42,13 @@
     def frame_size(self, box_type):
         if box_type == FLOAT:
             return 2
+        elif box_type == INT:
+            return 1
         else:
-            return 1
+            raise ValueError(box_type)
+    def get_loc_index(self, loc):
+        assert isinstance(loc, FakeFramePos)
+        return loc.pos
 
 class MockAsm(object):
     def __init__(self):
@@ -282,7 +287,7 @@
             rm.force_allocate_reg(b)
         rm.before_call()
         assert len(rm.reg_bindings) == 2
-        assert fm.frame_depth == 2
+        assert fm.get_frame_depth() == 2
         assert len(asm.moves) == 2
         rm._check_invariants()
         rm.after_call(boxes[-1])
@@ -305,7 +310,7 @@
             rm.force_allocate_reg(b)
         rm.before_call(save_all_regs=True)
         assert len(rm.reg_bindings) == 0
-        assert fm.frame_depth == 4
+        assert fm.get_frame_depth() == 4
         assert len(asm.moves) == 4
         rm._check_invariants()
         rm.after_call(boxes[-1])
@@ -327,7 +332,7 @@
         xrm = XRegisterManager(longevity, frame_manager=fm, assembler=asm)
         xrm.loc(f0)
         rm.loc(b0)
-        assert fm.frame_depth == 3
+        assert fm.get_frame_depth() == 3
         
         
 
@@ -351,20 +356,14 @@
 
 
     def test_hint_frame_locations_1(self):
-        b0, b1 = newboxes(0, 1)
-        longevity = {b0: (0, 1), b1: (0, 1)}
+        b0, = newboxes(0)
         fm = TFrameManager()
-        asm = MockAsm()
-        rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
-        rm.hint_frame_locations[b0] = "some_stack_loc"
-        rm.freed_frame_locations["some_stack_loc"] = None
-        rm.force_allocate_reg(b0)
-        rm.force_allocate_reg(b1)
-        rm.force_spill_var(b0)
-        rm.force_spill_var(b1)
-        assert rm.loc(b0) == "some_stack_loc"
-        assert isinstance(rm.loc(b1), FakeFramePos)
-        rm._check_invariants()
+        loc123 = FakeFramePos(123, INT)
+        fm.hint_frame_locations[b0] = loc123
+        assert fm.get_frame_depth() == 0
+        loc = fm.loc(b0)
+        assert loc == loc123
+        assert fm.get_frame_depth() == 124
 
     def test_hint_frame_locations_2(self):
         b0, b1, b2 = newboxes(0, 1, 2)
@@ -378,20 +377,99 @@
         rm.force_spill_var(b0)
         loc = rm.loc(b0)
         assert isinstance(loc, FakeFramePos)
+        assert fm.get_loc_index(loc) == 0
         rm.position = 1
-        assert loc not in rm.freed_frame_locations
+        assert fm.used == [True]
         rm.possibly_free_var(b0)
-        assert loc in rm.freed_frame_locations
+        assert fm.used == [False]
         #
-        rm.hint_frame_locations[b1] = loc
+        fm.hint_frame_locations[b1] = loc
         rm.force_spill_var(b1)
         loc1 = rm.loc(b1)
-        assert loc1 is loc
-        assert rm.freed_frame_locations == {}
+        assert loc1 == loc
+        assert fm.used == [True]
         #
-        rm.hint_frame_locations[b2] = loc
+        fm.hint_frame_locations[b2] = loc
         rm.force_spill_var(b2)
         loc2 = rm.loc(b2)
-        assert loc2 is not loc1     # because it's not in freed_frame_locations
+        assert loc2 != loc1     # because it was not free
+        assert fm.used == [True, True]
         #
         rm._check_invariants()
+
+    def test_frame_manager_basic(self):
+        b0, b1 = newboxes(0, 1)
+        fm = TFrameManager()
+        loc0 = fm.loc(b0)
+        assert fm.get_loc_index(loc0) == 0
+        #
+        assert fm.get(b1) is None
+        loc1 = fm.loc(b1)
+        assert fm.get_loc_index(loc1) == 1
+        assert fm.get(b1) == loc1
+        #
+        loc0b = fm.loc(b0)
+        assert loc0b == loc0
+        #
+        fm.loc(BoxInt())
+        assert fm.get_frame_depth() == 3
+        #
+        f0 = BoxFloat()
+        locf0 = fm.loc(f0)
+        assert fm.get_loc_index(locf0) == 4
+        assert fm.get_frame_depth() == 6
+        #
+        f1 = BoxFloat()
+        locf1 = fm.loc(f1)
+        assert fm.get_loc_index(locf1) == 6
+        assert fm.get_frame_depth() == 8
+        assert fm.used == [True, True, True, False, True, True, True, True]
+        #
+        fm.mark_as_free(b0)
+        assert fm.used == [False, True, True, False, True, True, True, True]
+        fm.mark_as_free(b0)
+        assert fm.used == [False, True, True, False, True, True, True, True]
+        fm.mark_as_free(f1)
+        assert fm.used == [False, True, True, False, True, True, False, False]
+        #
+        fm.reserve_location_in_frame(1)
+        assert fm.get_frame_depth() == 9
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        #
+        assert b0 not in fm.bindings
+        fm.set_binding(b0, loc0)
+        assert b0 in fm.bindings
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        b3 = BoxInt()
+        assert not fm.try_to_reuse_location(b3, loc0)
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        fm.mark_as_free(b0)
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        assert fm.try_to_reuse_location(b3, loc0)
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        fm.mark_as_free(b0)   # already free
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        fm.mark_as_free(b3)
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        f3 = BoxFloat()
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(0, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(1, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(2, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(3, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(4, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(5, FLOAT))
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        assert fm.try_to_reuse_location(f3, fm.frame_pos(6, FLOAT))
+        assert fm.used == [False, True, True, False, True, True, True, True, True]
+        #
+        fm.used = [False]
+        assert fm.try_to_reuse_location(BoxFloat(), fm.frame_pos(0, FLOAT))
+        assert fm.used == [True, True]
+        #
+        fm.used = [True]
+        assert not fm.try_to_reuse_location(BoxFloat(), fm.frame_pos(0, FLOAT))
+        assert fm.used == [True]
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -702,7 +702,7 @@
         regalloc.walk_operations(operations)
         if we_are_translated() or self.cpu.dont_keepalive_stuff:
             self._regalloc = None   # else keep it around for debugging
-        frame_depth = regalloc.fm.frame_depth
+        frame_depth = regalloc.fm.get_frame_depth()
         param_depth = regalloc.param_depth
         jump_target_descr = regalloc.jump_target_descr
         if jump_target_descr is not None:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -139,6 +139,10 @@
             return 2
         else:
             return 1
+    @staticmethod
+    def get_loc_index(loc):
+        assert isinstance(loc, StackLoc)
+        return loc.position
 
 if WORD == 4:
     gpr_reg_mgr_cls = X86RegisterManager
@@ -185,7 +189,6 @@
                        allgcrefs):
         operations, _ = self._prepare(inputargs, operations, allgcrefs)
         self._update_bindings(arglocs, inputargs)
-        self.fm.frame_depth = prev_depths[0]
         self.param_depth = prev_depths[1]
         return operations
 
@@ -308,7 +311,7 @@
                     self.xrm.reg_bindings[arg] = loc
                     used[loc] = None
                 else:
-                    self.fm.frame_bindings[arg] = loc
+                    self.fm.set_binding(arg, loc)
             else:
                 if isinstance(loc, RegLoc):
                     if loc is ebp:
@@ -317,7 +320,7 @@
                         self.rm.reg_bindings[arg] = loc
                         used[loc] = None
                 else:
-                    self.fm.frame_bindings[arg] = loc
+                    self.fm.set_binding(arg, loc)
         self.rm.free_regs = []
         for reg in self.rm.all_regs:
             if reg not in used:
@@ -353,7 +356,7 @@
     def get_current_depth(self):
         # return (self.fm.frame_depth, self.param_depth), but trying to share
         # the resulting tuple among several calls
-        arg0 = self.fm.frame_depth
+        arg0 = self.fm.get_frame_depth()
         arg1 = self.param_depth
         result = self.assembler._current_depths_cache
         if result[0] != arg0 or result[1] != arg1:
@@ -1335,12 +1338,12 @@
                 loc = nonfloatlocs[i]
                 if isinstance(loc, StackLoc):
                     assert box.type != FLOAT
-                    self.rm.hint_frame_locations[box] = loc
+                    self.fm.hint_frame_locations[box] = loc
                 else:
                     loc = floatlocs[i]
                     if isinstance(loc, StackLoc):
                         assert box.type == FLOAT
-                        self.xrm.hint_frame_locations[box] = loc
+                        self.fm.hint_frame_locations[box] = loc
 
     def consider_jump(self, op):
         assembler = self.assembler
@@ -1386,7 +1389,7 @@
 
     def get_mark_gc_roots(self, gcrootmap, use_copy_area=False):
         shape = gcrootmap.get_basic_shape(IS_X86_64)
-        for v, val in self.fm.frame_bindings.items():
+        for v, val in self.fm.bindings.items():
             if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)):
                 assert isinstance(val, StackLoc)
                 gcrootmap.add_frame_offset(shape, get_ebp_ofs(val.position))
diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py
--- a/pypy/jit/backend/x86/test/test_recompilation.py
+++ b/pypy/jit/backend/x86/test/test_recompilation.py
@@ -44,6 +44,7 @@
         i5 = int_add(i4, 1)
         i6 = int_add(i5, 1)
         i7 = int_add(i5, i4)
+        force_spill(i5)
         i8 = int_add(i7, 1)
         i9 = int_add(i8, 1)
         finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2)
@@ -51,10 +52,9 @@
         bridge = self.attach_bridge(ops, loop, -2)
         descr = loop.operations[3].getdescr()
         new = descr._x86_bridge_frame_depth
-        assert descr._x86_bridge_param_depth == 0        
-        # XXX: Maybe add enough ops to force stack on 64-bit as well?
-        if IS_X86_32:
-            assert new > previous
+        assert descr._x86_bridge_param_depth == 0
+        # the force_spill() forces the stack to grow
+        assert new > previous
         self.cpu.set_future_value_int(0, 0)
         fail = self.run(loop)
         assert fail.identifier == 2
@@ -109,6 +109,9 @@
         i8 = int_add(i3, 1)
         i6 = int_add(i8, i10)
         i7 = int_add(i3, i6)
+        force_spill(i6)
+        force_spill(i7)
+        force_spill(i8)
         i12 = int_add(i7, i8)
         i11 = int_add(i12, i6)
         jump(i3, i12, i11, i10, i6, i7, descr=targettoken)
@@ -117,9 +120,8 @@
         guard_op = loop.operations[6]
         loop_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_depth
         assert loop._jitcelltoken.compiled_loop_token.param_depth == 0
-        # XXX: Maybe add enough ops to force stack on 64-bit as well?
-        if IS_X86_32:
-            assert guard_op.getdescr()._x86_bridge_frame_depth > loop_frame_depth
+        # the force_spill() forces the stack to grow
+        assert guard_op.getdescr()._x86_bridge_frame_depth > loop_frame_depth
         assert guard_op.getdescr()._x86_bridge_param_depth == 0
         self.cpu.set_future_value_int(0, 0)
         self.cpu.set_future_value_int(1, 0)
diff --git a/pypy/jit/metainterp/test/test_del.py b/pypy/jit/metainterp/test/test_del.py
--- a/pypy/jit/metainterp/test/test_del.py
+++ b/pypy/jit/metainterp/test/test_del.py
@@ -1,5 +1,7 @@
 import py
-from pypy.rlib.jit import JitDriver
+from pypy.rlib.jit import JitDriver, dont_look_inside
+from pypy.rlib.objectmodel import keepalive_until_here
+from pypy.rlib import rgc
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 
 
@@ -80,6 +82,47 @@
         assert res == 1
         self.check_resops(call=1)   # for the case B(), but not for the case A()
 
+    def test_keepalive(self):
+        py.test.skip("XXX fails")   # hum, I think the test itself is broken
+        #
+        mydriver = JitDriver(reds = ['n', 'states'], greens = [])
+        class State:
+            num = 1
+        class X:
+            def __init__(self, state):
+                self.state = state
+            def __del__(self):
+                self.state.num += 1
+        @dont_look_inside
+        def do_stuff():
+            pass
+        def f(n):
+            states = []
+            while n > 0:
+                mydriver.jit_merge_point(n=n, states=states)
+                state = State()
+                states.append(state)
+                x = X(state)
+                do_stuff()
+                state.num *= 1000
+                do_stuff()
+                keepalive_until_here(x)
+                n -= 1
+            return states
+        def main(n):
+            states = f(n)
+            rgc.collect()
+            rgc.collect()
+            err = 1001
+            for state in states:
+                if state.num != 1001:
+                    err = state.num
+                    print 'ERROR:', err
+            return err
+        assert main(20) == 1001
+        res = self.meta_interp(main, [20])
+        assert res == 1001
+
 
 class TestLLtype(DelTests, LLJitMixin):
     def test_signal_action(self):
diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py
--- a/pypy/jit/metainterp/test/test_fficall.py
+++ b/pypy/jit/metainterp/test/test_fficall.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 
 from pypy.jit.metainterp.test.support import LLJitMixin
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -864,6 +864,7 @@
         elif sys.platform.startswith('linux'):
             compile_extra.append("-Werror=implicit-function-declaration")
         export_symbols_eci.append('pypyAPI')
+        compile_extra.append('-g')
     else:
         kwds["includes"] = ['Python.h'] # this is our Python.h
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -483,10 +483,20 @@
     # XXX Check for frozen modules?
     #     when w_path is a string
 
+    delayed_builtin = None
+    w_lib_extensions = None
+
     if w_path is None:
         # check the builtin modules
         if modulename in space.builtin_modules:
-            return FindInfo(C_BUILTIN, modulename, None)
+            delayed_builtin = FindInfo(C_BUILTIN, modulename, None)
+            # a "real builtin module xx" shadows every file "xx.py" there
+            # could possibly be; a "pseudo-extension module" does not, and
+            # is only loaded at the point in sys.path where we find
+            # '.../lib_pypy/__extensions__'.
+            if modulename in space.MODULES_THAT_ALWAYS_SHADOW:
+                return delayed_builtin
+            w_lib_extensions = space.sys.get_state(space).w_lib_extensions
         w_path = space.sys.get('path')
 
     # XXX check frozen modules?
@@ -495,6 +505,9 @@
     if w_path is not None:
         for w_pathitem in space.unpackiterable(w_path):
             # sys.path_hooks import hook
+            if (w_lib_extensions is not None and
+                    space.eq_w(w_pathitem, w_lib_extensions)):
+                return delayed_builtin
             if use_loader:
                 w_loader = find_in_path_hooks(space, w_modulename, w_pathitem)
                 if w_loader:
@@ -527,7 +540,7 @@
                        # Out of file descriptors.
 
     # not found
-    return None
+    return delayed_builtin
 
 def _prepare_module(space, w_mod, filename, pkgdir):
     w = space.wrap
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -38,6 +38,8 @@
                     test_reload = "def test():\n    raise ValueError\n",
                     infinite_reload = "import infinite_reload; reload(infinite_reload)",
                     del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n",
+                    itertools = "hello_world = 42\n",
+                    gc = "should_never_be_seen = 42\n",
                     )
     root.ensure("notapackage", dir=1)    # empty, no __init__.py
     setuppkg("pkg",
@@ -147,6 +149,8 @@
 class AppTestImport:
 
     def setup_class(cls): # interpreter-level
+        cls.space = gettestobjspace(usemodules=['itertools'])
+        cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
         cls.saved_modules = _setup(cls.space)
         #XXX Compile class
 
@@ -571,6 +575,50 @@
         else:
             assert False, 'should not work'
 
+    def test_shadow_builtin(self):
+        if self.runappdirect: skip("hard to test: module is already imported")
+        # 'import gc' is supposed to always find the built-in module;
+        # like CPython, it is a built-in module, so it shadows everything,
+        # even though there is a gc.py.
+        import sys
+        assert 'gc' not in sys.modules
+        import gc
+        assert not hasattr(gc, 'should_never_be_seen')
+        assert '(built-in)' in repr(gc)
+        del sys.modules['gc']
+
+    def test_shadow_extension_1(self):
+        if self.runappdirect: skip("hard to test: module is already imported")
+        # 'import itertools' is supposed to find itertools.py if there is
+        # one in sys.path.
+        import sys
+        assert 'itertools' not in sys.modules
+        import itertools
+        assert hasattr(itertools, 'hello_world')
+        assert not hasattr(itertools, 'count')
+        assert '(built-in)' not in repr(itertools)
+        del sys.modules['itertools']
+
+    def test_shadow_extension_2(self):
+        if self.runappdirect: skip("hard to test: module is already imported")
+        # 'import itertools' is supposed to find the built-in module even
+        # if there is also one in sys.path as long as it is *after* the
+        # special entry '.../lib_pypy/__extensions__'.  (Note that for now
+        # there is one in lib_pypy/itertools.py, which should not be seen
+        # either; hence the (built-in) test below.)
+        import sys
+        assert 'itertools' not in sys.modules
+        sys.path.append(sys.path.pop(0))
+        try:
+            import itertools
+            assert not hasattr(itertools, 'hello_world')
+            assert hasattr(itertools, 'izip')
+            assert '(built-in)' in repr(itertools)
+        finally:
+            sys.path.insert(0, sys.path.pop())
+        del sys.modules['itertools']
+
+
 class TestAbi:
     def test_abi_tag(self):
         space1 = gettestobjspace(soabi='TEST')
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -170,3 +170,7 @@
     def get_flag(self, name):
         space = self.space
         return space.int_w(space.getattr(self.get('flags'), space.wrap(name)))
+
+    def get_state(self, space):
+        from pypy.module.sys import state
+        return state.get(space)
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -24,7 +24,7 @@
         # Initialize the default path
         pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
         srcdir = os.path.dirname(pypydir)
-        path = getinitialpath(srcdir)
+        path = getinitialpath(self, srcdir)
         self.w_path = space.newlist([space.wrap(p) for p in path])
 
 def checkdir(path):
@@ -35,7 +35,7 @@
 
 platform = sys.platform
 
-def getinitialpath(prefix):
+def getinitialpath(state, prefix):
     from pypy.module.sys.version import CPYTHON_VERSION
     dirname = '%d.%d' % (CPYTHON_VERSION[0],
                          CPYTHON_VERSION[1])
@@ -49,6 +49,12 @@
     checkdir(lib_pypy)
 
     importlist = []
+    #
+    if state is not None:    # 'None' for testing only
+        lib_extensions = os.path.join(lib_pypy, '__extensions__')
+        state.w_lib_extensions = state.space.wrap(lib_extensions)
+        importlist.append(lib_extensions)
+    #
     importlist.append(lib_pypy)
     importlist.append(python_std_lib_modified)
     importlist.append(python_std_lib)
@@ -71,7 +77,7 @@
 @unwrap_spec(srcdir=str)
 def pypy_initial_path(space, srcdir):
     try:
-        path = getinitialpath(srcdir)
+        path = getinitialpath(get(space), srcdir)
     except OSError:
         return space.w_None
     else:
diff --git a/pypy/module/sys/test/test_initialpath.py b/pypy/module/sys/test/test_initialpath.py
--- a/pypy/module/sys/test/test_initialpath.py
+++ b/pypy/module/sys/test/test_initialpath.py
@@ -13,7 +13,7 @@
 
 def test_stdlib_in_prefix(tmpdir):
     dirs = build_hierarchy(tmpdir)
-    path = getinitialpath(str(tmpdir))
+    path = getinitialpath(None, str(tmpdir))
     # we get at least 'dirs', and maybe more (e.g. plat-linux2)
     assert path[:len(dirs)] == map(str, dirs)
 
@@ -21,7 +21,7 @@
     lib_pypy, lib_python_modified, lib_python = build_hierarchy(tmpdir)
     lib_tk_modified = lib_python_modified.join('lib-tk')
     lib_tk = lib_python.join('lib-tk')
-    path = getinitialpath(str(tmpdir))
+    path = getinitialpath(None, str(tmpdir))
     i = path.index(str(lib_tk_modified))
     j = path.index(str(lib_tk))
     assert i < j
diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py
--- a/pypy/rlib/rdynload.py
+++ b/pypy/rlib/rdynload.py
@@ -115,7 +115,8 @@
 if _WIN32:
     DLLHANDLE = rwin32.HMODULE
 
-    def dlopen(name):
+    def dlopen(name, mode=-1):
+        # mode is unused on windows, but a consistant signature
         res = rwin32.LoadLibrary(name)
         if not res:
             err = rwin32.GetLastError()
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -2,8 +2,7 @@
 
 Environment variables can be used to fine-tune the following parameters:
 
- PYPY_GC_NURSERY        The nursery size.  Defaults to half the size of
-                        the L2 cache.  Try values like '1.2MB'.  Small values
+ PYPY_GC_NURSERY        The nursery size.  Defaults to '4MB'.  Small values
                         (like 1 or 1KB) are useful for debugging.
 
  PYPY_GC_MAJOR_COLLECT  Major collection memory factor.  Default is '1.82',
@@ -61,7 +60,7 @@
 #
 #  * young objects: allocated in the nursery if they are not too large, or
 #    raw-malloced otherwise.  The nursery is a fixed-size memory buffer of
-#    half the size of the L2 cache.  When full, we do a minor collection;
+#    4MB by default.  When full, we do a minor collection;
 #    the surviving objects from the nursery are moved outside, and the
 #    non-surviving raw-malloced objects are freed.  All surviving objects
 #    become old.
@@ -329,7 +328,8 @@
             # size (needed to handle mallocs just below 'large_objects') but
             # hacking at the current nursery position in collect_and_reserve().
             if newsize <= 0:
-                newsize = env.estimate_best_nursery_size()
+                newsize = 4*1024*1024   # fixed to 4MB by default
+                #        (it was env.estimate_best_nursery_size())
                 if newsize <= 0:
                     newsize = defaultsize
             if newsize < minsize:
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -672,7 +672,7 @@
     def pypy_initial_path(s):
         from pypy.module.sys.state import getinitialpath
         try:
-            return getinitialpath(s)
+            return getinitialpath(None, s)
         except OSError:
             return None
 
diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py
--- a/pypy/translator/goal/test2/test_app_main.py
+++ b/pypy/translator/goal/test2/test_app_main.py
@@ -821,6 +821,8 @@
             newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found
             assert newpath == sys.path
             newpath = app_main.get_library_path(self.fake_exe)
+            if newpath[0].endswith('__extensions__'):
+                newpath = newpath[1:]
             # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
             assert newpath[:len(self.expected_path)] == self.expected_path
         finally:


More information about the pypy-commit mailing list