[pypy-commit] pypy improve-heap-caching-tracing: merge default

cfbolz noreply at buildbot.pypy.org
Sun Sep 4 12:31:54 CEST 2011


Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: improve-heap-caching-tracing
Changeset: r47067:304d094f3f3c
Date: 2011-09-04 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/304d094f3f3c/

Log:	merge default

diff --git a/lib-python/modified-2.7/sqlite3/test/regression.py b/lib-python/modified-2.7/sqlite3/test/regression.py
--- a/lib-python/modified-2.7/sqlite3/test/regression.py
+++ b/lib-python/modified-2.7/sqlite3/test/regression.py
@@ -274,6 +274,18 @@
         cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
         self.assertEqual(cur.description, None)
 
+    def CheckStatementCache(self):
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        values = [(i,) for i in xrange(5)]
+        cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+        self.con.commit()
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+
 def suite():
     regression_suite = unittest.makeSuite(RegressionTests, "Check")
     return unittest.TestSuite((regression_suite,))
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -54,7 +54,8 @@
     def get_ffi_argtype(self):
         if self._ffiargtype:
             return self._ffiargtype
-        return _shape_to_ffi_type(self._ffiargshape)
+        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape)
+        return self._ffiargtype
 
     def _CData_output(self, resbuffer, base=None, index=-1):
         #assert isinstance(resbuffer, _rawffi.ArrayInstance)
@@ -225,6 +226,7 @@
     'Z' : _ffi.types.void_p,
     'X' : _ffi.types.void_p,
     'v' : _ffi.types.sshort,
+    '?' : _ffi.types.ubyte,
     }
 
 
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -293,7 +293,7 @@
         #
         if stat.in_use:
             stat = Statement(self.connection, sql)
-        stat.set_cursor_and_factory(cursor, row_factory)
+        stat.set_row_factory(row_factory)
         return stat
 
 
@@ -705,6 +705,8 @@
         from sqlite3.dump import _iterdump
         return _iterdump(self)
 
+DML, DQL, DDL = range(3)
+
 class Cursor(object):
     def __init__(self, con):
         if not isinstance(con, Connection):
@@ -735,9 +737,9 @@
         self.statement = self.connection.statement_cache.get(sql, self, self.row_factory)
 
         if self.connection._isolation_level is not None:
-            if self.statement.kind == "DDL":
+            if self.statement.kind == DDL:
                 self.connection.commit()
-            elif self.statement.kind == "DML":
+            elif self.statement.kind == DML:
                 self.connection._begin()
 
         self.statement.set_params(params)
@@ -748,18 +750,18 @@
             self.statement.reset()
             raise self.connection._get_exception(ret)
 
-        if self.statement.kind == "DQL"and ret == SQLITE_ROW:
+        if self.statement.kind == DQL and ret == SQLITE_ROW:
             self.statement._build_row_cast_map()
-            self.statement._readahead()
+            self.statement._readahead(self)
         else:
             self.statement.item = None
             self.statement.exhausted = True
 
-        if self.statement.kind in ("DML", "DDL"):
+        if self.statement.kind == DML or self.statement.kind == DDL:
             self.statement.reset()
 
         self.rowcount = -1
-        if self.statement.kind == "DML":
+        if self.statement.kind == DML:
             self.rowcount = sqlite.sqlite3_changes(self.connection.db)
 
         return self
@@ -771,8 +773,8 @@
             sql = sql.encode("utf-8")
         self._check_closed()
         self.statement = self.connection.statement_cache.get(sql, self, self.row_factory)
-        
-        if self.statement.kind == "DML":
+
+        if self.statement.kind == DML:
             self.connection._begin()
         else:
             raise ProgrammingError, "executemany is only for DML statements"
@@ -824,7 +826,7 @@
         return self
 
     def __iter__(self):
-        return self.statement
+        return iter(self.fetchone, None)
 
     def _check_reset(self):
         if self.reset:
@@ -841,7 +843,7 @@
             return None
 
         try:
-            return self.statement.next()
+            return self.statement.next(self)
         except StopIteration:
             return None
 
@@ -855,7 +857,7 @@
         if size is None:
             size = self.arraysize
         lst = []
-        for row in self.statement:
+        for row in self:
             lst.append(row)
             if len(lst) == size:
                 break
@@ -866,7 +868,7 @@
         self._check_reset()
         if self.statement is None:
             return []
-        return list(self.statement)
+        return list(self)
 
     def _getdescription(self):
         if self._description is None:
@@ -904,16 +906,15 @@
         self.sql = sql # DEBUG ONLY
         first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper()
         if first_word in ("INSERT", "UPDATE", "DELETE", "REPLACE"):
-            self.kind = "DML"
+            self.kind = DML
         elif first_word in ("SELECT", "PRAGMA"):
-            self.kind = "DQL"
+            self.kind = DQL
         else:
-            self.kind = "DDL"
+            self.kind = DDL
         self.exhausted = False
         self.in_use = False
         #
-        # set by set_cursor_and_factory
-        self.cur = None
+        # set by set_row_factory
         self.row_factory = None
 
         self.statement = c_void_p()
@@ -923,7 +924,7 @@
         if ret == SQLITE_OK and self.statement.value is None:
             # an empty statement, we work around that, as it's the least trouble
             ret = sqlite.sqlite3_prepare_v2(self.con.db, "select 42", -1, byref(self.statement), byref(next_char))
-            self.kind = "DQL"
+            self.kind = DQL
 
         if ret != SQLITE_OK:
             raise self.con._get_exception(ret)
@@ -935,8 +936,7 @@
 
         self._build_row_cast_map()
 
-    def set_cursor_and_factory(self, cur, row_factory):
-        self.cur = weakref.ref(cur)
+    def set_row_factory(self, row_factory):
         self.row_factory = row_factory
 
     def _build_row_cast_map(self):
@@ -1039,10 +1039,7 @@
                     raise ProgrammingError("missing parameter '%s'" %param)
                 self.set_param(idx, param)
 
-    def __iter__(self):
-        return self
-
-    def next(self):
+    def next(self, cursor):
         self.con._check_closed()
         self.con._check_thread()
         if self.exhausted:
@@ -1058,10 +1055,10 @@
             sqlite.sqlite3_reset(self.statement)
             raise exc
 
-        self._readahead()
+        self._readahead(cursor)
         return item
 
-    def _readahead(self):
+    def _readahead(self, cursor):
         self.column_count = sqlite.sqlite3_column_count(self.statement)
         row = []
         for i in xrange(self.column_count):
@@ -1096,13 +1093,14 @@
 
         row = tuple(row)
         if self.row_factory is not None:
-            row = self.row_factory(self.cur(), row)
+            row = self.row_factory(cursor, row)
         self.item = row
 
     def reset(self):
         self.row_cast_map = None
         ret = sqlite.sqlite3_reset(self.statement)
         self.in_use = False
+        self.exhausted = False
         return ret
 
     def finalize(self):
@@ -1118,7 +1116,7 @@
         self.statement = None
 
     def _get_description(self):
-        if self.kind == "DML":
+        if self.kind == DML:
             return None
         desc = []
         for i in xrange(sqlite.sqlite3_column_count(self.statement)):
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -59,7 +59,12 @@
         #
         while not target:
             if not target.__started:
-                _continulet.__init__(target, _greenlet_start, *args)
+                if unbound_method != _continulet.throw:
+                    greenlet_func = _greenlet_start
+                else:
+                    greenlet_func = _greenlet_throw
+                _continulet.__init__(target, greenlet_func, *args)
+                unbound_method = _continulet.switch
                 args = ()
                 target.__started = True
                 break
@@ -136,3 +141,11 @@
         if greenlet.parent is not _tls.main:
             _continuation.permute(greenlet, greenlet.parent)
     return (res,)
+
+def _greenlet_throw(greenlet, exc, value, tb):
+    _tls.current = greenlet
+    try:
+        raise exc, value, tb
+    finally:
+        if greenlet.parent is not _tls.main:
+            _continuation.permute(greenlet, greenlet.parent)
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -109,25 +109,19 @@
             self.getc() == self.getc(+2)):
             self.pos += 3
             while 1: # Deal with a triple quoted docstring
-                if self.getc() == '\\':
-                    self.pos += 2
+                c = self.getc()
+                if c == '\\':
+                    self.pos += 1
+                    self._skip_next_char_from_docstring()
+                elif c != endchar:
+                    self._skip_next_char_from_docstring()
                 else:
-                    c = self.getc()
-                    if c != endchar:
-                        self.pos += 1
-                        if c == '\n':
-                            self.atbol()
-                        elif c == '\r':
-                            if self.getc() == '\n':
-                                self.pos += 1
-                                self.atbol()
-                    else:
-                        self.pos += 1
-                        if (self.getc() == endchar and
-                            self.getc(+1) == endchar):
-                            self.pos += 2
-                            self.consume_empty_line()
-                            break
+                    self.pos += 1
+                    if (self.getc() == endchar and
+                        self.getc(+1) == endchar):
+                        self.pos += 2
+                        self.consume_empty_line()
+                        break
 
         else: # Deal with a single quoted docstring
             self.pos += 1
@@ -138,17 +132,21 @@
                     self.consume_empty_line()
                     return
                 elif c == '\\':
-                    # Deal with linefeeds
-                    if self.getc() != '\r':
-                        self.pos += 1
-                    else:
-                        self.pos += 1
-                        if self.getc() == '\n':
-                            self.pos += 1
+                    self._skip_next_char_from_docstring()
                 elif c in '\r\n':
                     # Syntax error
                     return
 
+    def _skip_next_char_from_docstring(self):
+        c = self.getc()
+        self.pos += 1
+        if c == '\n':
+            self.atbol()
+        elif c == '\r':
+            if self.getc() == '\n':
+                self.pos += 1
+            self.atbol()
+
     def consume_continuation(self):
         c = self.getc()
         if c in '\n\r':
diff --git a/pypy/interpreter/pyparser/test/test_futureautomaton.py b/pypy/interpreter/pyparser/test/test_futureautomaton.py
--- a/pypy/interpreter/pyparser/test/test_futureautomaton.py
+++ b/pypy/interpreter/pyparser/test/test_futureautomaton.py
@@ -221,6 +221,14 @@
     assert f.lineno == 3
     assert f.col_offset == 0
 
+def test_lots_of_continuation_lines():
+    s = "\\\n\\\n\\\n\\\n\\\n\\\n\nfrom __future__ import with_statement\n"
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == fut.CO_FUTURE_WITH_STATEMENT
+    assert f.lineno == 8
+    assert f.col_offset == 0
+
 # This looks like a bug in cpython parser
 # and would require extensive modifications
 # to future.py in order to emulate the same behaviour
@@ -239,3 +247,19 @@
         raise AssertionError('IndentationError not raised')
     assert f.lineno == 2
     assert f.col_offset == 0
+
+def test_continuation_lines_in_docstring_single_quoted():
+    s = '"\\\n\\\n\\\n\\\n\\\n\\\n"\nfrom  __future__ import division\n'
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == fut.CO_FUTURE_DIVISION
+    assert f.lineno == 8
+    assert f.col_offset == 0
+
+def test_continuation_lines_in_docstring_triple_quoted():
+    s = '"""\\\n\\\n\\\n\\\n\\\n\\\n"""\nfrom  __future__ import division\n'
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == fut.CO_FUTURE_DIVISION
+    assert f.lineno == 8
+    assert f.col_offset == 0
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -130,8 +130,15 @@
     results = _find_jit_marker(graphs, 'jit_merge_point')
     if not results:
         raise Exception("no jit_merge_point found!")
+    seen = set([graph for graph, block, pos in results])
+    assert len(seen) == len(results), (
+        "found several jit_merge_points in the same graph")
     return results
 
+def locate_jit_merge_point(graph):
+    [(graph, block, pos)] = find_jit_merge_points([graph])
+    return block, pos, block.operations[pos]
+
 def find_set_param(graphs):
     return _find_jit_marker(graphs, 'set_param')
 
@@ -235,7 +242,7 @@
     def split_graph_and_record_jitdriver(self, graph, block, pos):
         op = block.operations[pos]
         jd = JitDriverStaticData()
-        jd._jit_merge_point_pos = (graph, op)
+        jd._jit_merge_point_in = graph
         args = op.args[2:]
         s_binding = self.translator.annotator.binding
         jd._portal_args_s = [s_binding(v) for v in args]
@@ -504,7 +511,8 @@
             self.make_args_specification(jd)
 
     def make_args_specification(self, jd):
-        graph, op = jd._jit_merge_point_pos
+        graph = jd._jit_merge_point_in
+        _, _, op = locate_jit_merge_point(graph)
         greens_v, reds_v = support.decode_hp_hint_args(op)
         ALLARGS = [v.concretetype for v in (greens_v + reds_v)]
         jd._green_args_spec = [v.concretetype for v in greens_v]
@@ -552,7 +560,7 @@
             assert jitdriver in sublists, \
                    "can_enter_jit with no matching jit_merge_point"
             jd, sublist = sublists[jitdriver]
-            origportalgraph = jd._jit_merge_point_pos[0]
+            origportalgraph = jd._jit_merge_point_in
             if graph is not origportalgraph:
                 sublist.append((graph, block, index))
                 jd.no_loop_header = False
@@ -582,7 +590,7 @@
             can_enter_jits = [(jd.portal_graph, jd.portal_graph.startblock, 0)]
 
         for graph, block, index in can_enter_jits:
-            if graph is jd._jit_merge_point_pos[0]:
+            if graph is jd._jit_merge_point_in:
                 continue
 
             op = block.operations[index]
@@ -640,7 +648,7 @@
         #           while 1:
         #               more stuff
         #
-        origportalgraph = jd._jit_merge_point_pos[0]
+        origportalgraph = jd._jit_merge_point_in
         portalgraph = jd.portal_graph
         PORTALFUNC = jd._PORTAL_FUNCTYPE
 
@@ -794,14 +802,7 @@
         # ____________________________________________________________
         # Now mutate origportalgraph to end with a call to portal_runner_ptr
         #
-        _, op = jd._jit_merge_point_pos
-        for origblock in origportalgraph.iterblocks():
-            if op in origblock.operations:
-                break
-        else:
-            assert False, "lost the operation %r in the graph %r" % (
-                op, origportalgraph)
-        origindex = origblock.operations.index(op)
+        origblock, origindex, op = locate_jit_merge_point(origportalgraph)
         assert op.opname == 'jit_marker'
         assert op.args[0].value == 'jit_merge_point'
         greens_v, reds_v = support.decode_hp_hint_args(op)
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -43,11 +43,11 @@
     def switch(self, w_to):
         to = self.space.interp_w(W_Continulet, w_to, can_be_None=True)
         if to is not None:
-            if self is to:    # double-switch to myself: no-op
-                return get_result()
             if to.sthread is None:
                 start_state.clear()
                 raise geterror(self.space, "continulet not initialized yet")
+            if self is to:    # double-switch to myself: no-op
+                return get_result()
         if self.sthread is None:
             start_state.clear()
             raise geterror(self.space, "continulet not initialized yet")
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -8,24 +8,12 @@
 
 
 class WeakrefLifeline(W_Root):
+    cached_weakref_index = -1
+    cached_proxy_index = -1
+
     def __init__(self, space):
         self.space = space
         self.refs_weak = []
-        self.cached_weakref_index = -1
-        self.cached_proxy_index = -1
-
-    def __del__(self):
-        """This runs when the interp-level object goes away, and allows
-        its lifeline to go away.  The purpose of this is to activate the
-        callbacks even if there is no __del__ method on the interp-level
-        W_Root subclass implementing the object.
-        """
-        for i in range(len(self.refs_weak) - 1, -1, -1):
-            w_ref = self.refs_weak[i]()
-            if w_ref is not None and w_ref.w_callable is not None:
-                w_ref.enqueue_for_destruction(self.space,
-                                              W_WeakrefBase.activate_callback,
-                                              'weakref callback of ')
 
     def clear_all_weakrefs(self):
         """Clear all weakrefs.  This is called when an app-level object has
@@ -39,12 +27,11 @@
         # weakref callbacks are not invoked eagerly here.  They are
         # invoked by self.__del__() anyway.
 
-    @jit.dont_look_inside
-    def get_or_make_weakref(self, space, w_subtype, w_obj, w_callable):
+    def get_or_make_weakref(self, w_subtype, w_obj):
+        space = self.space
         w_weakreftype = space.gettypeobject(W_Weakref.typedef)
         is_weakreftype = space.is_w(w_weakreftype, w_subtype)
-        can_reuse = space.is_w(w_callable, space.w_None)
-        if is_weakreftype and can_reuse and self.cached_weakref_index >= 0:
+        if is_weakreftype and self.cached_weakref_index >= 0:
             w_cached = self.refs_weak[self.cached_weakref_index]()
             if w_cached is not None:
                 return w_cached
@@ -52,16 +39,15 @@
                 self.cached_weakref_index = -1
         w_ref = space.allocate_instance(W_Weakref, w_subtype)
         index = len(self.refs_weak)
-        W_Weakref.__init__(w_ref, space, w_obj, w_callable)
+        W_Weakref.__init__(w_ref, space, w_obj, None)
         self.refs_weak.append(weakref.ref(w_ref))
-        if is_weakreftype and can_reuse:
+        if is_weakreftype:
             self.cached_weakref_index = index
         return w_ref
 
-    @jit.dont_look_inside
-    def get_or_make_proxy(self, space, w_obj, w_callable):
-        can_reuse = space.is_w(w_callable, space.w_None)
-        if can_reuse and self.cached_proxy_index >= 0:
+    def get_or_make_proxy(self, w_obj):
+        space = self.space
+        if self.cached_proxy_index >= 0:
             w_cached = self.refs_weak[self.cached_proxy_index]()
             if w_cached is not None:
                 return w_cached
@@ -69,12 +55,11 @@
                 self.cached_proxy_index = -1
         index = len(self.refs_weak)
         if space.is_true(space.callable(w_obj)):
-            w_proxy = W_CallableProxy(space, w_obj, w_callable)
+            w_proxy = W_CallableProxy(space, w_obj, None)
         else:
-            w_proxy = W_Proxy(space, w_obj, w_callable)
+            w_proxy = W_Proxy(space, w_obj, None)
         self.refs_weak.append(weakref.ref(w_proxy))
-        if can_reuse:
-            self.cached_proxy_index = index
+        self.cached_proxy_index = index
         return w_proxy
 
     def get_any_weakref(self, space):
@@ -90,6 +75,45 @@
                 return w_ref
         return space.w_None
 
+
+class WeakrefLifelineWithCallbacks(WeakrefLifeline):
+
+    def __init__(self, space, oldlifeline=None):
+        self.space = space
+        if oldlifeline is None:
+            self.refs_weak = []
+        else:
+            self.refs_weak = oldlifeline.refs_weak
+
+    def __del__(self):
+        """This runs when the interp-level object goes away, and allows
+        its lifeline to go away.  The purpose of this is to activate the
+        callbacks even if there is no __del__ method on the interp-level
+        W_Root subclass implementing the object.
+        """
+        for i in range(len(self.refs_weak) - 1, -1, -1):
+            w_ref = self.refs_weak[i]()
+            if w_ref is not None and w_ref.w_callable is not None:
+                w_ref.enqueue_for_destruction(self.space,
+                                              W_WeakrefBase.activate_callback,
+                                              'weakref callback of ')
+
+    def make_weakref_with_callback(self, w_subtype, w_obj, w_callable):
+        space = self.space
+        w_ref = space.allocate_instance(W_Weakref, w_subtype)
+        W_Weakref.__init__(w_ref, space, w_obj, w_callable)
+        self.refs_weak.append(weakref.ref(w_ref))
+        return w_ref
+
+    def make_proxy_with_callback(self, w_obj, w_callable):
+        space = self.space
+        if space.is_true(space.callable(w_obj)):
+            w_proxy = W_CallableProxy(space, w_obj, w_callable)
+        else:
+            w_proxy = W_Proxy(space, w_obj, w_callable)
+        self.refs_weak.append(weakref.ref(w_proxy))
+        return w_proxy
+
 # ____________________________________________________________
 
 class Dummy:
@@ -103,8 +127,7 @@
 
 class W_WeakrefBase(Wrappable):
     def __init__(w_self, space, w_obj, w_callable):
-        if space.is_w(w_callable, space.w_None):
-            w_callable = None
+        assert w_callable is not space.w_None    # should be really None
         w_self.space = space
         assert w_obj is not None
         w_self.w_obj_weak = weakref.ref(w_obj)
@@ -177,16 +200,39 @@
     def descr__ne__(self, space, w_ref2):
         return space.not_(space.eq(self, w_ref2))
 
+def getlifeline(space, w_obj):
+    lifeline = w_obj.getweakref()
+    if lifeline is None:
+        lifeline = WeakrefLifeline(space)
+        w_obj.setweakref(space, lifeline)
+    return lifeline
+
+def getlifelinewithcallbacks(space, w_obj):
+    lifeline = w_obj.getweakref()
+    if not isinstance(lifeline, WeakrefLifelineWithCallbacks):  # or None
+        oldlifeline = lifeline
+        lifeline = WeakrefLifelineWithCallbacks(space, oldlifeline)
+        w_obj.setweakref(space, lifeline)
+    return lifeline
+
+ at jit.dont_look_inside
+def get_or_make_weakref(space, w_subtype, w_obj):
+    return getlifeline(space, w_obj).get_or_make_weakref(w_subtype, w_obj)
+
+ at jit.dont_look_inside
+def make_weakref_with_callback(space, w_subtype, w_obj, w_callable):
+    lifeline = getlifelinewithcallbacks(space, w_obj)
+    return lifeline.make_weakref_with_callback(w_subtype, w_obj, w_callable)
+
 def descr__new__weakref(space, w_subtype, w_obj, w_callable=None,
                         __args__=None):
     if __args__.arguments_w:
         raise OperationError(space.w_TypeError, space.wrap(
             "__new__ expected at most 2 arguments"))
-    lifeline = w_obj.getweakref()
-    if lifeline is None:
-        lifeline = WeakrefLifeline(space)
-        w_obj.setweakref(space, lifeline)
-    return lifeline.get_or_make_weakref(space, w_subtype, w_obj, w_callable)
+    if space.is_w(w_callable, space.w_None):
+        return get_or_make_weakref(space, w_subtype, w_obj)
+    else:
+        return make_weakref_with_callback(space, w_subtype, w_obj, w_callable)
 
 W_Weakref.typedef = TypeDef("weakref",
     __doc__ = """A weak reference to an object 'obj'.  A 'callback' can be given,
@@ -239,15 +285,23 @@
         w_obj = force(space, self)
         return space.call_args(w_obj, __args__)
 
+ at jit.dont_look_inside
+def get_or_make_proxy(space, w_obj):
+    return getlifeline(space, w_obj).get_or_make_proxy(w_obj)
+
+ at jit.dont_look_inside
+def make_proxy_with_callback(space, w_obj, w_callable):
+    lifeline = getlifelinewithcallbacks(space, w_obj)
+    return lifeline.make_proxy_with_callback(w_obj, w_callable)
+
 def proxy(space, w_obj, w_callable=None):
     """Create a proxy object that weakly references 'obj'.
 'callback', if given, is called with the proxy as an argument when 'obj'
 is about to be finalized."""
-    lifeline = w_obj.getweakref()
-    if lifeline is None:
-        lifeline = WeakrefLifeline(space)
-        w_obj.setweakref(space, lifeline)
-    return lifeline.get_or_make_proxy(space, w_obj, w_callable)
+    if space.is_w(w_callable, space.w_None):
+        return get_or_make_proxy(space, w_obj)
+    else:
+        return make_proxy_with_callback(space, w_obj, w_callable)
 
 def descr__new__proxy(space, w_subtype, w_obj, w_callable=None):
     raise OperationError(
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -369,6 +369,26 @@
             return A
         raises(TypeError, tryit)
 
+    def test_proxy_to_dead_object(self):
+        import _weakref, gc
+        class A(object):
+            pass
+        p = _weakref.proxy(A())
+        gc.collect()
+        raises(ReferenceError, "p + 1")
+
+    def test_proxy_with_callback(self):
+        import _weakref, gc
+        class A(object):
+            pass
+        a2 = A()
+        def callback(proxy):
+            a2.seen = proxy
+        p = _weakref.proxy(A(), callback)
+        gc.collect()
+        raises(ReferenceError, "p + 1")
+        assert a2.seen is p
+
     def test_repr(self):
         import _weakref, gc
         for kind in ('ref', 'proxy'):
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -8,7 +8,8 @@
                 modname == '__builtin__.interp_classobj' or
                 modname == '__builtin__.functional' or
                 modname == '__builtin__.descriptor' or
-                modname == 'thread.os_local'):
+                modname == 'thread.os_local' or
+                modname == 'thread.os_thread'):
             return True
         if '.' in modname:
             modname, _ = modname.split('.', 1)
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -34,7 +34,9 @@
 
 def test_thread_local():
     from pypy.module.thread.os_local import Local
+    from pypy.module.thread.os_thread import get_ident
     assert pypypolicy.look_inside_function(Local.getdict.im_func)
+    assert pypypolicy.look_inside_function(get_ident)
 
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
diff --git a/pypy/module/pypyjit/test_pypy_c/test_globals.py b/pypy/module/pypyjit/test_pypy_c/test_globals.py
--- a/pypy/module/pypyjit/test_pypy_c/test_globals.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_globals.py
@@ -23,6 +23,4 @@
             guard_not_invalidated(descr=...)
             p19 = getfield_gc(ConstPtr(p17), descr=<GcPtrFieldDescr .*W_DictMultiObject.inst_strategy .*>)
             guard_value(p19, ConstPtr(ptr20), descr=...)
-            p22 = getfield_gc(ConstPtr(ptr21), descr=<GcPtrFieldDescr .*ModuleCell.inst_w_value .*>)
-            guard_nonnull(p22, descr=...)
-        """)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -181,8 +181,7 @@
         assert loop.match_by_id("contains", """
             guard_not_invalidated(descr=...)
             i11 = force_token()
-            i12 = int_add_ovf(i5, i7)
-            guard_no_overflow(descr=...)
+            i12 = int_add(i5, 1)
         """)
 
     def test_id_compare_optimization(self):
diff --git a/pypy/module/sys/test/test_encoding.py b/pypy/module/sys/test/test_encoding.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/test/test_encoding.py
@@ -0,0 +1,30 @@
+import os, py
+from pypy.rlib import rlocale
+from pypy.module.sys.interp_encoding import _getfilesystemencoding
+from pypy.module.sys.interp_encoding import base_encoding
+
+
+def test__getfilesystemencoding(space):
+    if not (rlocale.HAVE_LANGINFO and rlocale.CODESET):
+        py.test.skip("requires HAVE_LANGINFO and CODESET")
+
+    def clear():
+        for key in os.environ.keys():
+            if key == 'LANG' or key.startswith('LC_'):
+                del os.environ[key]
+
+    def get(**env):
+        original_env = os.environ.copy()
+        try:
+            clear()
+            os.environ.update(env)
+            return _getfilesystemencoding(space)
+        finally:
+            clear()
+            os.environ.update(original_env)
+
+    assert get() in (base_encoding, 'ANSI_X3.4-1968')
+    assert get(LANG='foobar') in (base_encoding, 'ANSI_X3.4-1968')
+    assert get(LANG='en_US.UTF-8') == 'UTF-8'
+    assert get(LC_ALL='en_US.UTF-8') == 'UTF-8'
+    assert get(LC_CTYPE='en_US.UTF-8') == 'UTF-8'
diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/test_lib_pypy/test_greenlet.py
--- a/pypy/module/test_lib_pypy/test_greenlet.py
+++ b/pypy/module/test_lib_pypy/test_greenlet.py
@@ -231,3 +231,13 @@
         assert res == "next step"
         res = g2.switch("goes to f1 instead")
         assert res == "all ok"
+
+    def test_throw_in_not_started_yet(self):
+        from greenlet import greenlet
+        #
+        def f1():
+            never_reached
+        #
+        g1 = greenlet(f1)
+        raises(ValueError, g1.throw, ValueError)
+        assert g1.dead
diff --git a/pypy/module/test_lib_pypy/test_stackless.py b/pypy/module/test_lib_pypy/test_stackless_pickle.py
rename from pypy/module/test_lib_pypy/test_stackless.py
rename to pypy/module/test_lib_pypy/test_stackless_pickle.py
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -65,6 +65,10 @@
         if isinstance(cell, ModuleCell):
             cell.w_value = w_value
             return
+        # If the new value and the current value are the same, don't create a
+        # level of indirection, or mutate are version.
+        if self.space.is_w(w_value, cell):
+            return
         if cell is not None:
             w_value = ModuleCell(w_value)
         self.mutated()
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -355,9 +355,13 @@
     y = w_float2.floatval
     if y == 0.0:
         raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float modulo"))
-    mod = math.fmod(x, y)
-    if (mod and ((y < 0.0) != (mod < 0.0))):
-        mod += y
+    try:
+        mod = math.fmod(x, y)
+    except ValueError:
+        mod = rfloat.NAN
+    else:
+        if (mod and ((y < 0.0) != (mod < 0.0))):
+            mod += y
 
     return W_FloatObject(mod)
 
@@ -366,7 +370,10 @@
     y = w_float2.floatval
     if y == 0.0:
         raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float modulo"))
-    mod = math.fmod(x, y)
+    try:
+        mod = math.fmod(x, y)
+    except ValueError:
+        return [W_FloatObject(rfloat.NAN), W_FloatObject(rfloat.NAN)]
     # fmod is typically exact, so vx-mod is *mathematically* an
     # exact multiple of wx.  But this is fp arithmetic, and fp
     # vx - mod is an approximation; the result is that div may
diff --git a/pypy/objspace/std/test/test_celldict.py b/pypy/objspace/std/test/test_celldict.py
--- a/pypy/objspace/std/test/test_celldict.py
+++ b/pypy/objspace/std/test/test_celldict.py
@@ -39,6 +39,20 @@
         assert d.getitem("a") is None
         assert d.strategy.getdictvalue_no_unwrapping(d, "a") is None
 
+    def test_same_key_set_twice(self):
+        strategy = ModuleDictStrategy(space)
+        storage = strategy.get_empty_storage()
+        d = W_DictMultiObject(space, strategy, storage)
+
+        v1 = strategy.version
+        x = object()
+        d.setitem("a", x)
+        v2 = strategy.version
+        assert v1 is not v2
+        d.setitem("a", x)
+        v3 = strategy.version
+        assert v2 is v3
+
 class AppTestModuleDict(object):
     def setup_class(cls):
         cls.space = gettestobjspace(**{"objspace.std.withcelldict": True})
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -767,3 +767,19 @@
 
     def test_invalid(self):
         raises(ValueError, float.fromhex, "0P")
+
+    def test_division_edgecases(self):
+        import math
+
+        # inf
+        inf = float("inf")
+        assert math.isnan(inf % 3)
+        assert math.isnan(inf // 3)
+        x, y = divmod(inf, 3)
+        assert math.isnan(x)
+        assert math.isnan(y)
+
+        # divide by 0
+        raises(ZeroDivisionError, lambda: inf % 0)
+        raises(ZeroDivisionError, lambda: inf // 0)
+        raises(ZeroDivisionError, divmod, inf, 0)
\ No newline at end of file
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -134,20 +134,24 @@
 
     def test_custom_metaclass(self):
         import __pypy__
-        class MetaA(type):
-            def __getattribute__(self, x):
-                return 1
-        def f(self):
-            return 42
-        A = type.__new__(MetaA, "A", (), {"f": f})
-        l = [type.__getattribute__(A, "__new__")(A)] * 10
-        __pypy__.reset_method_cache_counter()
-        for i, a in enumerate(l):
-            assert a.f() == 42
-        cache_counter = __pypy__.method_cache_counter("f")
-        assert cache_counter[0] >= 5
-        assert cache_counter[1] >= 1 # should be (27, 3)
-        assert sum(cache_counter) == 10
+        for j in range(20):
+            class MetaA(type):
+                def __getattribute__(self, x):
+                    return 1
+            def f(self):
+                return 42
+            A = type.__new__(MetaA, "A", (), {"f": f})
+            l = [type.__getattribute__(A, "__new__")(A)] * 10
+            __pypy__.reset_method_cache_counter()
+            for i, a in enumerate(l):
+                assert a.f() == 42
+            cache_counter = __pypy__.method_cache_counter("f")
+            assert sum(cache_counter) == 10
+            if cache_counter == (9, 1):
+                break
+            #else the moon is misaligned, try again
+        else:
+            raise AssertionError("cache_counter = %r" % (cache_counter,))
 
     def test_mutate_class(self):
         import __pypy__
diff --git a/pypy/rlib/libffi.py b/pypy/rlib/libffi.py
--- a/pypy/rlib/libffi.py
+++ b/pypy/rlib/libffi.py
@@ -206,6 +206,7 @@
     _immutable_fields_ = ['funcsym']
     argtypes = []
     restype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    flags = 0
     funcsym = lltype.nullptr(rffi.VOIDP.TO)
 
     def __init__(self, name, argtypes, restype, funcsym, flags=FUNCFLAG_CDECL,
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1461,6 +1461,7 @@
         # We will fix such references to point to the copy of the young
         # objects when we walk 'old_objects_pointing_to_young'.
         self.old_objects_pointing_to_young.append(newobj)
+    _trace_drag_out._always_inline_ = True
 
     def _visit_young_rawmalloced_object(self, obj):
         # 'obj' points to a young, raw-malloced object.
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -459,7 +459,7 @@
             if t._hints.get('immutable'):
                 return
             if 'immutable_fields' in t._hints:
-                skip = t._hints['immutable_fields'].fields
+                skip = t._hints['immutable_fields'].all_immutable_fields()
         for n, t2 in t._flds.iteritems():
             if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc':
                 if n not in skip:
diff --git a/pypy/rpython/memory/test/test_gctypelayout.py b/pypy/rpython/memory/test/test_gctypelayout.py
--- a/pypy/rpython/memory/test/test_gctypelayout.py
+++ b/pypy/rpython/memory/test/test_gctypelayout.py
@@ -4,7 +4,7 @@
 from pypy.rpython.memory.gctypelayout import gc_pointers_inside
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.test.test_llinterp import get_interpreter
-from pypy.rpython.rclass import IR_IMMUTABLE
+from pypy.rpython.rclass import IR_IMMUTABLE, IR_QUASIIMMUTABLE
 from pypy.objspace.flow.model import Constant
 
 class FakeGC:
@@ -102,7 +102,7 @@
     accessor = rclass.FieldListAccessor()
     S3 = lltype.GcStruct('S', ('x', PT), ('y', PT),
                          hints={'immutable_fields': accessor})
-    accessor.initialize(S3, {'x': IR_IMMUTABLE})
+    accessor.initialize(S3, {'x': IR_IMMUTABLE, 'y': IR_QUASIIMMUTABLE})
     #
     s1 = lltype.malloc(S1)
     adr = llmemory.cast_ptr_to_adr(s1)
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -16,6 +16,13 @@
         for x in fields.itervalues():
             assert isinstance(x, ImmutableRanking)
 
+    def all_immutable_fields(self):
+        result = set()
+        for key, value in self.fields.iteritems():
+            if value in (IR_IMMUTABLE, IR_IMMUTABLE_ARRAY):
+                result.add(key)
+        return result
+
     def __repr__(self):
         return '<FieldListAccessor for %s>' % getattr(self, 'TYPE', '?')
 
diff --git a/pypy/tool/py.cleanup b/pypy/tool/py.cleanup
--- a/pypy/tool/py.cleanup
+++ b/pypy/tool/py.cleanup
@@ -1,16 +1,31 @@
 #!/usr/bin/env python
-import py, sys
+import sys, os, stat
 
-def shouldremove(p):
-    return p.ext == '.pyc'
+def clean(path):
+    global count
+    try:
+        content = os.listdir(path)
+    except OSError:
+        print >> sys.stderr, "skipping", path
+        return
+    for fn in content:
+        filename = os.path.join(path, fn)
+        st = os.lstat(filename)
+        if stat.S_ISDIR(st.st_mode):
+            clean(filename)
+            if fn == '__pycache__':
+                try:
+                    os.rmdir(filename)
+                except OSError:
+                    pass
+        elif fn.endswith('.pyc') or fn.endswith('.pyo'):
+            os.unlink(filename)
+            count += 1
 
 count = 0
 
 for arg in sys.argv[1:] or ['.']:
-    path = py.path.local(arg)
-    print "cleaning path", path, "of .pyc files"
-    for x in path.visit(shouldremove, lambda x: x.check(dotfile=0, link=0)):
-        x.remove()
-        count += 1
+    print "cleaning path", arg, "of .pyc/.pyo/__pycache__ files"
+    clean(arg)
 
 print "%d files removed" % (count,)
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -260,6 +260,8 @@
     try:
         import _file
     except ImportError:
+        if sys.version_info < (2, 7):
+            return
         import ctypes # HACK: while running on top of CPython
         set_file_encoding = ctypes.pythonapi.PyFile_SetEncodingAndErrors
         set_file_encoding.argtypes = [ctypes.py_object, ctypes.c_char_p, ctypes.c_char_p]
@@ -479,7 +481,8 @@
             print >> sys.stderr, "'import site' failed"
 
     readenv = not ignore_environment
-    io_encoding = readenv and os.getenv("PYTHONIOENCODING")
+    io_encoding = ((readenv and os.getenv("PYTHONIOENCODING"))
+                   or sys.getfilesystemencoding())
     if io_encoding:
         set_io_encoding(io_encoding)
 
diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py
--- a/pypy/translator/goal/test2/test_app_main.py
+++ b/pypy/translator/goal/test2/test_app_main.py
@@ -739,6 +739,19 @@
         data = self.run(p + os.sep)
         assert data == p + os.sep + '\n'
 
+    def test_getfilesystemencoding(self):
+        if sys.version_info < (2, 7):
+            skip("test requires Python >= 2.7")
+        p = getscript_in_dir("""
+        import sys
+        sys.stdout.write(u'15\u20ac')
+        sys.stdout.flush()
+        """)
+        env = os.environ.copy()
+        env["LC_CTYPE"] = 'en_US.UTF-8'
+        data = self.run(p, env=env)
+        assert data == '15\xe2\x82\xac'
+
     def test_pythonioencoding(self):
         if sys.version_info < (2, 7):
             skip("test requires Python >= 2.7")


More information about the pypy-commit mailing list