[pypy-commit] pypy default: Found another way to fix http://bugs.python.org/issue29006 in PyPy:

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89537:235e8a388979
Date: 2017-01-13 12:10 +0100
http://bitbucket.org/pypy/pypy/changeset/235e8a388979/

Log:	Found another way to fix http://bugs.python.org/issue29006 in PyPy:
	remember which exact statements were open during the previous
	commit, and if we get SQLITE_LOCKED we close these ones

diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -351,10 +351,7 @@
         self.assertRaises(ValueError, cur.execute, " \0select 2")
         self.assertRaises(ValueError, cur.execute, "select 2\0")
 
-    @test_support.impl_detail(pypy=False)
     def CheckCommitCursorReset(self):
-        # This test is for logic added in 2.7.13 which PyPy doesn't
-        # implement.  See http://bugs.python.org/issue29006
         """
         Connection.commit() did reset cursors, which made sqlite3
         to return rows multiple times when fetched from cursors
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -220,6 +220,7 @@
         self.__statements_counter = 0
         self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
+        self.__statements_already_committed = weakref.WeakSet()
 
         self.__func_cache = {}
         self.__aggregates = {}
@@ -363,6 +364,12 @@
                 if cursor is not None:
                     cursor._reset = True
 
+    def _reset_already_committed_statements(self):
+        lst = list(self.__statements_already_committed)
+        self.__statements_already_committed.clear()
+        for statement in lst:
+            statement._reset()
+
     @_check_thread_wrap
     @_check_closed_wrap
     def __call__(self, sql):
@@ -418,15 +425,21 @@
         if not self._in_transaction:
             return
 
-        # The following line is a KNOWN DIFFERENCE with CPython 2.7.13.
-        # More precisely, the corresponding line was removed in the
-        # version 2.7.13 of CPython, but this is causing troubles for
-        # PyPy (and potentially for CPython too):
-        #
-        #     http://bugs.python.org/issue29006
-        #
-        # So for now, we keep this line.
-        self.__do_all_statements(Statement._reset, False)
+        # PyPy fix for non-refcounting semantics: since 2.7.13 (and in
+        # <= 2.6.x), the statements are not automatically reset upon
+        # commit.  However, if this is followed by some specific SQL
+        # operations like "drop table", these open statements come in
+        # the way and cause the "drop table" to fail.  On CPython the
+        # problem is much less important because typically all the old
+        # statements are freed already by reference counting.  So here,
+        # we add all the still-alive statements to a WeakSet which is
+        # usually ignored, except if we get SQLITE_LOCKED
+        # afterwards---at which point we reset all statements in this
+        # WeakSet.
+        for weakref in self.__statements:
+            statement = weakref()
+            if statement is not None:
+                self.__statements_already_committed.add(statement)
 
         statement_star = _ffi.new('sqlite3_stmt **')
         ret = _lib.sqlite3_prepare_v2(self._db, b"COMMIT", -1,
@@ -827,8 +840,18 @@
                 self.__statement._set_params(params)
 
                 # Actually execute the SQL statement
+
                 ret = _lib.sqlite3_step(self.__statement._statement)
 
+                # PyPy: if we get SQLITE_LOCKED, it's probably because
+                # one of the cursors created previously is still alive
+                # and not reset and the operation we're trying to do
+                # makes Sqlite unhappy about that.  In that case, we
+                # automatically reset all old cursors and try again.
+                if ret == _lib.SQLITE_LOCKED:
+                    self.__connection._reset_already_committed_statements()
+                    ret = _lib.sqlite3_step(self.__statement._statement)
+
                 if ret == _lib.SQLITE_ROW:
                     if multiple:
                         raise ProgrammingError("executemany() can only execute DML statements.")
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -483,11 +483,6 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* The ``sqlite`` module was updated on 2.7.13 to no longer reset all
-  cursors when there is a commit.  This causes troubles for PyPy (and
-  potentially for CPython too), and so for now we didn't port this change:
-  see http://bugs.python.org/issue29006.
-
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/