[Python-checkins] python/dist/src/Lib/bsddb __init__.py,1.11,1.12

[greg at users.sourceforge.net]

Update of /cvsroot/python/python/dist/src/Lib/bsddb
In directory sc8-pr-cvs1:/tmp/cvs-serv10424/Lib/bsddb

Modified Files:
	__init__.py 
Log Message:
* Use weakref's of DBCursor objects for the iterator cursors to avoid a
  memory leak that would've occurred for all iterators that were
  destroyed before having iterated until they raised StopIteration.

* Simplify some code.

* Add new test cases to check for the memleak and ensure that mixing
  iteration with modification of the values for existing keys works.


Index: __init__.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/bsddb/__init__.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** __init__.py	2 Nov 2003 09:10:16 -0000	1.11
--- __init__.py	3 Nov 2003 01:04:41 -0000	1.12
***************
*** 68,115 ****
      exec """
  import UserDict
  class _iter_mixin(UserDict.DictMixin):
      def __iter__(self):
          try:
!             cur = self.db.cursor()
!             self._iter_cursors[str(cur)] = cur
  
              # since we're only returning keys, we call the cursor
              # methods with flags=0, dlen=0, dofs=0
!             curkey = cur.first(0,0,0)[0]
!             yield curkey
  
              next = cur.next
              while 1:
                  try:
!                     curkey = next(0,0,0)[0]
!                     yield curkey
                  except _bsddb.DBCursorClosedError:
!                     # our cursor object was closed since we last yielded
!                     # create a new one and attempt to reposition to the
!                     # right place
!                     cur = self.db.cursor()
!                     self._iter_cursors[str(cur)] = cur
                      # FIXME-20031101-greg: race condition.  cursor could
!                     # be closed by another thread before this set call.
!                     try:
!                         cur.set(curkey,0,0,0)
!                     except _bsddb.DBCursorClosedError:
!                         # halt iteration on race condition...
!                         raise _bsddb.DBNotFoundError
                      next = cur.next
          except _bsddb.DBNotFoundError:
!             try:
!                 del self._iter_cursors[str(cur)]
!             except KeyError:
!                 pass
              return
  
      def iteritems(self):
          try:
!             cur = self.db.cursor()
!             self._iter_cursors[str(cur)] = cur
  
              kv = cur.first()
!             curkey = kv[0]
              yield kv
  
--- 68,123 ----
      exec """
  import UserDict
+ from weakref import ref
  class _iter_mixin(UserDict.DictMixin):
+     def _make_iter_cursor(self):
+         cur = self.db.cursor()
+         key = id(cur)
+         self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
+         return cur
+ 
+     def _gen_cref_cleaner(self, key):
+         # use generate the function for the weakref callback here
+         # to ensure that we do not hold a strict reference to cur
+         # in the callback.
+         return lambda ref: self._cursor_refs.pop(key, None)
+ 
      def __iter__(self):
          try:
!             cur = self._make_iter_cursor()
! 
!             # FIXME-20031102-greg: race condition.  cursor could
!             # be closed by another thread before this call.
  
              # since we're only returning keys, we call the cursor
              # methods with flags=0, dlen=0, dofs=0
!             key = cur.first(0,0,0)[0]
!             yield key
  
              next = cur.next
              while 1:
                  try:
!                     key = next(0,0,0)[0]
!                     yield key
                  except _bsddb.DBCursorClosedError:
!                     cur = self._make_iter_cursor()
                      # FIXME-20031101-greg: race condition.  cursor could
!                     # be closed by another thread before this call.
!                     cur.set(key,0,0,0)
                      next = cur.next
          except _bsddb.DBNotFoundError:
!             return
!         except _bsddb.DBCursorClosedError:
!             # the database was modified during iteration.  abort.
              return
  
      def iteritems(self):
          try:
!             cur = self._make_iter_cursor()
! 
!             # FIXME-20031102-greg: race condition.  cursor could
!             # be closed by another thread before this call.
  
              kv = cur.first()
!             key = kv[0]
              yield kv
  
***************
*** 118,142 ****
                  try:
                      kv = next()
!                     curkey = kv[0]
                      yield kv
                  except _bsddb.DBCursorClosedError:
!                     # our cursor object was closed since we last yielded
!                     # create a new one and attempt to reposition to the
!                     # right place
!                     cur = self.db.cursor()
!                     self._iter_cursors[str(cur)] = cur
                      # FIXME-20031101-greg: race condition.  cursor could
!                     # be closed by another thread before this set call.
!                     try:
!                         cur.set(curkey,0,0,0)
!                     except _bsddb.DBCursorClosedError:
!                         # halt iteration on race condition...
!                         raise _bsddb.DBNotFoundError
                      next = cur.next
          except _bsddb.DBNotFoundError:
!             try:
!                 del self._iter_cursors[str(cur)]
!             except KeyError:
!                 pass
              return
  """
--- 126,141 ----
                  try:
                      kv = next()
!                     key = kv[0]
                      yield kv
                  except _bsddb.DBCursorClosedError:
!                     cur = self._make_iter_cursor()
                      # FIXME-20031101-greg: race condition.  cursor could
!                     # be closed by another thread before this call.
!                     cur.set(key,0,0,0)
                      next = cur.next
          except _bsddb.DBNotFoundError:
!             return
!         except _bsddb.DBCursorClosedError:
!             # the database was modified during iteration.  abort.
              return
  """
***************
*** 160,164 ****
          # reason is that _checkCursor and _closeCursors are not atomic
          # operations.  Doing our own locking around self.dbc,
!         # self.saved_dbc_key and self._iter_cursors could prevent this.
          # TODO: A test case demonstrating the problem needs to be written.
  
--- 159,163 ----
          # reason is that _checkCursor and _closeCursors are not atomic
          # operations.  Doing our own locking around self.dbc,
!         # self.saved_dbc_key and self._cursor_refs could prevent this.
          # TODO: A test case demonstrating the problem needs to be written.
  
***************
*** 170,182 ****
          # a collection of all DBCursor objects currently allocated
          # by the _iter_mixin interface.
!         self._iter_cursors = {}
! 
  
      def __del__(self):
          self.close()
  
-     def _get_dbc(self):
-         return self.dbc
- 
      def _checkCursor(self):
          if self.dbc is None:
--- 169,177 ----
          # a collection of all DBCursor objects currently allocated
          # by the _iter_mixin interface.
!         self._cursor_refs = {}
  
      def __del__(self):
          self.close()
  
      def _checkCursor(self):
          if self.dbc is None:
***************
*** 198,202 ****
              c.close()
              del c
!         map(lambda c: c.close(), self._iter_cursors.values())
  
      def _checkOpen(self):
--- 193,200 ----
              c.close()
              del c
!         for cref in self._cursor_refs.values():
!             c = cref()
!             if c is not None:
!                 c.close()
  
      def _checkOpen(self):