- bsddb: multithreaded DB access using the simple bsddb module interface

  now works reliably.  It has been updated to use automatic BerkeleyDB
  deadlock detection and the bsddb.dbutils.DeadlockWrap wrapper to retry
  database calls that would previously deadlock. [SF python bug #775414]

Lib/bsddb/__init__.py

@@ -33,7 +33,10 @@
 #----------------------------------------------------------------------
 
 
-"""Support for BerkeleyDB 3.2 through 4.2.
+"""Support for BerkeleyDB 3.3 through 4.4 with a simple interface.
+
+For the full featured object oriented interface use the bsddb.db module
+instead.  It mirrors the Sleepycat BerkeleyDB C API.
 """
 
 try:
@@ -43,8 +46,10 @@ try:
         # python as bsddb._bsddb.
         import _pybsddb
         _bsddb = _pybsddb
+        from bsddb3.dbutils import DeadlockWrap as _DeadlockWrap
     else:
         import _bsddb
+        from bsddb.dbutils import DeadlockWrap as _DeadlockWrap
 except ImportError:
     # Remove ourselves from sys.modules
     import sys
@@ -70,7 +75,7 @@ if sys.version >= '2.3':
     exec """
 class _iter_mixin(UserDict.DictMixin):
     def _make_iter_cursor(self):
-        cur = self.db.cursor()
+        cur = _DeadlockWrap(self.db.cursor)
         key = id(cur)
         self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
         return cur
@@ -90,19 +95,19 @@ class _iter_mixin(UserDict.DictMixin):
 
             # since we're only returning keys, we call the cursor
             # methods with flags=0, dlen=0, dofs=0
-            key = cur.first(0,0,0)[0]
+            key = _DeadlockWrap(cur.first, 0,0,0)[0]
             yield key
 
             next = cur.next
             while 1:
                 try:
-                    key = next(0,0,0)[0]
+                    key = _DeadlockWrap(next, 0,0,0)[0]
                     yield key
                 except _bsddb.DBCursorClosedError:
                     cur = self._make_iter_cursor()
                     # FIXME-20031101-greg: race condition.  cursor could
                     # be closed by another thread before this call.
-                    cur.set(key,0,0,0)
+                    _DeadlockWrap(cur.set, key,0,0,0)
                     next = cur.next
         except _bsddb.DBNotFoundError:
             return
@@ -119,21 +124,21 @@ class _iter_mixin(UserDict.DictMixin):
             # FIXME-20031102-greg: race condition.  cursor could
             # be closed by another thread before this call.
 
-            kv = cur.first()
+            kv = _DeadlockWrap(cur.first)
             key = kv[0]
             yield kv
 
             next = cur.next
             while 1:
                 try:
-                    kv = next()
+                    kv = _DeadlockWrap(next)
                     key = kv[0]
                     yield kv
                 except _bsddb.DBCursorClosedError:
                     cur = self._make_iter_cursor()
                     # FIXME-20031101-greg: race condition.  cursor could
                     # be closed by another thread before this call.
-                    cur.set(key,0,0,0)
+                    _DeadlockWrap(cur.set, key,0,0,0)
                     next = cur.next
         except _bsddb.DBNotFoundError:
             return
@@ -177,9 +182,9 @@ class _DBWithCursor(_iter_mixin):
 
     def _checkCursor(self):
         if self.dbc is None:
-            self.dbc = self.db.cursor()
+            self.dbc = _DeadlockWrap(self.db.cursor)
             if self.saved_dbc_key is not None:
-                self.dbc.set(self.saved_dbc_key)
+                _DeadlockWrap(self.dbc.set, self.saved_dbc_key)
                 self.saved_dbc_key = None
 
     # This method is needed for all non-cursor DB calls to avoid
@@ -192,15 +197,15 @@ class _DBWithCursor(_iter_mixin):
             self.dbc = None
             if save:
                 try:
-                    self.saved_dbc_key = c.current(0,0,0)[0]
+                    self.saved_dbc_key = _DeadlockWrap(c.current, 0,0,0)[0]
                 except db.DBError:
                     pass
-            c.close()
+            _DeadlockWrap(c.close)
             del c
         for cref in self._cursor_refs.values():
             c = cref()
             if c is not None:
-                c.close()
+                _DeadlockWrap(c.close)
 
     def _checkOpen(self):
         if self.db is None:
@@ -211,73 +216,77 @@ class _DBWithCursor(_iter_mixin):
 
     def __len__(self):
         self._checkOpen()
-        return len(self.db)
+        return _DeadlockWrap(lambda: len(self.db))  # len(self.db)
 
     def __getitem__(self, key):
         self._checkOpen()
-        return self.db[key]
+        return _DeadlockWrap(lambda: self.db[key])  # self.db[key]
 
     def __setitem__(self, key, value):
         self._checkOpen()
         self._closeCursors()
-        self.db[key] = value
+        def wrapF():
+            self.db[key] = value
+        _DeadlockWrap(wrapF)  # self.db[key] = value
 
     def __delitem__(self, key):
         self._checkOpen()
         self._closeCursors()
-        del self.db[key]
+        def wrapF():
+            del self.db[key]
+        _DeadlockWrap(wrapF)  # del self.db[key]
 
     def close(self):
         self._closeCursors(save=0)
         if self.dbc is not None:
-            self.dbc.close()
+            _DeadlockWrap(self.dbc.close)
         v = 0
         if self.db is not None:
-            v = self.db.close()
+            v = _DeadlockWrap(self.db.close)
         self.dbc = None
         self.db = None
         return v
 
     def keys(self):
         self._checkOpen()
-        return self.db.keys()
+        return _DeadlockWrap(self.db.keys)
 
     def has_key(self, key):
         self._checkOpen()
-        return self.db.has_key(key)
+        return _DeadlockWrap(self.db.has_key, key)
 
     def set_location(self, key):
         self._checkOpen()
         self._checkCursor()
-        return self.dbc.set_range(key)
+        return _DeadlockWrap(self.dbc.set_range, key)
 
     def next(self):
         self._checkOpen()
         self._checkCursor()
-        rv = self.dbc.next()
+        rv = _DeadlockWrap(self.dbc.next)
         return rv
 
     def previous(self):
         self._checkOpen()
         self._checkCursor()
-        rv = self.dbc.prev()
+        rv = _DeadlockWrap(self.dbc.prev)
         return rv
 
     def first(self):
         self._checkOpen()
         self._checkCursor()
-        rv = self.dbc.first()
+        rv = _DeadlockWrap(self.dbc.first)
         return rv
 
     def last(self):
         self._checkOpen()
         self._checkCursor()
-        rv = self.dbc.last()
+        rv = _DeadlockWrap(self.dbc.last)
         return rv
 
     def sync(self):
         self._checkOpen()
-        return self.db.sync()
+        return _DeadlockWrap(self.db.sync)
 
 
 #----------------------------------------------------------------------
@@ -385,5 +394,4 @@ try:
 except ImportError:
     db.DB_THREAD = 0
 
-
 #----------------------------------------------------------------------

Lib/bsddb/dbutils.py

@@ -22,14 +22,14 @@
 
 #
 # import the time.sleep function in a namespace safe way to allow
-# "from bsddb.db import *"
+# "from bsddb.dbutils import *"
 #
 from time import sleep as _sleep
 
 import db
 
 # always sleep at least N seconds between retrys
-_deadlock_MinSleepTime = 1.0/64
+_deadlock_MinSleepTime = 1.0/128
 # never sleep more than N seconds between retrys
 _deadlock_MaxSleepTime = 3.14159
 
@@ -57,7 +57,7 @@ def DeadlockWrap(function, *_args, **_kwargs):
     max_retries = _kwargs.get('max_retries', -1)
     if _kwargs.has_key('max_retries'):
         del _kwargs['max_retries']
-    while 1:
+    while True:
         try:
             return function(*_args, **_kwargs)
         except db.DBLockDeadlockError:

Misc/NEWS

@@ -152,8 +152,14 @@ Extension Modules
   aborts the db transaction safely when a modifier callback fails.
   Fixes SF python patch/bug #1408584.
 
+- bsddb: multithreaded DB access using the simple bsddb module interface
+  now works reliably.  It has been updated to use automatic BerkeleyDB 
+  deadlock detection and the bsddb.dbutils.DeadlockWrap wrapper to retry
+  database calls that would previously deadlock. [SF python bug #775414]
+
 - Patch #1446489: add support for the ZIP64 extensions to zipfile. 
 
+
 Library
 -------