bsddb 4.1.6:

 * Extended DB & DBEnv set_get_returns_none functionality to take a
   "level" instead of a boolean flag.  The boolean 0 and 1 values still
   have the same effect.  A value of 2 extends the "return None instead
   of raising an exception" behaviour to the DBCursor set methods.
   This will become the default behaviour in pybsddb 4.2.
 * Fixed a typo in DBCursor.join_item method that made it crash instead
   of returning a value.  Obviously nobody uses it.  Wrote a test case
   for join and join_item.

Lib/bsddb/dbobj.py

@@ -15,6 +15,12 @@
 #           implied.
 #
 
+#
+# TODO it would be *really nice* to have an automatic shadow class populator
+# so that new methods don't need to be added  here manually after being
+# added to _bsddb.c.
+#
+
 import db
 
 try:
@@ -57,6 +63,8 @@ class DBEnv:
         return apply(self._cobj.set_lk_max_objects, args, kwargs)
     def set_mp_mmapsize(self, *args, **kwargs):
         return apply(self._cobj.set_mp_mmapsize, args, kwargs)
+    def set_timeout(self, *args, **kwargs):
+        return apply(self._cobj.set_timeout, args, kwargs)
     def set_tmp_dir(self, *args, **kwargs):
         return apply(self._cobj.set_tmp_dir, args, kwargs)
     def txn_begin(self, *args, **kwargs):

Lib/bsddb/dbtables.py

@@ -155,6 +155,9 @@ class bsdTableDB :
         if truncate:
             myflags |= DB_TRUNCATE
         self.db = DB(self.env)
+        # this code relies on DBCursor.set* methods to raise exceptions
+        # rather than returning None
+        self.db.set_get_returns_none(1)
         # allow duplicate entries [warning: be careful w/ metadata]
         self.db.set_flags(DB_DUP)
         self.db.open(filename, DB_BTREE, dbflags | myflags, mode)

Lib/bsddb/test/test_associate.py

 """
-TestCases for multi-threaded access to a DB.
+TestCases for DB.associate.
 """
 
 import sys, os, string

Lib/bsddb/test/test_basics.py

@@ -282,11 +282,11 @@ class BasicTestCase(unittest.TestCase):
 
     #----------------------------------------
 
-    def test03_SimpleCursorStuff(self):
+    def test03_SimpleCursorStuff(self, get_raises_error=0, set_raises_error=1):
         if verbose:
             print '\n', '-=' * 30
-            print "Running %s.test03_SimpleCursorStuff..." % \
-                  self.__class__.__name__
+            print "Running %s.test03_SimpleCursorStuff (get_error %s, set_error %s)..." % \
+                  (self.__class__.__name__, get_raises_error, set_raises_error)
 
         if self.env and self.dbopenflags & db.DB_AUTO_COMMIT:
             txn = self.env.txn_begin()
@@ -300,7 +300,15 @@ class BasicTestCase(unittest.TestCase):
             count = count + 1
             if verbose and count % 100 == 0:
                 print rec
-            rec = c.next()
+            try:
+                rec = c.next()
+            except db.DBNotFoundError, val:
+                if get_raises_error:
+                    assert val[0] == db.DB_NOTFOUND
+                    if verbose: print val
+                    rec = None
+                else:
+                    self.fail("unexpected DBNotFoundError")
 
         assert count == 1000
 
@@ -311,7 +319,15 @@ class BasicTestCase(unittest.TestCase):
             count = count + 1
             if verbose and count % 100 == 0:
                 print rec
-            rec = c.prev()
+            try:
+                rec = c.prev()
+            except db.DBNotFoundError, val:
+                if get_raises_error:
+                    assert val[0] == db.DB_NOTFOUND
+                    if verbose: print val
+                    rec = None
+                else:
+                    self.fail("unexpected DBNotFoundError")
 
         assert count == 1000
 
@@ -322,23 +338,29 @@ class BasicTestCase(unittest.TestCase):
         assert rec[1] == self.makeData('0505')
 
         try:
-            c.set('bad key')
+            n = c.set('bad key')
         except db.DBNotFoundError, val:
             assert val[0] == db.DB_NOTFOUND
             if verbose: print val
         else:
-            self.fail("expected exception")
+            if set_raises_error:
+                self.fail("expected exception")
+            if n != None:
+                self.fail("expected None: "+`n`)
 
         rec = c.get_both('0404', self.makeData('0404'))
         assert rec == ('0404', self.makeData('0404'))
 
         try:
-            c.get_both('0404', 'bad data')
+            n = c.get_both('0404', 'bad data')
         except db.DBNotFoundError, val:
             assert val[0] == db.DB_NOTFOUND
             if verbose: print val
         else:
-            self.fail("expected exception")
+            if get_raises_error:
+                self.fail("expected exception")
+            if n != None:
+                self.fail("expected None: "+`n`)
 
         if self.d.get_type() == db.DB_BTREE:
             rec = c.set_range('011')
@@ -414,6 +436,29 @@ class BasicTestCase(unittest.TestCase):
         # SF pybsddb bug id 667343
         del oldcursor
 
+    def test03b_SimpleCursorWithoutGetReturnsNone0(self):
+        # same test but raise exceptions instead of returning None
+        if verbose:
+            print '\n', '-=' * 30
+            print "Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone..." % \
+                  self.__class__.__name__
+
+        old = self.d.set_get_returns_none(0)
+        assert old == 1
+        self.test03_SimpleCursorStuff(get_raises_error=1, set_raises_error=1)
+
+    def test03c_SimpleCursorGetReturnsNone2(self):
+        # same test but raise exceptions instead of returning None
+        if verbose:
+            print '\n', '-=' * 30
+            print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \
+                  self.__class__.__name__
+
+        old = self.d.set_get_returns_none(2)
+        assert old == 1
+        old = self.d.set_get_returns_none(2)
+        assert old == 2
+        self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0)
 
     #----------------------------------------
 

Lib/bsddb/test/test_join.py

 """TestCases for using the DB.join and DBCursor.join_item methods.
 """
 
+import sys, os, string
+import tempfile
+import time
+from pprint import pprint
+
+try:
+    from threading import Thread, currentThread
+    have_threads = 1
+except ImportError:
+    have_threads = 0
+
 import unittest
+from test_all import verbose
+
+try:
+    # For Python 2.3
+    from bsddb import db, dbshelve
+except ImportError:
+    # For earlier Pythons w/distutils pybsddb
+    from bsddb3 import db, dbshelve
+
+
+#----------------------------------------------------------------------
+
+ProductIndex = [
+    ('apple', "Convenience Store"),
+    ('blueberry', "Farmer's Market"),
+    ('shotgun', "S-Mart"),              # Aisle 12
+    ('pear', "Farmer's Market"),
+    ('chainsaw', "S-Mart"),             # "Shop smart.  Shop S-Mart!"
+    ('strawberry', "Farmer's Market"),
+]
+
+ColorIndex = [
+    ('blue', "blueberry"),
+    ('red', "apple"),
+    ('red', "chainsaw"),
+    ('red', "strawberry"),
+    ('yellow', "peach"),
+    ('yellow', "pear"),
+    ('black', "shotgun"),
+]
+
+class JoinTestCase(unittest.TestCase):
+    keytype = ''
+
+    def setUp(self):
+        self.filename = self.__class__.__name__ + '.db'
+        homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home')
+        self.homeDir = homeDir
+        try: os.mkdir(homeDir)
+        except os.error: pass
+        self.env = db.DBEnv()
+        self.env.open(homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK )
+
+    def tearDown(self):
+        self.env.close()
+        import glob
+        files = glob.glob(os.path.join(self.homeDir, '*'))
+        for file in files:
+            os.remove(file)
+
+    def test01_join(self):
+        if verbose:
+            print '\n', '-=' * 30
+            print "Running %s.test01_join..." % \
+                  self.__class__.__name__
+
+        # create and populate primary index
+        priDB = db.DB(self.env)
+        priDB.open(self.filename, "primary", db.DB_BTREE, db.DB_CREATE)
+        map(lambda t: apply(priDB.put, t), ProductIndex)
+
+        # create and populate secondary index
+        secDB = db.DB(self.env)
+        secDB.set_flags(db.DB_DUP | db.DB_DUPSORT)
+        secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE)
+        map(lambda t: apply(secDB.put, t), ColorIndex)
+
+        sCursor = None
+        jCursor = None
+        try:
+            # lets look up all of the red Products
+            sCursor = secDB.cursor()
+            assert sCursor.set('red')
+
+            # FIXME: jCursor doesn't properly hold a reference to its
+            # cursors, if they are closed before jcursor is used it
+            # can cause a crash.
+            jCursor = priDB.join([sCursor])
+
+            if jCursor.get(0) != ('apple', "Convenience Store"):
+                self.fail("join cursor positioned wrong")
+            if jCursor.join_item() != 'chainsaw':
+                self.fail("DBCursor.join_item returned wrong item")
+            if jCursor.get(0)[0] != 'strawberry':
+                self.fail("join cursor returned wrong thing")
+            if jCursor.get(0):  # there were only three red items to return
+                self.fail("join cursor returned too many items")
+        finally:
+            if jCursor:
+                jCursor.close()
+            if sCursor:
+                sCursor.close()
+            priDB.close()
+            secDB.close()
 
 
 def test_suite():
     suite = unittest.TestSuite()
+
+    suite.addTest(unittest.makeSuite(JoinTestCase))
+
     return suite

Modules/_bsddb.c

@@ -93,7 +93,7 @@
 /* 40 = 4.0, 33 = 3.3; this will break if the second number is > 9 */
 #define DBVER (DB_VERSION_MAJOR * 10 + DB_VERSION_MINOR)
 
-#define PY_BSDDB_VERSION "4.1.5"
+#define PY_BSDDB_VERSION "4.1.6"
 static char *rcs_id = "$Id$";
 
 
@@ -141,12 +141,6 @@ static PyInterpreterState* _db_interpreterState = NULL;
 
 #endif
 
-
-/* What is the default behaviour when DB->get or DBCursor->get returns a
-   DB_NOTFOUND error?  Return None or raise an exception? */
-#define GET_RETURNS_NONE_DEFAULT 1
-
-
 /* Should DB_INCOMPLETE be turned into a warning or an exception? */
 #define INCOMPLETE_IS_WARNING 1
 
@@ -189,12 +183,24 @@ static PyObject* DBPermissionsError;    /* EPERM  */
 /* --------------------------------------------------------------------- */
 /* Structure definitions */
 
+struct behaviourFlags {
+    /* What is the default behaviour when DB->get or DBCursor->get returns a
+       DB_NOTFOUND error?  Return None or raise an exception? */
+    unsigned int getReturnsNone : 1;
+    /* What is the default behaviour for DBCursor.set* methods when DBCursor->get
+     * returns a DB_NOTFOUND error?  Return None or raise an exception? */
+    unsigned int cursorSetReturnsNone : 1;
+};
+
+#define DEFAULT_GET_RETURNS_NONE                1
+#define DEFAULT_CURSOR_SET_RETURNS_NONE         0   /* 0 in pybsddb < 4.2, python < 2.4 */
+
 typedef struct {
     PyObject_HEAD
     DB_ENV*     db_env;
     u_int32_t   flags;             /* saved flags from open() */
     int         closed;
-    int         getReturnsNone;
+    struct behaviourFlags moduleFlags;
 } DBEnvObject;
 
 
@@ -205,7 +211,7 @@ typedef struct {
     u_int32_t       flags;     /* saved flags from open() */
     u_int32_t       setflags;  /* saved flags from set_flags() */
     int             haveStat;
-    int             getReturnsNone;
+    struct behaviourFlags moduleFlags;
 #if (DBVER >= 33)
     PyObject*       associateCallback;
     int             primaryDBType;
@@ -595,7 +601,7 @@ static PyObject* _DBCursor_get(DBCursorObject* self, int extra_flags,
     err = self->dbc->c_get(self->dbc, &key, &data, flags);
     MYDB_END_ALLOW_THREADS;
 
-    if ((err == DB_NOTFOUND) && self->mydb->getReturnsNone) {
+    if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) {
         Py_INCREF(Py_None);
         retval = Py_None;
     }
@@ -681,9 +687,10 @@ newDBObject(DBEnvObject* arg, int flags)
     }
 
     if (self->myenvobj)
-        self->getReturnsNone = self->myenvobj->getReturnsNone;
+        self->moduleFlags = self->myenvobj->moduleFlags;
     else
-        self->getReturnsNone = GET_RETURNS_NONE_DEFAULT;
+        self->moduleFlags.getReturnsNone = DEFAULT_GET_RETURNS_NONE;
+        self->moduleFlags.cursorSetReturnsNone = DEFAULT_CURSOR_SET_RETURNS_NONE;
 
     MYDB_BEGIN_ALLOW_THREADS;
     err = db_create(&self->db, db_env, flags);
@@ -797,7 +804,8 @@ newDBEnvObject(int flags)
 
     self->closed = 1;
     self->flags = flags;
-    self->getReturnsNone = GET_RETURNS_NONE_DEFAULT;
+    self->moduleFlags.getReturnsNone = DEFAULT_GET_RETURNS_NONE;
+    self->moduleFlags.cursorSetReturnsNone = DEFAULT_CURSOR_SET_RETURNS_NONE;
 
     MYDB_BEGIN_ALLOW_THREADS;
     err = db_env_create(&self->db_env, flags);
@@ -1182,7 +1190,7 @@ _DB_consume(DBObject* self, PyObject* args, PyObject* kwargs, int consume_flag)
     err = self->db->get(self->db, txn, &key, &data, flags|consume_flag);
     MYDB_END_ALLOW_THREADS;
 
-    if ((err == DB_NOTFOUND) && self->getReturnsNone) {
+    if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) {
         err = 0;
         Py_INCREF(Py_None);
         retval = Py_None;
@@ -1324,7 +1332,7 @@ DB_get(DBObject* self, PyObject* args, PyObject* kwargs)
         Py_INCREF(dfltobj);
         retval = dfltobj;
     }
-    else if ((err == DB_NOTFOUND) && self->getReturnsNone) {
+    else if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) {
         err = 0;
         Py_INCREF(Py_None);
         retval = Py_None;
@@ -1424,7 +1432,7 @@ DB_get_both(DBObject* self, PyObject* args, PyObject* kwargs)
     err = self->db->get(self->db, txn, &key, &data, flags);
     MYDB_END_ALLOW_THREADS;
 
-    if ((err == DB_NOTFOUND) && self->getReturnsNone) {
+    if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) {
         err = 0;
         Py_INCREF(Py_None);
         retval = Py_None;
@@ -1525,6 +1533,11 @@ DB_join(DBObject* self, PyObject* args)
     free(cursors);
     RETURN_IF_ERR();
 
+    // FIXME: this is a buggy interface.  The returned cursor
+    // contains internal references to the passed in cursors
+    // but does not hold python references to them or prevent
+    // them from being closed prematurely.  This can cause
+    // python to crash when things are done in the wrong order.
     return (PyObject*) newDBCursorObject(dbc, self);
 }
 
@@ -2156,14 +2169,18 @@ static PyObject*
 DB_set_get_returns_none(DBObject* self, PyObject* args)
 {
     int flags=0;
-    int oldValue;
+    int oldValue=0;
 
     if (!PyArg_ParseTuple(args,"i:set_get_returns_none", &flags))
         return NULL;
     CHECK_DB_NOT_CLOSED(self);
 
-    oldValue = self->getReturnsNone;
-    self->getReturnsNone = flags;
+    if (self->moduleFlags.getReturnsNone)
+        ++oldValue;
+    if (self->moduleFlags.cursorSetReturnsNone)
+        ++oldValue;
+    self->moduleFlags.getReturnsNone = (flags >= 1);
+    self->moduleFlags.cursorSetReturnsNone = (flags >= 2);
     return PyInt_FromLong(oldValue);
 }
 
@@ -2643,7 +2660,7 @@ DBC_get(DBCursorObject* self, PyObject* args, PyObject *kwargs)
     MYDB_END_ALLOW_THREADS;
 
 
-    if ((err == DB_NOTFOUND) && self->mydb->getReturnsNone) {
+    if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) {
         Py_INCREF(Py_None);
         retval = Py_None;
     }
@@ -2790,7 +2807,11 @@ DBC_set(DBCursorObject* self, PyObject* args, PyObject *kwargs)
     MYDB_BEGIN_ALLOW_THREADS;
     err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET);
     MYDB_END_ALLOW_THREADS;
-    if (makeDBError(err)) {
+    if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) {
+        Py_INCREF(Py_None);
+        retval = Py_None;
+    }
+    else if (makeDBError(err)) {
         retval = NULL;
     }
     else {
@@ -2848,7 +2869,11 @@ DBC_set_range(DBCursorObject* self, PyObject* args, PyObject* kwargs)
     MYDB_BEGIN_ALLOW_THREADS;
     err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET_RANGE);
     MYDB_END_ALLOW_THREADS;
-    if (makeDBError(err)) {
+    if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) {
+        Py_INCREF(Py_None);
+        retval = Py_None;
+    }
+    else if (makeDBError(err)) {
         retval = NULL;
     }
     else {
@@ -2875,19 +2900,15 @@ DBC_set_range(DBCursorObject* self, PyObject* args, PyObject* kwargs)
     return retval;
 }
 
-
 static PyObject*
-DBC_get_both(DBCursorObject* self, PyObject* args)
+_DBC_get_set_both(DBCursorObject* self, PyObject* keyobj, PyObject* dataobj,
+                  int flags, unsigned int returnsNone)
 {
-    int err, flags=0;
+    int err;
     DBT key, data;
-    PyObject* retval, *keyobj, *dataobj;
-
-    if (!PyArg_ParseTuple(args, "OO|i:get_both", &keyobj, &dataobj, &flags))
-        return NULL;
-
-    CHECK_CURSOR_NOT_CLOSED(self);
+    PyObject* retval;
 
+    // the caller did this:  CHECK_CURSOR_NOT_CLOSED(self);
     if (!make_key_dbt(self->mydb, keyobj, &key, NULL))
         return NULL;
     if (!make_dbt(dataobj, &data))
@@ -2896,7 +2917,11 @@ DBC_get_both(DBCursorObject* self, PyObject* args)
     MYDB_BEGIN_ALLOW_THREADS;
     err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_GET_BOTH);
     MYDB_END_ALLOW_THREADS;
-    if (makeDBError(err)) {
+    if ((err == DB_NOTFOUND) && returnsNone) {
+        Py_INCREF(Py_None);
+        retval = Py_None;
+    }
+    else if (makeDBError(err)) {
         retval = NULL;
     }
     else {
@@ -2922,6 +2947,38 @@ DBC_get_both(DBCursorObject* self, PyObject* args)
     return retval;
 }
 
+static PyObject*
+DBC_get_both(DBCursorObject* self, PyObject* args)
+{
+    int flags=0;
+    PyObject *keyobj, *dataobj;
+
+    if (!PyArg_ParseTuple(args, "OO|i:get_both", &keyobj, &dataobj, &flags))
+        return NULL;
+
+    // if the cursor is closed, self->mydb may be invalid
+    CHECK_CURSOR_NOT_CLOSED(self);
+
+    return _DBC_get_set_both(self, keyobj, dataobj, flags,
+                self->mydb->moduleFlags.getReturnsNone);
+}
+
+static PyObject*
+DBC_set_both(DBCursorObject* self, PyObject* args)
+{
+    int flags=0;
+    PyObject *keyobj, *dataobj;
+
+    if (!PyArg_ParseTuple(args, "OO|i:set_both", &keyobj, &dataobj, &flags))
+        return NULL;
+
+    // if the cursor is closed, self->mydb may be invalid
+    CHECK_CURSOR_NOT_CLOSED(self);
+
+    return _DBC_get_set_both(self, keyobj, dataobj, flags,
+                self->mydb->moduleFlags.cursorSetReturnsNone);
+}
+
 
 static PyObject*
 DBC_set_recno(DBCursorObject* self, PyObject* args, PyObject *kwargs)
@@ -2965,7 +3022,11 @@ DBC_set_recno(DBCursorObject* self, PyObject* args, PyObject *kwargs)
     MYDB_BEGIN_ALLOW_THREADS;
     err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET_RECNO);
     MYDB_END_ALLOW_THREADS;
-    if (makeDBError(err)) {
+    if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) {
+        Py_INCREF(Py_None);
+        retval = Py_None;
+    }
+    else if (makeDBError(err)) {
         retval = NULL;
     }
     else {  /* Can only be used for BTrees, so no need to return int key */
@@ -3010,11 +3071,11 @@ DBC_prev_nodup(DBCursorObject* self, PyObject* args, PyObject *kwargs)
 static PyObject*
 DBC_join_item(DBCursorObject* self, PyObject* args)
 {
-    int err;
+    int err, flags=0;
     DBT key, data;
     PyObject* retval;
 
-    if (!PyArg_ParseTuple(args, ":join_item"))
+    if (!PyArg_ParseTuple(args, "|i:join_item", &flags))
         return NULL;
 
     CHECK_CURSOR_NOT_CLOSED(self);
@@ -3027,9 +3088,13 @@ DBC_join_item(DBCursorObject* self, PyObject* args)
     }
 
     MYDB_BEGIN_ALLOW_THREADS;
-    err = self->dbc->c_get(self->dbc, &key, &data, DB_JOIN_ITEM);
+    err = self->dbc->c_get(self->dbc, &key, &data, flags | DB_JOIN_ITEM);
     MYDB_END_ALLOW_THREADS;
-    if (makeDBError(err)) {
+    if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) {
+        Py_INCREF(Py_None);
+        retval = Py_None;
+    }
+    else if (makeDBError(err)) {
         retval = NULL;
     }
     else {
@@ -3748,14 +3813,18 @@ static PyObject*
 DBEnv_set_get_returns_none(DBEnvObject* self, PyObject* args)
 {
     int flags=0;
-    int oldValue;
+    int oldValue=0;
 
     if (!PyArg_ParseTuple(args,"i:set_get_returns_none", &flags))
         return NULL;
     CHECK_ENV_NOT_CLOSED(self);
 
-    oldValue = self->getReturnsNone;
-    self->getReturnsNone = flags;
+    if (self->moduleFlags.getReturnsNone)
+        ++oldValue;
+    if (self->moduleFlags.cursorSetReturnsNone)
+        ++oldValue;
+    self->moduleFlags.getReturnsNone = (flags >= 1);
+    self->moduleFlags.cursorSetReturnsNone = (flags >= 2);
     return PyInt_FromLong(oldValue);
 }
 
@@ -3977,7 +4046,7 @@ static PyMethodDef DBCursor_methods[] = {
     {"set",             (PyCFunction)DBC_set,           METH_VARARGS|METH_KEYWORDS},
     {"set_range",       (PyCFunction)DBC_set_range,     METH_VARARGS|METH_KEYWORDS},
     {"get_both",        (PyCFunction)DBC_get_both,      METH_VARARGS},
-    {"set_both",        (PyCFunction)DBC_get_both,      METH_VARARGS},
+    {"set_both",        (PyCFunction)DBC_set_both,      METH_VARARGS},
     {"set_recno",       (PyCFunction)DBC_set_recno,     METH_VARARGS|METH_KEYWORDS},
     {"consume",         (PyCFunction)DBC_consume,       METH_VARARGS|METH_KEYWORDS},
     {"next_dup",        (PyCFunction)DBC_next_dup,      METH_VARARGS|METH_KEYWORDS},