close[t:5134] le_cursor api cleaned up, followed up on a TODO to get rid of...

close[t:5134] le_cursor api cleaned up, followed up on a TODO to get rid of the DB parameter from is_key_right (and improved its name too) by using a fake_db in the le_cursor plus the new toku_ft_get_descriptor() api to grab a descriptor reference. refactored some tests to get it all compiling. LE and hot indexer tests pass


git-svn-id: file:///svn/toku/tokudb@44895 c7de825b-a66e-492c-adef-691d508d4ae1

ft/le-cursor.c

@@ -4,48 +4,62 @@
 #ident "Copyright (c) 2010 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
+#include <db.h>
 #include "includes.h"
 #include "le-cursor.h"
 
-// A LE_CURSOR is a special type of FT_CURSOR that retrieves all of the leaf entries in a BRT.
+// A LE_CURSOR is a special purpose FT_CURSOR that:
+//  - enables prefetching
+//  - does not perform snapshot reads. it reads everything, including uncommitted.
+//
+// A LE_CURSOR is good for scanning a FT from beginning to end. Useful for hot indexing.
+//
 // It caches the key that is was last positioned over to speed up key comparisions.
 
 struct le_cursor {
+    // TODO: remove DBs from the ft layer comparison function 
+    // so this is never necessary
+    // use a fake db for comparisons. 
+    struct __toku_db fake_db;
     FT_CURSOR ft_cursor;
     DBT key;           // the key that the le cursor is positioned at
                        // TODO a better implementation would fetch the key from the brt cursor 
-    BOOL neg_infinity; // TRUE when the le cursor is positioned at -infinity (initial setting)
-    BOOL pos_infinity; // TRUE when the le cursor is positioned at +infinity (when _next returns DB_NOTFOUND)
+    bool neg_infinity; // true when the le cursor is positioned at -infinity (initial setting)
+    bool pos_infinity; // true when the le cursor is positioned at +infinity (when _next returns DB_NOTFOUND)
 };
 
 int 
-le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE brt, TOKUTXN txn) {
+toku_le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE brt, TOKUTXN txn) {
     int result = 0;
     LE_CURSOR le_cursor = (LE_CURSOR) toku_malloc(sizeof (struct le_cursor));
-    if (le_cursor == NULL)
+    if (le_cursor == NULL) {
         result = errno;
+    }
     else {
-        result = toku_ft_cursor(brt, &le_cursor->ft_cursor, txn, FALSE, FALSE);
+        result = toku_ft_cursor(brt, &le_cursor->ft_cursor, txn, false, false);
         if (result == 0) {
             // TODO move the leaf mode to the brt cursor constructor
             toku_ft_cursor_set_leaf_mode(le_cursor->ft_cursor);
             toku_init_dbt(&le_cursor->key); 
             le_cursor->key.flags = DB_DBT_REALLOC;
-            le_cursor->neg_infinity = TRUE;
-            le_cursor->pos_infinity = FALSE;
+            le_cursor->neg_infinity = true;
+            le_cursor->pos_infinity = false;
+            // zero out the fake DB. this is a rare operation so it's not too slow.
+            memset(&le_cursor->fake_db, 0, sizeof(le_cursor->fake_db));
         }
     }
 
-    if (result == 0)
+    if (result == 0) {
         *le_cursor_result = le_cursor;
-    else
+    } else {
         toku_free(le_cursor);
+    }
 
     return result;
 }
 
 int 
-le_cursor_close(LE_CURSOR le_cursor) {
+toku_le_cursor_close(LE_CURSOR le_cursor) {
     int result = toku_ft_cursor_close(le_cursor->ft_cursor);
     toku_destroy_dbt(&le_cursor->key);
     toku_free(le_cursor);
@@ -75,35 +89,40 @@ le_cursor_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, voi
 }
 
 int 
-le_cursor_next(LE_CURSOR le_cursor, DBT *le) {
+toku_le_cursor_next(LE_CURSOR le_cursor, DBT *le) {
     int result;
     if (le_cursor->pos_infinity)
         result = DB_NOTFOUND;
     else {
-        le_cursor->neg_infinity = FALSE;
+        le_cursor->neg_infinity = false;
         struct le_cursor_callback_arg arg = { &le_cursor->key, le };
         // TODO replace this with a non deprecated function
         result = toku_ft_cursor_get(le_cursor->ft_cursor, NULL, le_cursor_callback, &arg, DB_NEXT);
         if (result == DB_NOTFOUND)
-            le_cursor->pos_infinity = TRUE;
+            le_cursor->pos_infinity = true;
     }
     return result;
 }
 
-BOOL
-is_key_right_of_le_cursor(LE_CURSOR le_cursor, const DBT *key, DB *keycompare_db) {
-    BOOL result;
-    if (le_cursor->neg_infinity)
-        result = TRUE;      // all keys are right of -infinity
-    else if (le_cursor->pos_infinity)
-        result = FALSE;     // all keys are left of +infinity
-    else {
-        ft_compare_func keycompare = toku_ft_get_bt_compare(le_cursor->ft_cursor->ft_handle);
-        int r = keycompare(keycompare_db, &le_cursor->key, key);
-        if (r < 0)
-            result = TRUE;  // key is right of the cursor key
-        else
-            result = FALSE; // key is at or left of the cursor key
+bool
+toku_le_cursor_is_key_greater(LE_CURSOR le_cursor, const DBT *key) {
+    bool result;
+    if (le_cursor->neg_infinity) {
+        result = true;      // all keys are greater than -infinity
+    } else if (le_cursor->pos_infinity) {
+        result = false;     // all keys are less than +infinity
+    } else {
+        // get the comparison function and descriptor from the cursor's ft
+        FT_HANDLE ft_handle = le_cursor->ft_cursor->ft_handle;
+        ft_compare_func keycompare = toku_ft_get_bt_compare(ft_handle);
+        // store the descriptor in the fake DB to do a key comparison
+        le_cursor->fake_db.cmp_descriptor = toku_ft_get_cmp_descriptor(ft_handle);
+        int r = keycompare(&le_cursor->fake_db, &le_cursor->key, key);
+        if (r < 0) {
+            result = true;  // key is right of the cursor key
+        } else {
+            result = false; // key is at or left of the cursor key
+        }
     }
     return result;
 }

ft/le-cursor.h

@@ -20,28 +20,23 @@ typedef struct le_cursor *LE_CURSOR;
 // Create a leaf cursor for a tree (brt) within a transaction (txn)
 // Success: returns 0, stores the LE_CURSOR in the le_cursor_result
 // Failure: returns a non-zero error number
-int le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE brt, TOKUTXN txn);
+int toku_le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE brt, TOKUTXN txn);
 
 // Close and free the LE_CURSOR
 // Success: returns 0
 // Failure: returns a non-zero error number
-int le_cursor_close(LE_CURSOR le_cursor);
+int toku_le_cursor_close(LE_CURSOR le_cursor);
 
 // Retrieve the next leaf entry under the LE_CURSOR
 // Success: returns zero, stores the leaf entry key into the key dbt, and the leaf entry into the val dbt
 // Failure: returns a non-zero error number
-int le_cursor_next(LE_CURSOR le_cursor, DBT *le);
+int toku_le_cursor_next(LE_CURSOR le_cursor, DBT *le);
 
-// Return TRUE if the key is to the right of the LE_CURSOR position
-// Otherwise returns FALSE when the key is at or to the left of the LE_CURSOR position
+// Return TRUE if the key is to the right of the LE_CURSOR position. that is, current cursor key < given key
+// Otherwise returns FALSE when the key is at or to the left of the LE_CURSOR position. that is, current cursor key >= given key
 // The LE_CURSOR position is intialized to -infinity. Any key comparision with -infinity returns TRUE.
 // When the cursor runs off the right edge of the tree, the LE_CURSOR position is set to +infinity.  Any key comparision with +infinity
 // returns FALSE.
-// TODO: review the DB parameter. it probably makes sense to get rid of it
-// to reduce complexity. instead, we can get the descriptor from:
-// lecursor->ftcursor->ft_handle->h->desc
-// or:
-// some flavor of toku_ft_get_desc() etc
-BOOL is_key_right_of_le_cursor(LE_CURSOR le_cursor, const DBT *key, DB *keycompare_db);
+bool toku_le_cursor_is_key_greater(LE_CURSOR le_cursor, const DBT *key);
 
 #endif

ft/tests/le-cursor-provdel.c

@@ -131,7 +131,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
     assert(error == 0);
 
     LE_CURSOR cursor = NULL;
-    error = le_cursor_create(&cursor, brt, cursortxn);
+    error = toku_le_cursor_create(&cursor, brt, cursortxn);
     assert(error == 0);
 
     DBT key;
@@ -141,7 +141,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
 
     int i;
     for (i=0; ; i++) {
-        error = le_cursor_next(cursor, &val);
+        error = toku_le_cursor_next(cursor, &val);
         if (error != 0) 
             break;
         
@@ -157,7 +157,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
     toku_destroy_dbt(&key);
     toku_destroy_dbt(&val);
 
-    error = le_cursor_close(cursor);
+    error = toku_le_cursor_close(cursor);
     assert(error == 0);
 
     error = toku_txn_commit_txn(cursortxn, TRUE, NULL, NULL);

ft/tests/le-cursor-right.c

@@ -3,7 +3,7 @@
 #ident "$Id$"
 #ident "Copyright (c) 2010 Tokutek Inc.  All rights reserved."
 
-// test the LE_CURSOR is_key_right_of_le_cursor function
+// test the LE_CURSOR toku_le_cursor_is_key_greater function
 // - LE_CURSOR at neg infinity
 // - LE_CURSOR at pos infinity
 // - LE_CURSOR somewhere else
@@ -85,7 +85,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
     assert(error == 0);
 }
 
-// test is_key_right_of_le_cursor when the LE_CURSOR is positioned at -infinity
+// test toku_le_cursor_is_key_greater when the LE_CURSOR is positioned at -infinity
 static void 
 test_neg_infinity(const char *fname, int n) {
     if (verbose) fprintf(stderr, "%s %s %d\n", __FUNCTION__, fname, n);
@@ -101,18 +101,18 @@ test_neg_infinity(const char *fname, int n) {
 
     // position the cursor at -infinity
     LE_CURSOR cursor = NULL;
-    error = le_cursor_create(&cursor, brt, NULL);
+    error = toku_le_cursor_create(&cursor, brt, NULL);
     assert(error == 0);
 
     for (int i = 0; i < 2*n; i++) {
         int k = toku_htonl(i);
         DBT key;
         toku_fill_dbt(&key, &k, sizeof k);
-        int right = is_key_right_of_le_cursor(cursor, &key, null_db);
+        int right = toku_le_cursor_is_key_greater(cursor, &key);
         assert(right == TRUE);
     }
         
-    error = le_cursor_close(cursor);
+    error = toku_le_cursor_close(cursor);
     assert(error == 0);
 
     error = toku_close_ft_handle_nolsn(brt, 0);
@@ -122,7 +122,7 @@ test_neg_infinity(const char *fname, int n) {
     assert(error == 0);
 }
 
-// test is_key_right_of_le_cursor when the LE_CURSOR is positioned at +infinity
+// test toku_le_cursor_is_key_greater when the LE_CURSOR is positioned at +infinity
 static void 
 test_pos_infinity(const char *fname, int n) {
     if (verbose) fprintf(stderr, "%s %s %d\n", __FUNCTION__, fname, n);
@@ -138,7 +138,7 @@ test_pos_infinity(const char *fname, int n) {
 
     // position the LE_CURSOR at +infinity
     LE_CURSOR cursor = NULL;
-    error = le_cursor_create(&cursor, brt, NULL);
+    error = toku_le_cursor_create(&cursor, brt, NULL);
     assert(error == 0);
 
     DBT key;
@@ -148,7 +148,7 @@ test_pos_infinity(const char *fname, int n) {
 
     int i;
     for (i = 0; ; i++) {
-        error = le_cursor_next(cursor, &val);
+        error = toku_le_cursor_next(cursor, &val);
         if (error != 0) 
             break;
         
@@ -169,11 +169,11 @@ test_pos_infinity(const char *fname, int n) {
         int k = toku_htonl(i);
         DBT key2;
         toku_fill_dbt(&key2, &k, sizeof k);
-        int right = is_key_right_of_le_cursor(cursor, &key2, null_db);
+        int right = toku_le_cursor_is_key_greater(cursor, &key2);
         assert(right == FALSE);
     }
 
-    error = le_cursor_close(cursor);
+    error = toku_le_cursor_close(cursor);
     assert(error == 0);
 
     error = toku_close_ft_handle_nolsn(brt, 0);
@@ -183,7 +183,7 @@ test_pos_infinity(const char *fname, int n) {
     assert(error == 0);
 }
 
-// test is_key_right_of_le_cursor when the LE_CURSOR is positioned in between -infinity and +infinity
+// test toku_le_cursor_is_key_greater when the LE_CURSOR is positioned in between -infinity and +infinity
 static void 
 test_between(const char *fname, int n) {
     if (verbose) fprintf(stderr, "%s %s %d\n", __FUNCTION__, fname, n);
@@ -199,7 +199,7 @@ test_between(const char *fname, int n) {
 
     // position the LE_CURSOR at +infinity
     LE_CURSOR cursor = NULL;
-    error = le_cursor_create(&cursor, brt, NULL);
+    error = toku_le_cursor_create(&cursor, brt, NULL);
     assert(error == 0);
 
     DBT key;
@@ -210,7 +210,7 @@ test_between(const char *fname, int n) {
     int i;
     for (i = 0; ; i++) {
         // move the LE_CURSOR forward
-        error = le_cursor_next(cursor, &val);
+        error = toku_le_cursor_next(cursor, &val);
         if (error != 0) 
             break;
         
@@ -226,7 +226,7 @@ test_between(const char *fname, int n) {
             int k = toku_htonl(j);
             DBT key2;
             toku_fill_dbt(&key2, &k, sizeof k);
-            int right = is_key_right_of_le_cursor(cursor, &key2, null_db);
+            int right = toku_le_cursor_is_key_greater(cursor, &key2);
             assert(right == FALSE);
         }
 
@@ -235,7 +235,7 @@ test_between(const char *fname, int n) {
             int k = toku_htonl(j);
             DBT key2;
             toku_fill_dbt(&key2, &k, sizeof k);
-            int right = is_key_right_of_le_cursor(cursor, &key2, null_db);
+            int right = toku_le_cursor_is_key_greater(cursor, &key2);
             assert(right == TRUE);
         }
 
@@ -245,7 +245,7 @@ test_between(const char *fname, int n) {
     toku_destroy_dbt(&key);
     toku_destroy_dbt(&val);
 
-    error = le_cursor_close(cursor);
+    error = toku_le_cursor_close(cursor);
     assert(error == 0);
 
     error = toku_close_ft_handle_nolsn(brt, 0);

ft/tests/le-cursor-walk.c

@@ -97,7 +97,7 @@ walk_tree(const char *fname, int n) {
     assert(error == 0);
 
     LE_CURSOR cursor = NULL;
-    error = le_cursor_create(&cursor, brt, NULL);
+    error = toku_le_cursor_create(&cursor, brt, NULL);
     assert(error == 0);
 
     DBT key;
@@ -109,7 +109,7 @@ walk_tree(const char *fname, int n) {
     for (i = 0; ; i++) {
 	error = TOKUDB_TRY_AGAIN;
 	while (error == TOKUDB_TRY_AGAIN) {
-	    error = le_cursor_next(cursor, &val);
+	    error = toku_le_cursor_next(cursor, &val);
 	}
         if (error != 0) 
             break;
@@ -126,7 +126,7 @@ walk_tree(const char *fname, int n) {
     toku_destroy_dbt(&key);
     toku_destroy_dbt(&val);
 
-    error = le_cursor_close(cursor);
+    error = toku_le_cursor_close(cursor);
     assert(error == 0);
 
     error = toku_close_ft_handle_nolsn(brt, 0);

src/indexer.c

@@ -115,7 +115,9 @@ static void
 free_indexer_resources(DB_INDEXER *indexer) {
     if ( indexer->i ) {        
         toku_mutex_destroy(&indexer->i->indexer_lock);
-        if ( indexer->i->lec )   { le_cursor_close(indexer->i->lec); }
+        if ( indexer->i->lec ) {
+            toku_le_cursor_close(indexer->i->lec);
+        }
         if ( indexer->i->fnums ) { 
             toku_free(indexer->i->fnums); 
             indexer->i->fnums = NULL;
@@ -212,7 +214,7 @@ toku_indexer_create_indexer(DB_ENV *env,
     }
     
     // create and initialize the leafentry cursor
-    rval = le_cursor_create(&indexer->i->lec, db_struct_i(src_db)->ft_handle, db_txn_struct_i(txn)->tokutxn);
+    rval = toku_le_cursor_create(&indexer->i->lec, db_struct_i(src_db)->ft_handle, db_txn_struct_i(txn)->tokutxn);
     if ( !indexer->i->lec ) { goto create_exit; }
     
     // 2954: add recovery and rollback entries
@@ -271,9 +273,11 @@ toku_indexer_set_error_callback(DB_INDEXER *indexer,
     return 0;
 }
 
-BOOL
-toku_indexer_is_key_right_of_le_cursor(DB_INDEXER *indexer, DB *db, const DBT *key) {
-    return is_key_right_of_le_cursor(indexer->i->lec, key, db);
+// a key is to the right of the indexer's cursor if it compares
+// greater than the current le cursor position.
+bool
+toku_indexer_is_key_right_of_le_cursor(DB_INDEXER *indexer, const DBT *key) {
+    return toku_le_cursor_is_key_greater(indexer->i->lec, key);
 }
 
 static int 
@@ -283,14 +287,14 @@ build_index(DB_INDEXER *indexer) {
     DBT key; toku_init_dbt_flags(&key, DB_DBT_REALLOC);
     DBT le; toku_init_dbt_flags(&le, DB_DBT_REALLOC);
 
-    BOOL done = FALSE;
+    bool done = false;
     for (uint64_t loop_count = 0; !done; loop_count++) {
 
         toku_indexer_lock(indexer);
 
-        result = le_cursor_next(indexer->i->lec, &le);
+        result = toku_le_cursor_next(indexer->i->lec, &le);
         if (result != 0) {
-            done = TRUE;
+            done = true;
             if (result == DB_NOTFOUND)
                 result = 0;  // all done, normal way to exit loop successfully
         }
@@ -313,7 +317,7 @@ build_index(DB_INDEXER *indexer) {
         if (result == 0) 
             result = maybe_call_poll_func(indexer, loop_count);
         if (result != 0)
-            done = TRUE;
+            done = true;
     }
 
     toku_destroy_dbt(&key);

src/indexer.h

@@ -66,9 +66,9 @@ int toku_indexer_set_error_callback(DB_INDEXER *indexer,
                                     void *error_extra);
 
 // Is the key right of the indexer's leaf entry cursor?
-// Returns TRUE  if right of le_cursor
-// Returns FALSE if left or equal to le_cursor
-BOOL toku_indexer_is_key_right_of_le_cursor(DB_INDEXER *indexer, DB *db, const DBT *key);
+// Returns true  if right of le_cursor
+// Returns false if left or equal to le_cursor
+bool toku_indexer_is_key_right_of_le_cursor(DB_INDEXER *indexer, const DBT *key);
 
 // Get the indexer's source db
 DB *toku_indexer_get_src_db(DB_INDEXER *indexer);
@@ -92,7 +92,7 @@ typedef enum {
 } indexer_status_entry;
 
 typedef struct {
-    BOOL initialized;
+    bool initialized;
     TOKU_ENGINE_STATUS_ROW_S status[INDEXER_STATUS_NUM_ROWS];
 } INDEXER_STATUS_S, *INDEXER_STATUS;
 

src/ydb_write.c

@@ -365,7 +365,7 @@ do_del_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT keys[], DB *s
                 invariant(which_src_db < num_dbs);
                 indexer_src_key = &keys[which_src_db];
             }
-            do_delete = !toku_indexer_is_key_right_of_le_cursor(indexer, indexer_src_db, indexer_src_key);
+            do_delete = !toku_indexer_is_key_right_of_le_cursor(indexer, indexer_src_key);
         }
         if (r == 0 && do_delete) {
             r = toku_ft_maybe_delete(db->i->ft_handle, &keys[which_db], ttxn, FALSE, ZERO_LSN, FALSE);
@@ -546,7 +546,7 @@ do_put_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT keys[], DBT v
                 invariant(which_src_db < num_dbs);
                 indexer_src_key = &keys[which_src_db];
             }
-            do_put = !toku_indexer_is_key_right_of_le_cursor(indexer, indexer_src_db, indexer_src_key);
+            do_put = !toku_indexer_is_key_right_of_le_cursor(indexer, indexer_src_key);
         }
         if (r == 0 && do_put) {
             r = toku_ft_maybe_insert(db->i->ft_handle, &keys[which_db], &vals[which_db], ttxn, FALSE, ZERO_LSN, FALSE, FT_INSERT);