refs #5254 merge fast upserts to mainline. enabled on mysql 5.6 only for now.

git-svn-id: file:///svn/mysql/tokudb-engine/tokudb-engine@50363 c7de825b-a66e-492c-adef-691d508d4ae1

storage/tokudb/Makefile.am

-#called from the top level Makefile
-
-MYSQLDATAdir =          $(localstatedir)
-MYSQLSHAREdir =         $(pkgdatadir)
-MYSQLBASEdir =          $(prefix)
-MYSQLLIBdir =           $(pkglibdir)
-pkgplugindir =          $(pkglibdir)/plugin
-INCLUDES =              -I$(top_builddir)/include \
-			-I$(top_srcdir)/include \
-			-I$(top_srcdir)/regex \
-			-I$(top_srcdir)/sql \
-                        -I$(srcdir) \
-			-I$(TOKUFRACTALTREE)/include
-
-LDADD =
-
-DEFS =	@DEFS@
-
-noinst_HEADERS	  =	
-
-EXTRA_LTLIBRARIES =	ha_tokudb.la
-pkgplugin_LTLIBRARIES =	@plugin_tokudb_shared_target@
-ha_tokudb_la_LIBADD =   -L$(TOKUFRACTALTREE)/lib -l$(TOKUFRACTALTREE_LIBNAME) -l$(TOKUPORTABILITY_LIBNAME) -lz -lstdc++
-ha_tokudb_la_LDFLAGS =	-module -rpath $(pkgplugindir)
-ha_tokudb_la_CXXFLAGS =	$(AM_CXXFLAGS) -DMYSQL_DYNAMIC_PLUGIN -DTOKUDB_VERSION=\"$(TOKUDB_VERSION)\" -Wall
-ha_tokudb_la_SOURCES =	ha_tokudb.cc
-
-EXTRA_LIBRARIES =	libtokudb.a
-noinst_LIBRARIES =	@plugin_tokudb_static_target@
-libtokudb_a_CXXFLAGS =	$(AM_CXXFLAGS) -Wall
-libtokudb_a_SOURCES =	ha_tokudb.cc
-
-EXTRA_DIST =		CMakeLists.txt plug.in
-# Don't update the files from bitkeeper
-%::SCCS/s.%

storage/tokudb/Makefile.win

-
-TARGETS= CMakeLists.txt
-
-.DELETE_ON_ERROR:
-.PHONY: CMakeLists.txt #It depends on the contents of the TOKUDB_DIR variable, simpler to just make it phony then depend on that.
-ifeq ($(TOKUDB_DIR),)
-    #error
-CMakeLists.txt: CMakeLists.in
-	false #Need to define TOKUDB_DIR
-else
-CMakeLists.txt: CMakeLists.in
-	sed -e "s?TOKUDB_DIR_REPLACE_ME?$(TOKUDB_DIR)?g" \
-	    -e "s?TOKUDB_OBJ_DIR_REPLACE_ME?$(TOKUDB_OBJ_DIR)?g" \
-	    -e "s?TOKUDB_VERSION_REPLACE_ME?$(TOKUDB_VERSION)?g" < $< > $@
-endif
-
-clean:
-	rm -f $(TARGETS)
-

storage/tokudb/ha_tokudb.cc

@@ -661,15 +661,14 @@ void set_key_filter(MY_BITMAP* key_filter, KEY* key, TABLE* table, bool get_offs
                 // key is only the first 3 characters, and we end up losing the last 7 bytes of the
                 // column
                 //
-                TOKU_TYPE toku_type;
-                toku_type = mysql_to_toku_type(field);
-                switch(toku_type) {
-                case(toku_type_blob):
+                TOKU_TYPE toku_type = mysql_to_toku_type(field);
+                switch (toku_type) {
+                case toku_type_blob:
                     break;
-                case(toku_type_varbinary):
-                case(toku_type_varstring):
-                case(toku_type_fixbinary):
-                case(toku_type_fixstring):
+                case toku_type_varbinary:
+                case toku_type_varstring:
+                case toku_type_fixbinary:
+                case toku_type_fixstring:
                     if (key->key_part[curr_key_index].length == field->field_length) {
                         bitmap_set_bit(key_filter,i);
                     }
@@ -3829,7 +3828,7 @@ int ha_tokudb::write_row(uchar * record) {
             else {
                 update_max_auto_inc(share->status_block, share->last_auto_increment);
             }
-         }
+        }
         pthread_mutex_unlock(&share->mutex);
     }
 
@@ -8017,8 +8016,13 @@ void ha_tokudb::set_dup_value_for_pk(DBT* key) {
 // table admin 
 #include "ha_tokudb_admin.cc"
 
-// alter table code for various mysql distros
+// update functions
 #include "ha_tokudb_update_fun.cc"
+
+// fast updates
+#include "ha_tokudb_update.cc"
+
+// alter table code for various mysql distros
 #include "ha_tokudb_alter_51.cc"
 #include "ha_tokudb_alter_55.cc"
 #include "ha_tokudb_alter_56.cc"

storage/tokudb/ha_tokudb.h

@@ -669,6 +669,18 @@ private:
 #if TOKU_INCLUDE_WRITE_FRM_DATA
     int write_frm_data(const uchar *frm_data, size_t frm_len);
 #endif
+#if TOKU_INCLUDE_UPSERT
+public:
+    bool fast_update(THD *thd, List<Item> &fields, List<Item> &values, Item *conds, int *error_ret);
+private:
+    bool check_fast_update(THD *thd, List<Item> &fields, List<Item> &values, Item *conds);
+    int send_update_message(List<Item> &fields, List<Item> &values, Item *conds, DB_TXN *txn);
+public:
+    bool upsert(THD *thd, uchar *record, List<Item> &update_fields, List<Item> &update_values, int *error_ret);
+private:
+    bool check_upsert(THD *thd, List<Item> &update_fields, List<Item> &update_values);
+    int send_upsert_message(THD *thd, uchar *record, List<Item> &update_fields, List<Item> &update_values, DB_TXN *txn);
+#endif
 };
 
 #if MYSQL_VERSION_ID >= 50506

storage/tokudb/hatoku_defines.h

@@ -43,6 +43,7 @@
 #define TOKU_INCLUDE_XA 1
 #define TOKU_PARTITION_WRITE_FRM_DATA 1
 #define TOKU_INCLUDE_WRITE_FRM_DATA 1
+#define TOKU_INCLUDE_UPSERT 1
 
 #elif 50500 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50599
 #define TOKU_INCLUDE_ALTER_56 1

storage/tokudb/hatoku_hton.cc

@@ -139,6 +139,36 @@ static MYSQL_THDVAR_UINT(read_buf_size,
     1*1024*1024,   // max
     1      // blocksize???
 );
+#if TOKU_INCLUDE_UPSERT
+static MYSQL_THDVAR_BOOL(enable_fast_update, 
+    PLUGIN_VAR_THDLOCAL, 
+    "enable fast update",
+    NULL, // check
+    NULL, // update
+    false // default
+);
+static MYSQL_THDVAR_BOOL(disable_slow_update, 
+    PLUGIN_VAR_THDLOCAL, 
+    "disable slow update",
+    NULL, // check
+    NULL, // update
+    false // default
+);
+static MYSQL_THDVAR_BOOL(enable_fast_upsert, 
+    PLUGIN_VAR_THDLOCAL, 
+    "enable fast upsert",
+    NULL, // check
+    NULL, // update
+    false // default
+);
+static MYSQL_THDVAR_BOOL(disable_slow_upsert, 
+    PLUGIN_VAR_THDLOCAL, 
+    "disable slow upsert",
+    NULL, // check
+    NULL, // update
+    false // default
+);
+#endif
 
 static void tokudb_checkpoint_lock(THD * thd);
 static void tokudb_checkpoint_unlock(THD * thd);
@@ -707,6 +737,24 @@ uint get_tokudb_read_buf_size(THD* thd) {
     return THDVAR(thd, read_buf_size);
 }
 
+#if TOKU_INCLUDE_UPSERT
+bool get_enable_fast_update(THD* thd) {
+    return (THDVAR(thd, enable_fast_update) != 0);
+}
+
+bool get_disable_slow_update(THD* thd) {
+    return (THDVAR(thd, disable_slow_update) != 0);
+}
+
+bool get_enable_fast_upsert(THD* thd) {
+    return (THDVAR(thd, enable_fast_upsert) != 0);
+}
+
+bool get_disable_slow_upsert(THD* thd) {
+    return (THDVAR(thd, disable_slow_upsert) != 0);
+}
+#endif
+
 typedef struct txn_progress_info {
     char status[200];
     THD* thd;
@@ -1628,6 +1676,12 @@ static struct st_mysql_sys_var *tokudb_system_variables[] = {
     MYSQL_SYSVAR(read_block_size),
     MYSQL_SYSVAR(read_buf_size),
     MYSQL_SYSVAR(row_format),
+#if TOKU_INCLUDE_UPSERT
+    MYSQL_SYSVAR(enable_fast_update),
+    MYSQL_SYSVAR(disable_slow_update),
+    MYSQL_SYSVAR(enable_fast_upsert),
+    MYSQL_SYSVAR(disable_slow_upsert),
+#endif
     NULL
 };
 

storage/tokudb/hatoku_hton.h

@@ -33,6 +33,12 @@ uint get_tokudb_block_size(THD* thd);
 uint get_tokudb_read_block_size(THD* thd);
 uint get_tokudb_read_buf_size(THD* thd);
 srv_row_format_t get_row_format(THD *thd);
+#if TOKU_INCLUDE_UPSERT
+bool get_enable_fast_update(THD *thd);
+bool get_disable_slow_update(THD *thd);
+bool get_enable_fast_upsert(THD *thd);
+bool get_disable_slow_upsert(THD *thd);
+#endif
 
 extern HASH tokudb_open_tables;
 extern pthread_mutex_t tokudb_mutex;

storage/tokudb/plug.in

-MYSQL_STORAGE_ENGINE(tokudb,,[TokuDB Storage Engine with Fractal Trees], [TokuDB Engine with Fractal Trees], [max,max-no-ndb])
-MYSQL_PLUGIN_DIRECTORY(tokudb, [storage/tokudb])
-MYSQL_PLUGIN_DYNAMIC(tokudb, [ha_tokudb.la])

storage/tokudb/tests/Makefile

+SRCS = $(wildcard *.cc)
+TARGETS = $(patsubst %.cc,%,$(SRCS))
+CHECKS = $(patsubst %,%.check,$(TARGETS))
+CPPFLAGS = -I..
+CXXFLAGS = -g
+
+ifeq ($(GCOV),1)
+CXXFLAGS += -fprofile-arcs -ftest-coverage
+endif
+
+all: $(TARGETS)
+
+clean:
+	rm -rf $(TARGETS) *.gcov *.gcno *.gcda
+
+check: $(CHECKS)
+	true
+
+%.check: %
+	valgrind ./$<
+

storage/tokudb/tests/sint_test.cc

+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include <tokudb_math.h>
+using namespace tokudb;
+
+static void test(int length_bits) {
+    int64_t max = (1ULL << (length_bits-1)) - 1;
+    for (int64_t x = -max-1; x <= max; x++) {
+        for (int64_t y = -max-1; y <= max; y++) {
+            bool over;
+            int64_t n = int_add(x, y, length_bits, over);
+            printf("%lld %lld %lld %u\n", x, y, n, over);
+        }
+    }
+}
+
+int main(int argc, char *argv[]) {
+    if (argc > 1) {
+        for (int i = 1; i < argc; i++) {
+            test(atoi(argv[i]));
+        }
+    }
+    return 0;
+}

storage/tokudb/tests/tokudb_buffer_test.cc

+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <tokudb_buffer.h>
+
+static void test_null() {
+    tokudb::buffer b;
+    assert(b.data() == NULL && b.size() == 0 && b.limit() == 0);
+    b.append(NULL, 0);
+    assert(b.data() == NULL && b.size() == 0 && b.limit() == 0);
+}
+
+static void append_az(tokudb::buffer &b) {
+    for (char c = 'a'; c <= 'z'; c++) {
+        b.append(&c, sizeof c);
+    }
+}
+
+static void assert_az(tokudb::buffer &b) {
+    char *bp = NULL;
+    for (int i = 0; i < b.size(); i++) {
+        bp = (char *) b.data() + i;
+        assert(*bp == 'a'+i);    
+    }
+    assert(*bp == 'z');
+}
+
+static void assert_AZ(tokudb::buffer &b) {
+    char *bp = NULL;
+    for (int i = 0; i < b.size(); i++) {
+        bp = (char *) b.data() + i;
+        assert(*bp == 'A'+i);    
+    }
+    assert(*bp == 'Z');
+}
+
+static void test_append() {
+    tokudb::buffer a;
+
+    a.append(NULL, 0);
+    append_az(a);
+    a.append(NULL, 0);
+    assert(a.size() == 'z'-'a'+1);
+    assert(a.size() <= a.limit());
+    assert_az(a);
+
+    tokudb::buffer b(a.data(), 0, a.size());
+    for (int i = 0; i < b.limit(); i++) {
+        assert(i <= a.size());
+        char *ap = (char *) a.data() + i;
+        assert(i <= b.limit());
+        char *bp = (char *) b.data() + i;
+        assert(*ap == *bp);
+    }
+}
+
+static void test_consume() {
+    tokudb::buffer a;
+    append_az(a);
+    tokudb::buffer b(a.data(), 0, a.size());
+    for (int i = 0; i < b.limit(); i++) {
+        char c;
+        b.consume(&c, 1);
+        assert(c == 'a'+i);
+    }
+    assert(b.size() == b.limit());
+}
+
+static void test_consume_ptr() {
+    tokudb::buffer a;
+    append_az(a);
+    tokudb::buffer b(a.data(), 0, a.size());
+    for (int i = 0; i < b.limit(); i++) {
+        void *p = b.consume_ptr(1);
+        char c = *(char *)p;
+        assert(c == 'a'+i);
+    }
+    assert(b.size() == b.limit());
+    assert(b.consume_ptr(1) == NULL);
+}
+
+static void test_replace() {
+    tokudb::buffer a;
+    append_az(a);
+    assert_az(a);
+    for (int i = 0; i < a.size(); i++) {
+        char newc[1] = { 'A' + i };
+        a.replace(i, 1, newc, 1);
+    }
+    assert_AZ(a);
+}
+
+static void test_replace_grow() {
+    tokudb::buffer a;
+    append_az(a);
+    assert_az(a);
+
+    // grow field
+    int s = a.size();
+    for (int i = 0; i < s; i++) {
+        char newc[2] = { 'a'+i, 'a'+i };
+        size_t s = a.size();
+        a.replace(2*i, 1, newc, 2);
+        assert(a.size() == s+1);
+    }
+    for (int i = 0; i < a.size()/2; i++) {
+        char *cp = (char *) a.data() + 2*i;
+        assert(cp[0] == 'a'+i && cp[1] == 'a'+i);
+    }
+}
+
+static void test_replace_shrink() {
+    tokudb::buffer a;
+    for (char c = 'a'; c <= 'z'; c++) {
+        a.append(&c, sizeof c);
+        a.append(&c, sizeof c);
+    }
+
+    // shrink field
+    for (int i = 0; i < a.size(); i++) {
+        char newc[1] = { 'a'+i };
+        size_t s = a.size();
+        a.replace(i, 2, newc, 1);
+        assert(a.size() == s-1);
+    }
+    assert_az(a);
+}
+
+static void test_replace_null() {
+    tokudb::buffer a;
+    append_az(a);
+    assert_az(a);
+
+    // insert between all
+    int n = a.size();
+    for (int i = 0; i < n; i++) {
+        char newc[1] = { 'a'+i };
+        a.replace(2*i, 0, newc, 1);
+    }
+
+    a.replace(a.size(), 0, (void *)"!", 1);
+    a.append((void *)"?", 1);
+}
+
+int main() {
+    test_null();
+    test_append();
+    test_consume();
+    test_consume_ptr();
+    test_replace();
+    test_replace_grow();
+    test_replace_shrink();
+    test_replace_null();
+    return 0;
+}

storage/tokudb/tests/tokudb_math_test.cc

+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include <tokudb_math.h>
+using namespace tokudb;
+
+static void test_uint_range(uint length_bits) {
+    assert(uint_low_endpoint(length_bits) == 0);
+    if (length_bits == 64)
+        assert(uint_high_endpoint(length_bits) == ~0ULL);
+    else
+        assert(uint_high_endpoint(length_bits) == (1ULL<<length_bits)-1);
+}
+
+static void test_uint8() {
+    printf("%s\n", __FUNCTION__);
+    test_uint_range(8);
+    bool over;
+    uint8_t n;
+    uint64_t m;
+    for (uint64_t x = 0; x <= (1ULL<<8)-1; x++) {
+        for (uint64_t y = 0; y <= (1ULL<<8)-1; y++) {
+            n = uint_add(x, y, 8, over);
+            m = x + y;
+            if (m > (1ULL<<8)-1)
+                assert(over);
+            else 
+                assert(!over && n == (m % 256));
+            n = uint_sub(x, y, 8, over);
+            m = x - y;
+            if (m > x)
+                assert(over);
+            else
+                assert(!over && n == (m % 256));
+        }
+    }
+}
+
+static void test_uint16() {
+    printf("%s\n", __FUNCTION__);
+    test_uint_range(16);
+    bool over;
+    uint16_t n;
+    uint64_t m;
+    for (uint64_t x = 0; x <= (1ULL<<16)-1; x++) {
+        for (uint64_t y = 0; y <= (1ULL<<16)-1; y++) {
+            n = uint_add(x, y, 16, over);
+            m = x + y;
+            if (m > (1ULL<<16)-1)
+                assert(over);
+            else
+                assert(!over && n == (m % (1ULL<<16)));
+            n = uint_sub(x, y, 16, over);
+            m = x - y;
+            if (m > x)
+                assert(over);
+            else
+                assert(!over && n == (m % (1ULL<<16)));
+        }
+    }
+}
+
+static void test_uint24() {
+    printf("%s\n", __FUNCTION__);
+    test_uint_range(24);
+    bool over;
+    uint64_t s;
+
+    s = uint_add((1ULL<<24)-1, (1ULL<<24)-1, 24, over); assert(over);
+    s = uint_add((1ULL<<24)-1, 1, 24, over); assert(over);
+    s = uint_add((1ULL<<24)-1, 0, 24, over); assert(!over && s == (1ULL<<24)-1);
+    s = uint_add(0, 1, 24, over); assert(!over && s == 1);
+    s = uint_add(0, 0, 24, over); assert(!over && s == 0);
+    s = uint_sub(0, 0, 24, over); assert(!over && s == 0);
+    s = uint_sub(0, 1, 24, over); assert(over);
+    s = uint_sub(0, (1ULL<<24)-1, 24, over); assert(over);
+    s = uint_sub((1ULL<<24)-1, (1ULL<<24)-1, 24, over); assert(!over && s == 0);
+}
+
+static void test_uint32() {
+    printf("%s\n", __FUNCTION__);
+    test_uint_range(32);
+    bool over;
+    uint64_t s;
+
+    s = uint_add((1ULL<<32)-1, (1ULL<<32)-1, 32, over); assert(over);
+    s = uint_add((1ULL<<32)-1, 1, 32, over); assert(over);
+    s = uint_add((1ULL<<32)-1, 0, 32, over); assert(!over && s == (1ULL<<32)-1);
+    s = uint_add(0, 1, 32, over); assert(!over && s == 1);
+    s = uint_add(0, 0, 32, over); assert(!over && s == 0);
+    s = uint_sub(0, 0, 32, over); assert(!over && s == 0);
+    s = uint_sub(0, 1, 32, over); assert(over);
+    s = uint_sub(0, (1ULL<<32)-1, 32, over); assert(over);
+    s = uint_sub((1ULL<<32)-1, (1ULL<<32)-1, 32, over); assert(!over && s == 0);
+}
+
+static void test_uint64() {
+    printf("%s\n", __FUNCTION__);
+    test_uint_range(64);
+    bool over;
+    uint64_t s;
+
+    s = uint_add(~0ULL, ~0ULL, 64, over); assert(over);
+    s = uint_add(~0ULL, 1, 64, over); assert(over);
+    s = uint_add(~0ULL, 0, 64, over); assert(!over && s == ~0ULL);
+    s = uint_add(0, 1, 64, over); assert(!over && s == 1);
+    s = uint_add(0, 0, 64, over); assert(!over && s == 0);
+    s = uint_sub(0, 0, 64, over); assert(!over && s == 0);
+    s = uint_sub(0, 1, 64, over); assert(over);
+    s = uint_sub(0, ~0ULL, 64, over); assert(over);
+    s = uint_sub(~0ULL, ~0ULL, 64, over); assert(!over && s == 0);
+}
+
+static int64_t sign_extend(uint length_bits, int64_t n) {
+    return n | ~((1ULL<<(length_bits-1))-1);
+}
+
+static void test_int_range(uint length_bits) {
+    assert(int_high_endpoint(length_bits) == (1ULL<<(length_bits-1))-1);
+    assert(int_low_endpoint(length_bits) == sign_extend(length_bits, 1ULL<<(length_bits-1)));
+}
+
+static void test_int8() {
+    printf("%s\n", __FUNCTION__);
+    test_int_range(8);
+    int64_t max = (1LL << 7);
+    for (int64_t x = -max; x <= max-1; x++) {
+        for (int64_t y = -max; y <= max-1; y++) {
+            bool over;
+            int64_t n, m;
+            n = int_add(x, y, 8, over);
+            m = x + y;
+            if (m > max-1)
+                assert(over);
+            else if (m < -max)
+                assert(over);
+            else
+                assert(!over && n == m);
+            n = int_sub(x, y, 8, over);
+            m = x - y;
+            if (m > max-1)
+                assert(over);
+            else if (m < -max)
+                assert(over);
+            else
+                assert(!over && n == m);
+        }
+    }
+}
+
+static void test_int16() {
+    printf("%s\n", __FUNCTION__);
+    test_int_range(16);
+    int64_t max = (1LL << 15);
+    for (int64_t x = -max; x <= max-1; x++) {
+        for (int64_t y = -max; y <= max-1; y++) {
+            bool over;
+            int64_t n, m;
+            n = int_add(x, y, 16, over);
+            m = x + y;
+            if (m > max-1)
+                assert(over);
+            else if (m < -max)
+                assert(over);
+            else
+                assert(!over && n == m);
+            n = int_sub(x, y, 16, over);
+            m = x - y;
+            if (m > max-1)
+                assert(over);
+            else if (m < -max)
+                assert(over);
+            else
+                assert(!over && n == m);
+        }
+    }
+}
+
+static void test_int24() {
+    printf("%s\n", __FUNCTION__);
+    test_int_range(24);
+    int64_t s;
+    bool over;
+
+    s = int_add(1, (1ULL<<23)-1, 24, over); assert(over);
+    s = int_add((1ULL<<23)-1, 1, 24, over); assert(over);
+    s = int_sub(-1, (1ULL<<23), 24, over); assert(!over && s == (1ULL<<23)-1);
+    s = int_sub((1ULL<<23), 1, 24, over); assert(over);
+
+    s = int_add(0, 0, 24, over); assert(!over && s == 0);
+    s = int_sub(0, 0, 24, over); assert(!over && s == 0);
+    s = int_add(0, -1, 24, over); assert(!over && s == -1);
+    s = int_sub(0, 1, 24, over); assert(!over && s == -1);
+    s = int_add(0, (1ULL<<23), 24, over); assert(!over && (s & (1ULL<<24)-1) == (1ULL<<23));
+    s = int_sub(0, (1ULL<<23)-1, 24, over); assert(!over && (s & (1ULL<<24)-1) == (1ULL<<23)+1);
+
+    s = int_add(-1, 0, 24, over); assert(!over && s == -1);
+    s = int_add(-1, 1, 24, over); assert(!over && s == 0);
+    s = int_sub(-1, -1, 24, over); assert(!over && s == 0);
+    s = int_sub(-1, (1ULL<<23)-1, 24, over); assert(!over && (s & (1ULL<<24)-1) == (1ULL<<23));
+}
+
+static void test_int32() {
+    printf("%s\n", __FUNCTION__);
+    test_int_range(32);
+    int64_t s;
+    bool over;
+
+    s = int_add(1, (1ULL<<31)-1, 32, over); assert(over);
+    s = int_add((1ULL<<31)-1, 1, 32, over); assert(over);
+    s = int_sub(-1, (1ULL<<31), 32, over); assert(s == (1ULL<<31)-1 && !over);
+    s = int_sub((1ULL<<31), 1, 32, over); assert(over);
+
+    s = int_add(0, 0, 32, over); assert(s == 0 && !over);
+    s = int_sub(0, 0, 32, over); assert(s == 0 && !over);
+    s = int_add(0, -1, 32, over); assert(s == -1 && !over);
+    s = int_sub(0, 1, 32, over); assert(s == -1 && !over);
+    s = int_add(0, (1ULL<<31), 32, over); assert((s & (1ULL<<32)-1) == (1ULL<<31) && !over);
+    s = int_sub(0, (1ULL<<31)-1, 32, over); assert((s & (1ULL<<32)-1) == (1ULL<<31)+1 && !over);
+
+    s = int_add(-1, 0, 32, over); assert(s == -1 && !over);
+    s = int_add(-1, 1, 32, over); assert(s == 0 && !over);
+    s = int_sub(-1, -1, 32, over); assert(s == 0 && !over);
+    s = int_sub(-1, (1ULL<<31)-1, 32, over); assert((s & (1ULL<<32)-1) == (1ULL<<31) && !over);
+}
+
+static void test_int64() {
+    printf("%s\n", __FUNCTION__);
+    test_int_range(64);
+    int64_t s;
+    bool over;
+
+    s = int_add(1, (1ULL<<63)-1, 64, over); assert(over);
+    s = int_add((1ULL<<63)-1, 1, 64, over); assert(over);
+    s = int_sub(-1, (1ULL<<63), 64, over); assert(s == (1ULL<<63)-1 && !over);
+    s = int_sub((1ULL<<63), 1, 64, over); assert(over);
+
+    s = int_add(0, 0, 64, over); assert(s == 0 && !over);
+    s = int_sub(0, 0, 64, over); assert(s == 0 && !over);
+    s = int_add(0, -1, 64, over); assert(s == -1 && !over);
+    s = int_sub(0, 1, 64, over); assert(s == -1 && !over);
+    s = int_add(0, (1ULL<<63), 64, over); assert(s == (1ULL<<63) && !over);
+    s = int_sub(0, (1ULL<<63)-1, 64, over); assert(s == (1ULL<<63)+1 && !over);
+
+    s = int_add(-1, 0, 64, over); assert(s == -1 && !over);
+    s = int_add(-1, 1, 64, over); assert(s == 0 && !over);
+    s = int_sub(-1, -1, 64, over); assert(s == 0 && !over);
+    s = int_sub(-1, (1ULL<<63)-1, 64, over); assert(s == (1ULL<<63) && !over);
+}
+
+static void test_int_sign(uint length_bits) {
+    printf("%s %u\n", __FUNCTION__, length_bits);
+    int64_t n;
+    
+    n = int_high_endpoint(length_bits);
+    assert(int_sign_extend(n, length_bits) == n);
+    n = (1ULL<<(length_bits-1));
+    assert(int_sign_extend(n, length_bits) == -n);
+}
+
+static void test_int_sign() {
+    test_int_sign(8);
+    test_int_sign(16);
+    test_int_sign(24);
+    test_int_sign(32);
+    test_int_sign(64);
+}
+
+int main(int argc, char *argv[]) {
+    if (1) test_int_sign();
+    if (1) test_int8();
+    if (1) test_int16();
+    if (1) test_int24();
+    if (1) test_int32();
+    if (1) test_int64();
+
+    if (1) test_uint8();
+    if (1) test_uint16();
+    if (1) test_uint24();
+    if (1) test_uint32();
+    if (1) test_uint64();
+    return 0;
+}
+

storage/tokudb/tests/uint_test.cc

+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include <tokudb_math.h>
+using namespace tokudb;
+
+static void test(int length_bits) {
+    printf("%s %d\n", __FUNCTION__, length_bits);
+    uint64_t max = (1ULL << length_bits) - 1;
+    for (uint64_t x = 0; x <= max; x++) {
+        for (uint64_t y = 0; y <= max; y++) {
+            bool over;
+            uint64_t n = uint_add(x, y, max, over);
+            printf("%llu %llu %llu\n", x, y, n);
+        }
+    }
+}
+
+int main(int argc, char *argv[]) {
+    if (argc > 1) {
+        for (int i = 1; i < argc; i++) {
+            test(atoi(argv[i]));
+        }
+    }
+    return 0;
+}

storage/tokudb/tokudb_buffer.h

+#if !defined(_TOKUDB_BUFFER_H)
+#define _TOKUDB_BUFFER_H
+
+namespace tokudb {
+
+// A Buffer manages a contiguous chunk of memory and supports appending new data to the end of the buffer, and
+// consuming chunks from the beginning of the buffer.  The buffer will reallocate memory when appending
+// new data to a full buffer. 
+
+class buffer {
+public:
+    buffer(void *data, size_t s, size_t l) : m_data(data), m_size(s), m_limit(l), m_is_static(true) {
+    }
+    buffer() : m_data(NULL), m_size(0), m_limit(0), m_is_static(false) {
+    }
+    virtual ~buffer() {
+        if (!m_is_static)
+            free(m_data);
+    }
+
+    // Return a pointer to the end of the buffer suitable for appending a fixed number of bytes.
+    void *append_ptr(size_t s) {
+        maybe_realloc(s);
+        void *p = (char *) m_data + m_size;
+        m_size += s;
+        return p;
+    }
+
+    // Append bytes to the buffer
+    void append(void *p, size_t s) {
+        memcpy(append_ptr(s), p, s);
+    }
+
+    // Return a pointer to the next location in the buffer where bytes are consumed from.
+    void *consume_ptr(size_t s) {
+        if (m_size + s > m_limit)
+            return NULL;
+        void *p = (char *) m_data + m_size;
+        m_size += s;
+        return p;
+    }
+
+    // Consume bytes from the buffer.
+    void consume(void *p, size_t s) {
+        memcpy(p, consume_ptr(s), s);
+    }
+
+    // Replace a field in the buffer with new data.  If the new data size is different, then readjust the 
+    // size of the buffer and move things around.
+    void replace(size_t offset, size_t old_s, void *new_p, size_t new_s) {
+        assert(offset + old_s <= m_size);
+        if (new_s > old_s)
+            maybe_realloc(new_s - old_s);
+        char *data_offset = (char *) m_data + offset;
+        if (new_s != old_s) {
+            size_t n = m_size - (offset + old_s);
+            assert(offset + new_s + n <= m_limit && offset + old_s + n <= m_limit);
+            memmove(data_offset + new_s, data_offset + old_s, n);
+            if (new_s > old_s)
+                m_size += new_s - old_s;
+            else
+                m_size -= old_s - new_s;
+            assert(m_size <= m_limit);
+        }
+        memcpy(data_offset, new_p, new_s);
+    }
+
+    // Return a pointer to the data in the buffer
+    void *data() {
+        return m_data;
+    }
+
+    // Return the size of the data in the buffer
+    size_t size() {
+        return m_size;
+    }
+
+    // Return the size of the underlying memory in the buffer
+    size_t limit() {
+        return m_limit;
+    }
+private:
+    // Maybe reallocate the buffer when it becomes full by doubling its size.
+    void maybe_realloc(size_t s) {
+        if (m_size + s > m_limit) {
+            size_t new_limit = m_limit * 2;
+            if (new_limit < m_size + s)
+                new_limit = m_size + s;
+            assert(!m_is_static);
+            m_data = realloc(m_data, new_limit);
+            m_limit = new_limit;
+        }
+    }   
+private:
+    void *m_data;
+    size_t m_size;
+    size_t m_limit;
+    bool m_is_static;
+};
+
+};
+
+#endif

storage/tokudb/tokudb_math.h

+#if !defined(_TOKUDB_MATH_H)
+#define _TOKUDB_MATH_H
+
+namespace tokudb {
+
+// Add and subtract ints with overflow detection.
+// Overflow detection adapted from "Hackers Delight", Henry S. Warren
+
+// Return a bit mask for bits 0 .. length_bits-1
+static uint64_t uint_mask(uint length_bits) __attribute__((unused));
+static uint64_t uint_mask(uint length_bits) {
+    return length_bits == 64 ? ~0ULL : (1ULL<<length_bits)-1;
+}
+
+// Return the highest unsigned int with a given number of bits
+static uint64_t uint_high_endpoint(uint length_bits) __attribute__((unused));
+static uint64_t uint_high_endpoint(uint length_bits) {
+    return uint_mask(length_bits);
+}
+
+// Return the lowest unsigned int with a given number of bits
+static uint64_t uint_low_endpoint(uint length_bits) __attribute__((unused));
+static uint64_t uint_low_endpoint(uint length_bits) {
+    return 0;
+}
+
+// Add two unsigned integers with max maximum value.
+// If there is an overflow then set the sum to the max.
+// Return the sum and the overflow.
+static uint64_t uint_add(uint64_t x, uint64_t y, uint length_bits, bool &over) __attribute__((unused));
+static uint64_t uint_add(uint64_t x, uint64_t y, uint length_bits, bool &over) {
+    uint64_t mask = uint_mask(length_bits);
+    assert((x & ~mask) == 0 && (y & ~mask) == 0);
+    uint64_t s = (x + y) & mask;
+    over = s < x;     // check for overflow
+    return s;
+}
+
+// Subtract two unsigned ints with max maximum value.
+// If there is an over then set the difference to 0.
+// Return the difference and the overflow.
+static uint64_t uint_sub(uint64_t x, uint64_t y, uint length_bits, bool &over) __attribute__((unused));
+static uint64_t uint_sub(uint64_t x, uint64_t y, uint length_bits, bool &over) {
+    uint64_t mask = uint_mask(length_bits);
+    assert((x & ~mask) == 0 && (y & ~mask) == 0);
+    uint64_t s = (x - y) & mask;
+    over = s > x;    // check for overflow
+    return s;
+}
+
+// Return the highest int with a given number of bits
+static int64_t int_high_endpoint(uint length_bits) __attribute__((unused));
+static int64_t int_high_endpoint(uint length_bits) {
+    return (1ULL<<(length_bits-1))-1;
+}
+
+// Return the lowest int with a given number of bits
+static int64_t int_low_endpoint(uint length_bits) __attribute__((unused));
+static int64_t int_low_endpoint(uint length_bits) {
+    int64_t mask = uint_mask(length_bits);
+    return (1ULL<<(length_bits-1)) | ~mask;
+}
+
+// Sign extend to 64 bits an int with a given number of bits
+static int64_t int_sign_extend(int64_t n, uint length_bits) __attribute__((unused));
+static int64_t int_sign_extend(int64_t n, uint length_bits) {
+    if (n & (1ULL<<(length_bits-1)))
+        n |= ~uint_mask(length_bits);
+    return n;
+}
+
+// Add two signed ints with max maximum value.
+// If there is an overflow then set the sum to the max or the min of the int range,
+// depending on the sign bit.
+// Sign extend to 64 bits.
+// Return the sum and the overflow.
+static int64_t int_add(int64_t x, int64_t y, uint length_bits, bool &over) __attribute__((unused));
+static int64_t int_add(int64_t x, int64_t y, uint length_bits, bool &over) {
+    int64_t mask = uint_mask(length_bits);
+    int64_t n = (x + y) & mask;
+    over = (((n ^ x) & (n ^ y)) >> (length_bits-1)) & 1;    // check for overflow
+    if (n & (1LL<<(length_bits-1)))
+        n |= ~mask;    // sign extend
+    return n;
+}
+
+// Subtract two signed ints.
+// If there is an overflow then set the sum to the max or the min of the int range,
+// depending on the sign bit.
+// Sign extend to 64 bits.
+// Return the sum and the overflow.
+static int64_t int_sub(int64_t x, int64_t y, uint length_bits, bool &over) __attribute__((unused));
+static int64_t int_sub(int64_t x, int64_t y, uint length_bits, bool &over) {
+    int64_t mask = uint_mask(length_bits);
+    int64_t n = (x - y) & mask;
+    over = (((x ^ y) & (n ^ x)) >> (length_bits-1)) & 1;    // check for overflow
+    if (n & (1LL<<(length_bits-1)))
+        n |= ~mask;    // sign extend
+    return n;
+}
+
+} // namespace tokudb
+
+#endif