Rename

git-svn-id: file:///svn/tokudb@4 c7de825b-a66e-492c-adef-691d508d4ae1

Makefile

+TAGS: */*.c */*.h
+	etags */*.c */*.h

include/db.h

+#ifndef _YOBI_DB_H
+#define _YOBI_DB_H
+
+#include "ydb-constants.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#if 0
+}
+#endif
+#endif
+
+#include <sys/types.h>
+#include <stdio.h>
+
+typedef enum {
+	DB_BTREE=1,
+	//	DB_HASH=2,
+	//	DB_RECNO=3,
+	//	DB_QUEUE=4,
+	//	DB_UNKNOWN=5			/* Figure it out on open. */
+} DBTYPE;
+
+typedef enum {
+	DB_NOTICE_LOGFILE_CHANGED
+} db_notices;
+
+enum {
+  DB_VERB_CHKPOINT = 0x0001,
+  DB_VERB_DEADLOCK = 0x0002,
+  DB_VERB_RECOVERY = 0x0004
+  
+};
+
+typedef struct yobi_db DB;
+typedef struct yobi_db_btree_stat DB_BTREE_STAT;
+typedef struct yobi_db_env DB_ENV;
+typedef struct yobi_db_key_range DB_KEY_RANGE;
+typedef struct yobi_db_lsn DB_LSN;
+typedef struct yobi_db_txn DB_TXN;
+typedef struct yobi_db_txn_active DB_TXN_ACTIVE;
+typedef struct yobi_db_txn_stat DB_TXN_STAT;
+typedef struct yobi_dbc DBC;
+typedef struct yobi_dbt DBT;
+
+struct yobi_db {
+  void *app_private;
+  int  (*close) (DB *, u_int32_t);
+  int  (*cursor) (DB *, DB_TXN *, DBC **, u_int32_t);
+  int  (*del) (DB *, DB_TXN *, DBT *, u_int32_t);
+  int  (*get) (DB *, DB_TXN *, DBT *, DBT *, u_int32_t);
+  int  (*key_range) (DB *, DB_TXN *, DBT *, DB_KEY_RANGE *, u_int32_t);
+  int  (*open) (DB *, DB_TXN *,
+		const char *, const char *, DBTYPE, u_int32_t, int);
+  int  (*put) (DB *, DB_TXN *, DBT *, DBT *, u_int32_t);
+  int  (*remove) (DB *, const char *, const char *, u_int32_t);
+  int  (*rename) (DB *, const char *, const char *, const char *, u_int32_t);
+  int  (*set_bt_compare) (DB *,
+			  int (*)(DB *, const DBT *, const DBT *));
+  int  (*set_flags)    (DB *, u_int32_t);
+  int  (*stat) (DB *, void *, u_int32_t);
+
+  struct ydb_db_internal *i;
+};
+enum {
+  DB_DBT_MALLOC = 0x002,
+  DB_DBT_REALLOC = 0x010,
+  DB_DBT_USERMEM = 0x020,
+  DB_DBT_DUPOK  = 0x040
+};
+struct yobi_dbt {
+  void	   *app_private;
+  void     *data;
+  u_int32_t flags;
+  u_int32_t size;
+  u_int32_t ulen;
+};
+struct yobi_db_txn {
+  int (*commit) (DB_TXN*, u_int32_t);
+  u_int32_t (*id) (DB_TXN *);
+  // internal stuff
+  struct yobi_db_txn_internal *i;
+};
+struct yobi_dbc {
+  int (*c_get) (DBC *, DBT *, DBT *, u_int32_t);
+  int (*c_close) (DBC *);
+  int (*c_del) (DBC *, u_int32_t);
+  struct yobi_dbc_internal *i;
+};
+struct yobi_db_env {
+  // Methods used by MYSQL
+  void (*err) (const DB_ENV *, int, const char *, ...);
+  int  (*open) (DB_ENV *, const char *, u_int32_t, int);
+  int  (*close) (DB_ENV *, u_int32_t);
+  int  (*txn_checkpoint) (DB_ENV *, u_int32_t, u_int32_t, u_int32_t);
+  int  (*log_flush) (DB_ENV *, const DB_LSN *);
+  void (*set_errcall) (DB_ENV *, void (*)(const char *, char *));
+  void (*set_errpfx) (DB_ENV *, const char *);
+  void (*set_noticecall) (DB_ENV *, void (*)(DB_ENV *, db_notices));
+  int  (*set_flags) (DB_ENV *, u_int32_t, int);
+  int  (*set_data_dir) (DB_ENV *, const char *);
+  int  (*set_tmp_dir) (DB_ENV *, const char *);
+  int  (*set_verbose) (DB_ENV *, u_int32_t, int);
+  int  (*set_lg_bsize) (DB_ENV *, u_int32_t);
+  int  (*set_lg_dir) (DB_ENV *, const char *);
+  int  (*set_lg_max) (DB_ENV *, u_int32_t);
+  int  (*set_cachesize) (DB_ENV *, u_int32_t, u_int32_t, int);
+  int  (*set_lk_detect) (DB_ENV *, u_int32_t);
+  int  (*set_lk_max) (DB_ENV *, u_int32_t);
+  int  (*log_archive) (DB_ENV *, char **[], u_int32_t);
+  int  (*txn_stat) (DB_ENV *, DB_TXN_STAT **, u_int32_t);
+  int  (*txn_begin) (DB_ENV *, DB_TXN *, DB_TXN **, u_int32_t);
+  // Internal state
+  struct db_env_ydb_internal *i;
+};
+struct yobi_db_key_range {
+  double less,equal,grater;
+};
+struct yobi_db_btree_stat {
+  u_int32_t bt_ndata;
+  u_int32_t bt_nkeys;
+};
+struct yobi_db_txn_stat {
+  u_int32_t st_nactive;
+  DB_TXN_ACTIVE *st_txnarray;
+};
+struct yobi_db_lsn {
+  int hello;
+};
+struct yobi_db_txn_active {
+  DB_LSN	lsn;
+  u_int32_t	txnid;
+};
+
+#ifndef _YDB_WRAP_H
+#define DB_VERSION_STRING "Yobiduck: Fractal DB (November 19, 2006)"
+#else
+#define DB_VERSION_STRING_ydb "Yobiduck: Fractal DB (November 19, 2006) (wrapped bdb)"
+#endif
+
+enum {
+  DB_ARCH_ABS = 0x001,
+  DB_ARCH_LOG = 0x004
+};
+
+enum {
+  DB_CREATE     = 0x0000001,
+  DB_RDONLY     = 0x0000010,
+  DB_RECOVER    = 0x0000020,
+  DB_THREAD     = 0x0000040, 
+  DB_TXN_NOSYNC = 0x0000100,
+  
+  DB_PRIVATE    = 0x0100000
+};
+
+enum {
+  DB_LOCK_DEFAULT = 1,
+  DB_LOCK_OLDEST  = 7,
+  DB_LOCK_RANDOM  = 8 
+};
+
+enum {
+  DB_DUP = 0x000002
+};
+
+enum {
+  DB_NOOVERWRITE = 23
+};
+
+enum {
+  DB_INIT_LOCK  = 0x001000,
+  DB_INIT_LOG   = 0x002000,
+  DB_INIT_MPOOL = 0x004000,
+  DB_INIT_TXN   = 0x008000
+};
+
+int db_create (DB **, DB_ENV *, u_int32_t);
+int db_env_create (DB_ENV **, u_int32_t);
+
+int txn_begin (DB_ENV *, DB_TXN *, DB_TXN **, u_int32_t);
+int txn_commit (DB_TXN *, u_int32_t);
+int txn_abort (DB_TXN *);
+
+int log_compare (const DB_LSN *, const DB_LSN *);
+
+#if defined(__cplusplus)
+}
+#endif
+#endif

include/ydb-constants.h

+#ifndef _YDB_CONSTANTS_H
+#define _YDB_CONSTANTS_H
+
+enum {
+  DB_KEYEMPTY      = -30998,
+  DB_KEYEXIST      = -30997,
+  DB_LOCK_DEADLOCK = -30996,
+  DB_NOTFOUND      = -30991,
+
+  // Private
+  DB_BADFORMAT     = -31000
+};
+
+enum {
+  //DB_AFTER   =  1,
+  DB_FIRST     = 10,
+  DB_GET_BOTH  = 11,
+  DB_LAST      = 18,
+  DB_NEXT      = 19,
+  DB_NEXT_DUP  = 20,
+  DB_PREV      = 27,
+  DB_SET       = 30,
+  DB_SET_RANGE = 32,
+  DB_RMW = 0x40000000
+};
+
+#endif

newbrt/Makefile

+# GCOV_FLAGS = -fprofile-arcs -ftest-coverage
+#PROF_FLAGS = -pg
+#OPTFLAGS = -O2
+CFLAGS = -Wall -W $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS) -Werror
+LDFLAGS = $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS) 
+
+default: bins
+BINS =  pma-test brt-test cachetable-test brt-serialize-test randbrt randdb4 hashtest ybt-test
+bins: $(BINS)
+check: bins
+	./ybt-test
+	./mdict-test
+	./pma-test
+	./cachetable-test
+	./brt-serialize-test
+	./brt-test
+	./hashtest
+
+# pma: PROF_FLAGS=-fprofile-arcs -ftest-coverage
+key.o: brttypes.h key.h
+pma-test.o: pma-internal.h pma.h yerror.h memory.h  ../include/ydb-constants.h
+pma-test: pma.o memory.o key.o ybt.o
+pma.o: pma.h yerror.h pma-internal.h memory.h key.h ybt.h brttypes.h  ../include/ydb-constants.h
+ybt.o: ybt.h brttypes.h
+ybt-test: ybt-test.o ybt.o memory.o
+cachetable.o: cachetable.h
+brt-test: brt.o hashtable.o pma.o memory.o brt-serialize.o cachetable.o header-io.o ybt.o key.o
+brt-test.o brt.o: brt.h cachetable.h brttypes.h
+brt-serialize-test.o: pma.h yerror.h brt.h memory.h hashtable.h brttypes.h brt-internal.h
+brt.o: brt.h mdict.h pma.h brttypes.h memory.h brt-internal.h cachetable.h
+mdict.o: pma.h
+hashtable.o: hashtable.h brttypes.h memory.h key.h yerror.h ../include/ydb-constants.h
+memory.o: memory.h
+hashtest: hashtable.o memory.o
+brt-serialize.o: brt.h cachetable.h memory.h mdict.h pma.h brttypes.h brt-internal.h
+header-io.o: brttypes.h brt-internal.h memory.h
+mdict-test: hashtable.o pma.o memory.o
+
+brt-serialize-test: brt-serialize-test.o brt-serialize.o memory.o hashtable.o pma.o key.o ybt.o
+
+cachetable-test.o: cachetable.h memory.h
+cachetable-test: cachetable.o memory.o cachetable-test.o
+
+clean:
+	rm -rf *.o hashtest brt-test cachetable-test randbrt randdb4 *.bb *.bbg *.da
+
+randdb4: LOADLIBES=-ldb
+randbrt: brt.o hashtable.o cachetable.o memory.o brt-serialize.o
+TAGS: ../*/*.c ../*/*.h
+	etags ../*/*.c ../*/*.h

newbrt/brt-internal.h

+#include "cachetable.h"
+#include "hashtable.h"
+#include "pma.h"
+#include "brt.h"
+//#include "pma.h"
+
+typedef long long diskoff;  /* Offset in a disk. -1 is the NULL pointer. */
+
+enum { TREE_FANOUT = 16 }; //, NODESIZE=1<<20 };
+enum { KEY_VALUE_OVERHEAD = 8 }; /* Must store the two lengths. */
+struct nodeheader_in_file {
+    int n_in_buffer;
+    
+};
+enum { BUFFER_HEADER_SIZE = (4 // height//
+			     + 4 // n_children
+			     + TREE_FANOUT * 8 // children
+			     ) };
+typedef struct brtnode *BRTNODE;
+/* Internal nodes. */
+struct brtnode {
+    enum typ_tag tag;
+    unsigned int nodesize;
+    diskoff thisnodename;
+    int    height; /* height is always >= 0.  0 for leaf, >0 for nonleaf. */
+    union node {
+	struct nonleaf {
+	    int             n_children;  /* if n_children==TREE_FANOUT+1 then the tree needs to be rebalanced. */
+	    bytevec         childkeys[TREE_FANOUT];   /* Pivot keys.  Child 0's keys are <= childkeys[0].  Child 1's keys are <= childkeys[1].
+							 Note: It is possible that Child 1's keys are == to child 0's key's, so it is
+							 not necessarily true that child 1's keys are > childkeys[0].
+						         However, in the absense of duplicate keys, child 1's keys *are* > childkeys[0]. */
+	    unsigned int    childkeylens[TREE_FANOUT];
+	    unsigned int    totalchildkeylens;
+	    diskoff         children[TREE_FANOUT+1];  /* unused if height==0 */   /* Note: The last element of these arrays is used only temporarily while splitting a node. */
+	    HASHTABLE       htables[TREE_FANOUT+1];
+	    unsigned int    n_bytes_in_hashtable[TREE_FANOUT+1]; /* how many bytes are in each hashtable (including overheads) */
+	    unsigned int    n_bytes_in_hashtables;
+	} n;
+	struct leaf {
+	    PMA buffer;
+	    unsigned int n_bytes_in_buffer;
+	} l;
+    } u;
+};
+
+struct brt_header {
+    int dirty;
+    unsigned int nodesize;
+    diskoff freelist;
+    diskoff unused_memory;
+    diskoff unnamed_root;
+    int n_named_roots; /* -1 if the only one is unnamed */
+    char  **names;
+    diskoff *roots;
+};
+
+
+struct brt {
+    CACHEFILE cf;
+    char *database_name;
+    // The header is shared.  It is also ephemeral.
+    struct brt_header *h;
+
+    BRT_CURSOR cursors_head, cursors_tail;
+};
+
+/* serialization code */
+void serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node);
+int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesize);
+unsigned int serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
+unsigned int brtnode_which_child (BRTNODE node, bytevec key, ITEMLEN keylen);
+int keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
+
+void verify_counts(BRTNODE);
+
+int serialize_brt_header_to (int fd, struct brt_header *h);
+int deserialize_brtheader_from (int fd, diskoff off, struct brt_header **brth);
+
+static inline int brtnode_n_hashtables(BRTNODE node) { if (node->height==0) return 1; else return node->u.n.n_children; }
+
+//int write_brt_header (int fd, struct brt_header *header);
+
+#if 1
+#define DEADBEEF ((void*)0xDEADBEEF)
+#else
+#define DEADBEEF ((void*)0xDEADBEEFDEADBEEF)
+#endif
+

newbrt/brt-serialize-test.c

+#include "brt.h"
+#include "memory.h"
+#include "brt-internal.h"
+
+#include <fcntl.h>
+#include <assert.h>
+#include <string.h>
+
+void test_serialize(void) {
+//    struct brt source_brt;
+    struct brtnode sn,*dn; /* Source node, Dest node */
+    int fd = open("brt-serialize-test.brt", O_RDWR|O_CREAT, 0777);
+    int r;
+    assert(fd>=0);
+
+//    source_brt.fd=fd;
+    sn.nodesize = 1024;
+    sn.thisnodename = sn.nodesize*20;
+    sn.height = 1;
+    sn.u.n.n_children = 2;
+    sn.u.n.childkeys[0]    = strdup("hello");
+    sn.u.n.childkeylens[0] = 6;
+    sn.u.n.totalchildkeylens = 6;
+    sn.u.n.children[0] = sn.nodesize*30;
+    sn.u.n.children[1] = sn.nodesize*35;
+    r = hashtable_create(&sn.u.n.htables[0]); assert(r==0);
+    r = hashtable_create(&sn.u.n.htables[1]); assert(r==0);
+    r = hash_insert(sn.u.n.htables[0], "a", 2, "aval", 5); assert(r==0);
+    r = hash_insert(sn.u.n.htables[0], "b", 2, "bval", 5); assert(r==0);
+    r = hash_insert(sn.u.n.htables[1], "x", 2, "xval", 5); assert(r==0);
+    sn.u.n.n_bytes_in_hashtables = 3*(KEY_VALUE_OVERHEAD+2+5);
+
+    serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn);
+
+    deserialize_brtnode_from(fd, sn.nodesize*20, &dn, sn.nodesize);
+    assert(dn->thisnodename==sn.nodesize*20);
+    assert(dn->height == 1);
+    assert(dn->u.n.n_children==2);
+    assert(strcmp(dn->u.n.childkeys[0], "hello")==0);
+    assert(dn->u.n.childkeylens[0]==6);
+    assert(dn->u.n.totalchildkeylens==6);
+    assert(dn->u.n.children[0]==sn.nodesize*30);
+    assert(dn->u.n.children[1]==sn.nodesize*35);
+    {
+	bytevec data; ITEMLEN datalen;
+	int r = hash_find(dn->u.n.htables[0], "a", 2, &data, &datalen);
+	assert(r==0);
+	assert(strcmp(data,"aval")==0);
+	assert(datalen==5);
+
+	r=hash_find(dn->u.n.htables[0], "b", 2, &data, &datalen);
+	assert(r==0);
+	assert(strcmp(data,"bval")==0);
+	assert(datalen==5);
+
+	r=hash_find(dn->u.n.htables[1], "x", 2, &data, &datalen);
+	assert(r==0);
+	assert(strcmp(data,"xval")==0);
+	assert(datalen==5);
+
+    }
+}
+
+int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
+    memory_check = 1;
+    test_serialize();
+    return 0;
+}

newbrt/brt.h

+#ifndef BRT_H
+#define BRT_H
+
+// This must be first to make the 64-bit file mode work right in Linux
+#define _FILE_OFFSET_BITS 64
+#include "brttypes.h"
+
+#include "ybt.h"
+#include "../include/ydb-constants.h"
+#include "cachetable.h"
+typedef struct brt *BRT;
+int open_brt (const char *fname, const char *dbname, int is_create, BRT *, int nodesize, CACHETABLE);
+//int brt_create (BRT **, int nodesize, int n_nodes_in_cache); /* the nodesize and n_nodes in cache really should be separately configured. */
+//int brt_open (BRT *, char *fname, char *dbname);
+int brt_insert (BRT brt, bytevec key, ITEMLEN keylen, bytevec val, ITEMLEN vallen);
+int brt_lookup (BRT brt, bytevec key, ITEMLEN keylen, bytevec*val, ITEMLEN *vallen);
+int close_brt (BRT);
+int dump_brt (BRT brt);
+void brt_fsync (BRT); /* fsync, but don't clear the caches. */
+
+void brt_flush (BRT); /* fsync and clear the caches. */ 
+
+int brt_create_cachetable (CACHETABLE *t, int n_cachlines /* Pass 0 if you want the default. */);
+
+extern int brt_debug_mode;
+int verify_brt (BRT brt);
+
+int show_brt_blocknumbers(BRT);
+
+typedef struct brt_cursor *BRT_CURSOR;
+int brt_cursor (BRT, BRT_CURSOR*);
+int brt_c_get (BRT_CURSOR cursor, DBT *kbt, DBT *vbt, int brtc_flags);
+int brt_cursor_close (BRT_CURSOR curs);
+
+#endif

newbrt/brttypes.h

+#ifndef BRTTYPES_H
+#define BRTTYPES_H
+#define _XOPEN_SOURCE 500
+#define _FILE_OFFSET_BITS 64
+typedef unsigned int ITEMLEN;
+typedef const void *bytevec;
+//typedef const void *bytevec;
+#endif

newbrt/cachetable-test.c

+#include "memory.h"
+#include "cachetable.h"
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+struct item {
+    CACHEKEY key;
+    char *something;
+};
+
+int expect_n_flushes=0;
+CACHEKEY flushes[100];
+
+static void expect1(CACHEKEY key) {
+    expect_n_flushes=1;
+    flushes[0]=key;
+}
+static void expectN(CACHEKEY key) {
+    flushes[expect_n_flushes++]=key;
+}
+
+CACHEFILE expect_f;
+
+static void flush (CACHEFILE f, CACHEKEY key, void*value, int write_me __attribute__((__unused__)), int keep_mee __attribute__((__unused__))) {
+    struct item *it = value;
+    int i;
+
+    printf("Flushing %lld (it=>key=%lld)\n", key, it->key);
+
+    assert(expect_f==f);
+    assert(strcmp(it->something,"something")==0);
+    assert(it->key==key);
+
+    /* Verify that we expected the flush. */
+    for (i=0; i<expect_n_flushes; i++) {
+	if (key==flushes[i]) {
+	    flushes[i] = flushes[expect_n_flushes-1];
+	    expect_n_flushes--;
+	    goto found_flush;
+	}
+    }
+    printf("%lld was flushed, but I didn't expect it\n", key);
+    abort();
+ found_flush:
+    my_free(value);
+}
+
+struct item *make_item (CACHEKEY key) {
+    struct item *MALLOC(it);
+    it->key=key;
+    it->something="something";
+    return it;
+}
+
+CACHEKEY did_fetch=-1;
+int fetch (CACHEFILE f, CACHEKEY key, void**value, void*extraargs) {
+    printf("Fetch %lld\n", key);
+    assert (expect_f==f);
+    assert((long)extraargs==23);
+    *value = make_item(key);
+    did_fetch=key;
+    return 0;
+}
+
+
+void test0 (void) {
+    void* t3=(void*)23;
+    CACHETABLE t;
+    CACHEFILE f;
+    int r;
+    char fname[] = "test.dat";
+    r=create_cachetable(&t, 5);
+    assert(r==0);
+    unlink(fname);
+    r = cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, 0777);
+    assert(r==0);
+    expect_f = f;
+
+    expect_n_flushes=0;
+    r=cachetable_put(f, 1, make_item(1), flush, fetch, t3);   /* 1P */        /* this is the lru list.  1 is pinned. */
+    assert(r==0);
+    assert(expect_n_flushes==0);
+
+    expect_n_flushes=0;
+    r=cachetable_put(f, 2, make_item(2), flush, fetch, t3);
+    assert(r==0);
+    r=cachetable_unpin(f, 2, 1);           /* 2U 1P */
+    assert(expect_n_flushes==0);
+
+    expect_n_flushes=0;
+    r=cachetable_put(f, 3, make_item(3), flush, fetch, t3);
+    assert(r==0);
+    assert(expect_n_flushes==0);            /* 3P 2U 1P */   /* 3 is most recently used (pinned), 2 is next (unpinned), 1 is least recent (pinned) */
+
+    expect_n_flushes=0;
+    r=cachetable_put(f, 4, make_item(4), flush, fetch, t3);
+    assert(r==0);
+    assert(expect_n_flushes==0);            /* 4P 3P 2U 1P */
+
+    expect_n_flushes=0;
+    r=cachetable_put(f, 5, make_item(5), flush, fetch, t3);
+    assert(r==0);
+    r=cachetable_unpin(f, 5, 1);
+    assert(r==0);
+    r=cachetable_unpin(f, 3, 1);
+    assert(r==0);
+    assert(expect_n_flushes==0);            /* 5U 4P 3U 2U 1P */
+
+    expect1(2); /* 2 is the oldest unpinned item. */
+    r=cachetable_put(f, 6, make_item(6), flush, fetch, t3);   /* 6P 5U 4P 3U 1P */
+    assert(r==0);
+    assert(expect_n_flushes==0);
+
+
+    expect1(3);
+    r=cachetable_put(f, 7, make_item(7), flush, fetch, t3);
+    assert(r==0);
+    assert(expect_n_flushes==0);
+    r=cachetable_unpin(f, 7, 1);           /* 7U 6P 5U 4P 1P */
+    assert(r==0);
+
+    {
+	void *item_v=0;
+	expect_n_flushes=0;
+	r=cachetable_get_and_pin(f, 5, &item_v, flush, fetch, t3);  /* 5P 7U 6P 4P 1P */
+	assert(r==0);
+	assert(((struct item *)item_v)->key==5);
+	assert(strcmp(((struct item *)item_v)->something,"something")==0);
+	assert(expect_n_flushes==0);
+    }
+
+    {
+	void *item_v=0;
+	r=cachetable_unpin(f, 4, 1);
+	assert(r==0);
+	expect1(4);
+	did_fetch=-1;
+	r=cachetable_get_and_pin(f, 2, &item_v, flush, fetch, t3);  /* 2p 5P 7U 6P 1P */
+	assert(r==0);
+	assert(did_fetch==2); /* Expect that 2 is fetched in. */
+	assert(((struct item *)item_v)->key==2);
+	assert(strcmp(((struct item *)item_v)->something,"something")==0);
+	assert(expect_n_flushes==0);
+    }
+	
+    r=cachetable_unpin(f, 2, 1);
+    assert(r==0);
+    r=cachetable_unpin(f ,5, 1);
+    assert(r==0);
+    r=cachetable_unpin(f, 6, 1);
+    assert(r==0);
+    r=cachetable_unpin(f, 1, 1);
+    assert(r==0);
+    r=cachetable_assert_all_unpinned(t);
+    assert(r==0);
+
+    printf("Closing\n");
+    expect1(2);
+    expectN(5);
+    expectN(7);
+    expectN(6);
+    expectN(1);
+    r=cachefile_close(f);
+    assert(r==0);
+    r=cachetable_close(t);
+    assert(r==0);
+    assert(expect_n_flushes==0);
+    expect_f = 0; 
+    memory_check_all_free();
+}
+
+static void flush_n (CACHEFILE f __attribute__((__unused__)), CACHEKEY key __attribute__((__unused__)), void *value, int write_me __attribute__((__unused__)), int keep_me __attribute__((__unused__))) {
+    int *v = value;
+    assert(*v==0);
+}
+static int fetch_n (CACHEFILE f __attribute__((__unused__)), CACHEKEY key __attribute__((__unused__)), void**value, void*extraargs) {
+    assert((long)extraargs==42);
+    *value=0;
+    return 0;
+}
+
+
+void test_nested_pin (void) {
+    void *f2=(void*)42;
+    CACHETABLE t;
+    CACHEFILE f;
+    int i0, i1;
+    int r;
+    void *vv;
+    char fname[] = "test.dat";
+    r = create_cachetable(&t, 1);
+    assert(r==0);
+    unlink(fname);
+    r = cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, 0777);
+    assert(r==0);
+    expect_f = f;
+
+    i0=0; i1=0;
+    r = cachetable_put(f, 1, &i0, flush_n, fetch_n, f2);
+    assert(r==0);
+    r = cachetable_get_and_pin(f, 1, &vv, flush_n, fetch_n, f2);
+    assert(r==0);
+    assert(vv==&i0);
+    assert(i0==0);
+    r = cachetable_unpin(f, 1, 0);
+    assert(r==0);
+    r = cachetable_put(f, 2, &i1, flush_n, fetch_n, f2);
+    assert(r!=0); // previously pinned, we shouldn't be able to put.
+    r = cachetable_unpin(f, 1, 0);
+    assert(r==0);
+    r = cachetable_put(f, 2, &i1, flush_n, fetch_n, f2);
+    assert(r==0); // now it is unpinned, we can put it.
+    
+}
+
+
+void null_flush (CACHEFILE cf  __attribute__((__unused__)),
+		 CACHEKEY k    __attribute__((__unused__)),
+		 void *v       __attribute__((__unused__)),
+		 int write_me  __attribute__((__unused__)),
+		 int keep_me   __attribute__((__unused__))) {
+}
+int add123_fetch (CACHEFILE cf __attribute__((__unused__)), CACHEKEY key, void **value, void*extraargs) {
+    assert((long)extraargs==123);
+    *value = (void*)((unsigned long)key+123L);
+    return 0;
+}
+int add222_fetch (CACHEFILE cf __attribute__((__unused__)), CACHEKEY key, void **value, void*extraargs) {
+    assert((long)extraargs==222);
+    *value = (void*)((unsigned long)key+222L);
+    return 0;
+}
+
+
+void test_multi_filehandles (void) {
+    CACHETABLE t;
+    CACHEFILE f1,f2,f3;
+    char fname1[]="test.dat";
+    char fname2[]="test2.dat";
+    char fname3[]="test3.dat";
+    int r;
+    void *v;
+    unlink(fname1);
+    unlink(fname2);
+
+    r = create_cachetable(&t, 4);                                 assert(r==0);
+    r = cachetable_openf(&f1, t, fname1, O_RDWR|O_CREAT, 0777);   assert(r==0);
+    r = link(fname1, fname2);                                     assert(r==0);
+    r = cachetable_openf(&f2, t, fname2, O_RDWR|O_CREAT, 0777);   assert(r==0);
+    r = cachetable_openf(&f3, t, fname3, O_RDWR|O_CREAT, 0777);   assert(r==0);
+
+    assert(f1==f2);
+    assert(f1!=f3);
+    
+    r = cachetable_put(f1, 1, (void*)124, null_flush, add123_fetch, (void*)123); assert(r==0);
+    r = cachetable_get_and_pin(f2, 1, &v, null_flush, add123_fetch, (void*)123); assert(r==0);
+    assert((unsigned long)v==124);
+    r = cachetable_get_and_pin(f2, 2, &v, null_flush, add123_fetch, (void*)123); assert(r==0);
+    assert((unsigned long)v==125);
+    r = cachetable_get_and_pin(f3, 2, &v, null_flush, add222_fetch, (void*)222); assert(r==0);
+    assert((unsigned long)v==224);
+    r = cachetable_maybe_get_and_pin(f1, 2, &v); assert(r==0);
+    assert((unsigned long)v==125);
+    
+}
+
+int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
+    test0();
+    test_nested_pin();
+    test_multi_filehandles ();
+    printf("ok\n");
+    return 0;
+}

newbrt/cachetable.h

+#ifndef CACHETABLE_H
+#define CACHETABLE_H
+#include <fcntl.h>
+
+/* Implement the cache table. */
+
+typedef long long CACHEKEY;
+typedef struct cachetable *CACHETABLE;
+typedef struct cachefile *CACHEFILE;
+
+/* Maintain a cache mapping from cachekeys to values (void*)
+ * Some of the keys can be pinned.  Don't pin too many or for too long.
+ * If the cachetable is too full, it will call the flush_callback() function with the key, the value, and the otherargs
+   and then remove the key-value pair from the cache.
+ * The callback won't be any of the currently pinned keys.
+ * Also when flushing an object, the cachetable drops all references to it,
+ * so you may need to free() it.
+ * Note: The cachetable should use a common pool of memory, flushing things across cachetables.
+ *  (The first implementation doesn't)
+ * If you pin something twice, you must unpin it twice.
+ */
+int create_cachetable (CACHETABLE */*result*/, int /*n_entries*/);
+
+int cachetable_openf (CACHEFILE *,CACHETABLE, const char */*fname*/, int flags, mode_t mode);
+
+/* Error if already present.  On success, pin the value. */
+int cachetable_put (CACHEFILE, CACHEKEY, void*/*value*/,
+		    void(*flush_callback)(CACHEFILE, CACHEKEY key, void*value, int write_me, int keep_me),
+		    int(*fetch_callback)(CACHEFILE, CACHEKEY key, void**value,void*extraargs), /* If we are asked to fetch something, get it by calling this back. */
+		    void*extraargs
+		    );
+
+int cachetable_get_and_pin (CACHEFILE, CACHEKEY, void**/*value*/,
+			    void(*flush_callback)(CACHEFILE,CACHEKEY,void*,int write_me, int keep_me),
+			    int(*fetch_callback)(CACHEFILE, CACHEKEY key, void**value,void*extraargs), /* If we are asked to fetch something, get it by calling this back. */
+			    void*extraargs
+			    );
+
+/* If the the item is already in memory, then return 0 and store it in the void**.
+ * If the item is not in memory, then return nonzero. */
+int cachetable_maybe_get_and_pin (CACHEFILE, CACHEKEY, void**);
+int cachetable_unpin (CACHEFILE, CACHEKEY, int dirty); /* Note whether it is dirty when we unpin it. */
+int cachetable_remove (CACHEFILE, CACHEKEY, int /*write_me*/); /* Removing something already present is OK. */
+int cachetable_assert_all_unpinned (CACHETABLE);
+int cachefile_assert_all_unpinned (CACHEFILE);
+
+//int cachetable_fsync_all (CACHETABLE); /* Flush everything to disk, but keep it in cache. */
+int cachetable_close (CACHETABLE); /* Flushes everything to disk, and destroys the cachetable. */
+
+int cachefile_close (CACHEFILE);
+//int cachefile_flush (CACHEFILE); /* Flush everything related to the VOID* to disk and free all memory.  Don't destroy the cachetable. */
+
+// Return on success (different from pread and pwrite)
+//int cachefile_pwrite (CACHEFILE, const void *buf, size_t count, off_t offset);
+//int cachefile_pread  (CACHEFILE, void *buf, size_t count, off_t offset);
+
+int cachefile_fd (CACHEFILE);
+
+#endif

newbrt/hashtable.c

+/* Hash table with chaining. */
+#include "hashtable.h"
+#include "memory.h"
+#include "../include/ydb-constants.h"
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "key.h"
+#include "yerror.h"
+
+int hashtable_create (HASHTABLE *h) {
+    HASHTABLE MALLOC(tab);
+    int i;
+    if (tab==0) return -1;
+    tab->n_keys=0;
+    tab->arraysize=128;
+    assert(sizeof(*tab->array)==sizeof(void*));
+    tab->array = my_calloc(tab->arraysize, sizeof(*tab->array));
+    for (i=0; i<tab->arraysize; i++) tab->array[i]=0;
+    *h=tab;
+    return 0;
+}
+
+static unsigned int hash_key (const char *key, ITEMLEN keylen) {
+    /* From Sedgewick.  There are probably better hash functions. */
+    unsigned int b    = 378551;
+    unsigned int a    = 63689;
+    unsigned int hash = 0;
+    ITEMLEN i;
+    for (i = 0; i < keylen; i++ ) {
+	hash = hash * a + key[i];
+	a *= b;
+    }
+    return hash;
+}
+
+static void hash_find_internal (HASHTABLE tab, const char *key, ITEMLEN keylen, HASHELT *hashelt, HASHELT **prev_ptr) {
+    unsigned int h = hash_key (key, keylen) % tab->arraysize;
+    HASHELT he;
+    HASHELT *prev = &tab->array[h];
+    for (he=*prev; he; prev=&he->next, he=*prev) {
+	if (keylen==he->keylen && memcmp(key, he->key, keylen)==0) {
+	    *prev_ptr = prev;
+	    *hashelt = he;
+	    return;
+	}
+    }
+    *prev_ptr = prev;
+    *hashelt = 0;
+}
+    
+int hash_find (HASHTABLE tab, bytevec key, ITEMLEN keylen, bytevec *data, ITEMLEN *datalen) {
+    HASHELT he, *prev_ptr;
+    hash_find_internal(tab, key, keylen, &he, &prev_ptr);
+    if (he==0) {
+	return -1;
+    } else {
+	*data = he->val;
+	*datalen = he->vallen;
+	return 0;
+    }
+}
+
+
+int hash_insert (HASHTABLE tab, const char *key, ITEMLEN keylen, const char *val, ITEMLEN vallen)
+{
+    unsigned int h = hash_key (key,keylen)%tab->arraysize;
+    {
+	HASHELT he,*prev_ptr;
+	hash_find_internal(tab, key, keylen, &he, &prev_ptr);
+	if (he!=0) {
+	    return BRT_ALREADY_THERE;
+	}
+    }
+    {
+	/* Otherwise the key is not already present, so we need to add it. */
+	HASHELT MALLOC(he);
+	he->key = memdup(key, keylen);
+	he->keylen = keylen;
+	he->val = memdup(val, vallen);
+	he->vallen = vallen;
+	he->next = tab->array[h];
+	tab->array[h]=he;
+	tab->n_keys++;
+	if (tab->n_keys > tab->arraysize) {
+	    int newarraysize = tab->arraysize*2;
+	    HASHELT *newarray = my_calloc(newarraysize, sizeof(*tab->array));
+	    int i;
+	    assert(newarray!=0);
+	    for (i=0; i<newarraysize; i++) newarray[i]=0;
+	    for (i=0; i<tab->arraysize; i++) {
+		while ((he=tab->array[i])!=0) {
+		    h = hash_key(he->key, he->keylen)%newarraysize;
+		    tab->array[i] = he->next;
+		    he->next = newarray[h];
+		    newarray[h] = he;
+		}
+	    }
+	    my_free(tab->array);
+	   // printf("Freed\n");
+	    tab->array=newarray;
+	    tab->arraysize=newarraysize;
+	    //printf("Done growing\n");
+	}
+	return BRT_OK;
+    }
+}
+
+int hash_delete (HASHTABLE tab, const char *key, ITEMLEN keylen) {
+    HASHELT he, *prev_ptr;
+    //printf("%s:%d deleting %s (bucket %d)\n", __FILE__, __LINE__, key, hash_key(key,keylen)%tab->arraysize);
+    hash_find_internal(tab, key, keylen, &he, &prev_ptr);
+    if (he==0) return DB_NOTFOUND;
+    else {
+	//printf("%s:%d deleting %s %s\n", __FILE__, __LINE__, he->key, he->val);
+	assert(*prev_ptr==he);
+	*prev_ptr = he->next;
+	//printf("Freeing %s %s\n", he->key, he->val);
+	my_free(he->key);
+	my_free(he->val);
+	my_free(he);
+	tab->n_keys--;
+	return BRT_OK;
+    }
+}
+
+
+int hashtable_random_pick(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen) {
+    int i;
+    for (i=0; i<h->arraysize; i++) {
+	HASHELT he=h->array[i];
+	if (he) {
+	    *key = he->key;
+	    *keylen = he->keylen;
+	    *data = he->val;
+	    *datalen = he->vallen;
+	    return 0;
+	}
+    }
+    return -1;
+}
+
+#if 0
+int hashtable_find_last(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen) {
+    bytevec best_k=0, best_d;
+    ITEMLEN best_kl, best_dl;
+    HASHTABLE_ITERATE(h, this_k, this_kl, this_d, this_dl,
+		      ({
+			  if (best_k==0 || keycompare(best_k, best_kl, this_k, this_kl)<0) {
+			      best_k  = this_k;
+			      best_kl = this_kl;
+			      best_d  = this_d;
+			      best_dl = this_dl;
+			  }
+		      }));
+    if (best_k) {
+	*key     = best_k;
+	*keylen  = best_kl;
+	*data    = best_d;
+	*datalen = best_dl;
+	return 0;
+    } else {
+	return -1;
+    }
+}
+#endif
+
+void hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen, void*args), void* args) {
+  /*
+    int i;
+    for (i=0; i<tab->arraysize; i++) {
+	HASHELT he;
+	for (he=tab->array[i]; he; he=he->next) {
+	    f(he->key, he->keylen, he->val, he->vallen, args);
+	}
+    }
+  */
+    HASHTABLE_ITERATE(tab, key, keylen, val, vallen, f(key,keylen,val,vallen,args));
+}
+
+int hashtable_n_entries(HASHTABLE tab) {
+    return tab->n_keys;
+}
+
+/* Frees the list, but doesn't free the keys. */
+static void hasheltlist_free (HASHELT elt) {
+    if (elt==0) return;
+    else {
+	hasheltlist_free(elt->next);
+	my_free(elt->key);
+	my_free(elt->val);
+	my_free(elt);
+    }
+}
+
+/* Frees the table, but doesn't do anything to the contents of the table.   The keys are still alloc'd.  The internal storage of the hashtable is freed. */
+void hashtable_free(HASHTABLE *tab) {
+    //printf("%s:%d free hashtable %p\n", __FILE__, __LINE__, tab);
+    hashtable_clear(*tab);
+    //printf("%s:%d free %p\n", __FILE__, __LINE__, tab);n
+    my_free((*tab)->array);
+    my_free(*tab);
+    *tab=0;
+}
+
+
+void hashtable_clear(HASHTABLE tab) {
+    int i;
+    for (i=0; i<tab->arraysize; i++) {
+	hasheltlist_free(tab->array[i]);
+	tab->array[i]=0;
+    }
+    tab->n_keys = 0;
+}

newbrt/hashtable.h

+#ifndef HASHTABLE_H
+#define HASHTABLE_H
+
+#include "brttypes.h"
+/* Hash table with chaining. */
+/* The keys and values are byte sequences. */
+/* The keys and values are malloc'd by the hashtable. */
+   
+typedef struct hashtable *HASHTABLE;
+
+int hashtable_create (HASHTABLE*);
+
+/* Return 0 if the key is found in the hashtable, -1 otherwise. */
+/* Warning: The data returned points to the internals of the hashtable.  It is set to "const" to try to prevent you from messing it up. */
+int hash_find (HASHTABLE tab, bytevec key, ITEMLEN keylen, bytevec*data, ITEMLEN *datalen);
+
+/* Replace the key if it was already there. */
+int hash_insert (HASHTABLE tab, const char *key, ITEMLEN keylen, const char *data, ITEMLEN datalen);
+
+/* It is OK to delete something that isn't there. */
+int hash_delete (HASHTABLE tab, const char *key, ITEMLEN keylen);
+void hashtable_free(HASHTABLE *tab);
+int hashtable_n_entries(HASHTABLE);
+  
+void hashtable_clear(HASHTABLE);
+
+int hashtable_random_pick(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
+//int hashtable_find_last(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
+
+typedef struct hashelt *HASHELT;
+struct hashelt {
+    char *key; ITEMLEN keylen;  /* key is NULL for empty elements */
+    char *val; ITEMLEN vallen;
+    HASHELT next;
+};
+
+struct hashtable {
+    int n_keys;
+    int arraysize;
+    HASHELT *array;
+};
+
+/* You cannot add or delete elements from the hashtable while iterating. */
+void hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen,void*), void*);
+// If you don't want to use something, do something like use "key __attribute__((__unused__))" for keyvar.
+#define HASHTABLE_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,body) ({ \
+  int hi_counter;                                                            \
+  for (hi_counter=0; hi_counter<table->arraysize; hi_counter++) {            \
+    HASHELT hi_he;                                                           \
+    for (hi_he=table->array[hi_counter]; hi_he; hi_he=hi_he->next) {           \
+      const char *keyvar     = hi_he->key;                                   \
+      ITEMLEN     keylenvar  = hi_he->keylen;                                \
+      const char *datavar    = hi_he->val;                                   \
+      ITEMLEN     datalenvar = hi_he->vallen;                                \
+      body;                                                                  \
+  }}})
+
+#endif

newbrt/hashtest.c

+#include "key.h"
+#include "hashtable.h"
+#include <stdlib.h>
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+void verify_hash_instance (bytevec kv_v, ITEMLEN kl, bytevec dv_v, ITEMLEN dl,
+			    int N, int *data, char *saw) {
+    char *kv = (char*)kv_v;
+    char *dv = (char*)dv_v;
+    int num, k;
+    assert(kv[0]=='k');
+    assert(dv[0]=='d');
+    assert(strcmp(kv+1, dv+1)==0);
+    assert(strlen(kv)+1==kl);
+    assert(strlen(dv)+1==dl);
+    num = atoi(kv+1);
+    for (k=0; k<N; k++) {
+	if (data[k]==num) {
+	    assert(!saw[k]);
+	    saw[k]=1;
+	    return;
+	}
+    }
+    fprintf(stderr, "%s isn't there\n", kv); abort();
+}
+
+void verify_htable_instance (bytevec kv_v, ITEMLEN kl, bytevec dv_v, ITEMLEN dl,
+			    int N, int *data, char *saw) {
+    char *kv = (char*)kv_v;
+    char *dv = (char*)dv_v;
+    int num, k;
+    assert(kv[0]=='k');
+    assert(dv[0]=='d');
+    assert(strcmp(kv+1, dv+1)==0);
+    assert(strlen(kv)+1==kl);
+    assert(strlen(dv)+1==dl);
+    num = atoi(kv+1);
+    for (k=0; k<N; k++) {
+	if (data[k]==num) {
+	    assert(!saw[k]);
+	    saw[k]=1;
+	    return;
+	}
+    }
+    fprintf(stderr, "%s isn't there\n", kv); abort();
+}
+
+void verify_htable (HASHTABLE htable, int N, int *data, char *saw) {
+    int j;
+    for (j=0; j<N; j++) {
+	saw[j]=0;
+    }
+    HASHTABLE_ITERATE(htable, kv, kl, dv, dl,
+		      verify_htable_instance (kv, kl, dv, dl,
+					      N, data, saw));
+    for (j=0; j<N; j++) {
+	assert(saw[j]);
+    }
+}
+
+void test0 (void) {
+    int r, i, j;
+    HASHTABLE htable;
+    int n_ops=1000;
+    int *data=malloc(sizeof(*data)*n_ops);
+    char*saw =malloc(sizeof(*saw)*n_ops);
+    int  data_n = 0;
+    assert(data!=0);
+    r = hashtable_create(&htable); assert(r==0);
+    assert(hashtable_n_entries(htable)==0);
+#if 0
+    {
+	bytevec kv=(void*)0xdeadbeef;
+	bytevec dv=(void*)0xbeefdead;
+	ITEMLEN kl=42, dl=43;
+	r = mdict_find_last(htable,&kv,&kl,&dv,&dl);
+	assert(r!=0);
+	assert((unsigned long)kv==0xdeadbeef);
+	assert((unsigned long)dv==0xbeefdead);
+	assert(kl==42);
+	assert(dl==43);
+    }
+#endif
+    for (i=0; i<n_ops; i++) {
+	if (random()%4==1) {
+	    // Delete something random
+	} else if (random()%2 == 0) {
+	    // Insert something
+	try_another_random:
+	    {
+		int ra = random()%(1<<30);
+		char kv[100], dv[100];
+		for (j=0; j<data_n; j++) {
+		    if (ra==data[j]) goto try_another_random;
+		}
+		snprintf(kv, 99, "k%d", ra);
+		snprintf(dv, 99, "d%d", ra);
+		hash_insert(htable, kv, strlen(kv)+1, dv, strlen(dv)+1);
+		data[data_n++]=ra;
+	    }
+	} else {
+	    // Look up something
+	}
+	verify_htable(htable, data_n, data, saw);
+    }
+}
+
+int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
+    test0();
+    return 0;
+}

newbrt/header-io.c

+#include "brttypes.h"
+#include "brt-internal.h"
+#include "memory.h"
+#include <sys/types.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+
+int read_sint (int fd, int *result) {
+    unsigned char b[4];
+    int r = read(fd, b, 4);
+    if (r!=4) return 1;
+    *result = (b[0]<<24) | (b[1]<<16) | (b[2]<<8) | (b[3]<<0);
+    return 0;
+}
+int read_uint (int fd, unsigned int *result) {
+    int sresult;
+    int r = read_sint(fd, &sresult);
+    if (r==0) { *result = r; }
+    return r;
+}
+
+int write_int (int fd, unsigned int v) {
+    unsigned char b[4];
+    int r;
+    b[0] = (v>>24)&0xff;
+    b[1] = (v>>16)&0xff;
+    b[2] = (v>>8)&0xff;
+    b[3] = (v>>0)&0xff;
+    r = write(fd, b, 4);
+    if (r!=4) return 1;
+    return 0;
+}
+
+int read_diskoff (int fd, diskoff *result) {
+    unsigned int i0,i1;
+    int r;
+    r = read_uint(fd, &i0);  if(r!=0) return r;
+    r = read_uint(fd, &i1);  if(r!=0) return r;
+    *result = ((unsigned long long)i0)<<32 | ((unsigned long long)i1);
+    return 0;
+}
+
+int write_diskoff (int fd, diskoff v) {
+    int r;
+    r = write_int(fd, (unsigned int)(v>>32));        if (r!=0) return r;
+    r = write_int(fd, (unsigned int)(v&0xffffffff)); if (r!=0) return r;
+    return 0;
+}
+
+int read_bytes (int fd, int l, char *s) {
+    int r = read(fd, s, l);
+    if (r==l) return 0;
+    return -1;
+}
+
+int write_bytes (int fd, int l, char *s) {
+    int r= write(fd, s, l);
+    if (r==l) return 0;
+    return -1;
+}
+
+int read_brt_header (int fd, struct brt_header *header) {
+    {
+	off_t r = lseek(fd, 0, SEEK_SET);
+	assert(r==0);
+    }
+    /* Ignore magic for now.  We'll need some magic at the beginning of the file. */
+    {
+	int r;
+	r = read_uint(fd, &header->nodesize);
+	if (r!=0) return -1;
+	r = read_diskoff(fd, &header->freelist); assert(r==0); /* These asserts should do something smarter. */
+	r = read_diskoff(fd, &header->unused_memory); assert(r==0);
+	r = read_sint(fd, &header->n_named_roots); assert(r==0);
+	if (header->n_named_roots>0) {
+	    int i;
+	    header->unnamed_root = -1;
+	    MALLOC_N(header->n_named_roots, header->names);
+	    MALLOC_N(header->n_named_roots, header->roots);
+	    for (i=0; i<header->n_named_roots; i++) {
+		unsigned int l;
+		char *s;
+		r = read_diskoff(fd, &header->roots[i]); assert(r==0);
+		r = read_uint(fd, &l); assert(r==0); /* count includes the trailing null. */
+		MALLOC_N(l, s);
+		r = read_bytes(fd, l, s); assert(r==0);
+		assert(l>0 && s[l-1]==0);
+		header->names[i] = s;
+	    }
+	} else {
+	    r = read_diskoff(fd, &header->unnamed_root); assert(r==0);
+	    header->names = 0;
+	    header->roots = 0;
+	}
+    }
+    return 0;
+}
+
+int read_brt_h_unused_memory (int fd, diskoff *unused_memory) {
+    off_t r = lseek(fd, 12, SEEK_SET);
+    assert(r==12);
+    r = read_diskoff(fd, unused_memory);
+    return r;
+}
+
+int write_brt_h_unused_memory (int fd, diskoff unused_memory) {
+    off_t r = lseek(fd, 12, SEEK_SET);
+    assert(r==12);
+    r = write_diskoff(fd, unused_memory);
+    return r;
+}

newbrt/key.c

+#include "brt-internal.h"
+#include <assert.h>
+#include <string.h>
+
+int keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len) {
+    if (key1len==key2len) {
+	return memcmp(key1,key2,key1len);
+    } else if (key1len<key2len) {
+	int r = memcmp(key1,key2,key1len);
+	if (r<=0) return -1; /* If the keys are the same up to 1's length, then return -1, since key1 is shorter than key2. */
+	else return 1;
+    } else {
+	return -keycompare(key2,key2len,key1,key1len);
+    }
+}
+
+void test_keycompare (void) {
+    assert(keycompare("a",1, "a",1)==0);
+    assert(keycompare("aa",2, "a",1)>0);
+    assert(keycompare("a",1, "aa",2)<0);
+    assert(keycompare("b",1, "aa",2)>0);
+    assert(keycompare("aa",2, "b",1)<0);
+    assert(keycompare("aaaba",5, "aaaba",5)==0);
+    assert(keycompare("aaaba",5, "aaaaa",5)>0);
+    assert(keycompare("aaaaa",5, "aaaba",5)<0);
+    assert(keycompare("aaaaa",3, "aaaba",3)==0);
+}

newbrt/key.h

+#include "brttypes.h"
+
+int keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
+void test_keycompare (void) ;
+

newbrt/mdict-test.c

+#include "mdict.h"
+#include <stdlib.h>
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+void verify_mdict_instance (bytevec kv_v, ITEMLEN kl, bytevec dv_v, ITEMLEN dl,
+			    int N, int *data, char *saw) {
+    char *kv = (char*)kv_v;
+    char *dv = (char*)dv_v;
+    int num, k;
+    assert(kv[0]=='k');
+    assert(dv[0]=='d');
+    assert(strcmp(kv+1, dv+1)==0);
+    assert(strlen(kv)+1==kl);
+    assert(strlen(dv)+1==dl);
+    num = atoi(kv+1);
+    for (k=0; k<N; k++) {
+	if (data[k]==num) {
+	    assert(!saw[k]);
+	    saw[k]=1;
+	    return;
+	}
+    }
+    fprintf(stderr, "%s isn't there\n", kv); abort();
+}
+
+void verify_mdict (MDICT mdict, int N, int *data, char *saw) {
+    int j;
+    for (j=0; j<N; j++) {
+	saw[j]=0;
+    }
+    MDICT_ITERATE(mdict, kv, kl, dv, dl,
+		  verify_mdict_instance (kv, kl, dv, dl,
+					 N, data, saw));
+    for (j=0; j<N; j++) {
+	assert(saw[j]);
+    }
+}
+
+void test0 (void) {
+    int r, i, j;
+    MDICT mdict;
+    int n_ops=1000;
+    int *data=malloc(sizeof(*data)*n_ops);
+    char*saw =malloc(sizeof(*saw)*n_ops);
+    int  data_n = 0;
+    assert(data!=0);
+    r = mdict_create(&mdict); assert(r==0);
+    assert(mdict_n_entries(mdict)==0);
+    {
+	bytevec kv=(void*)0xdeadbeef;
+	bytevec dv=(void*)0xbeefdead;
+	ITEMLEN kl=42, dl=43;
+	r = mdict_find_last(mdict,&kv,&kl,&dv,&dl);
+	assert(r!=0);
+	assert((unsigned long)kv==0xdeadbeef);
+	assert((unsigned long)dv==0xbeefdead);
+	assert(kl==42);
+	assert(dl==43);
+    }
+    for (i=0; i<n_ops; i++) {
+	if (random()%4==1) {
+	    // Delete something random
+	} else if (random()%2 == 0) {
+	    // Insert something
+	try_another_random:
+	    {
+		int ra = random()%(1<<30);
+		char kv[100], dv[100];
+		for (j=0; j<data_n; j++) {
+		    if (ra==data[j]) goto try_another_random;
+		}
+		snprintf(kv, 99, "k%d", ra);
+		snprintf(dv, 99, "d%d", ra);
+		mdict_insert(mdict, kv, strlen(kv)+1, dv, strlen(dv)+1);
+		data[data_n++]=ra;
+	    }
+	} else {
+	    // Look up something
+	}
+	verify_mdict(mdict, data_n, data, saw);
+    }
+}
+
+int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
+    test0();
+    return 0;
+}

newbrt/mdict.c

+#include "mdict.h"
+#include "memory.h"
+#define USEPMA
+#ifdef USEPMA
+#include "pma.h"
+
+struct mdict {
+    PMA pma;
+};
+
+int mdict_create (MDICT* mdict) {
+    MDICT result;
+    int r;
+    MALLOC(result);
+    if (result==0) return -1;
+    r = pma_create(&result->pma);
+    if (r==0) {
+	*mdict = result;
+    }
+    return r;
+}
+	
+void mdict_free (MDICT m) {
+    pma_free(m->pma);
+    my_free(m);
+}
+
+int  mdict_n_entries (MDICT m) {
+    return pma_n_entries(m->pma);
+}
+    
+
+/* Returns an error if the key is already present. */
+/* The values returned should not be modified. */
+/* May damage the cursor. */
+int mdict_insert (MDICT m, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen) {
+    return pma_insert(m->pma, key, keylen, data, datalen);
+}
+/* This returns an error if the key is NOT present. */
+int mdict_replace (MDICT, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
+/* This returns an error if the key is NOT present. */
+int mdict_delete (MDICT m, bytevec key, ITEMLEN keylen) {
+    return pma_delete(m->pma, key, keylen);
+}
+
+/* Exposes internals of the MDICT by returning a pointer to the guts.
+ * Don't modify the returned data.  Don't free it. */
+int mdict_lookup (MDICT m, bytevec key, ITEMLEN keylen, bytevec*data, ITEMLEN *datalen) {
+    return pma_lookup(m->pma, key, keylen, data, datalen);
+}
+
+
+int mdict_random_pick(MDICT m, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen) {
+    return pma_random_pick(m->pma, key, keylen, data, datalen);
+}
+
+void mdict_iterate (MDICT m, void(*f)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), void*v) {
+    pma_iterate(m->pma, f, v);
+}
+
+
+#else
+foo
+#endif

newbrt/mdict.h

+#ifndef MDICT_H
+#define MDICT_H
+
+#include "brttypes.h"
+
+//#define USEPMA
+#define USEHASH
+
+int keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
+
+#ifdef USEPMA
+
+#include "pma.h"
+#define MDICT PMA
+#define MDICT_OK PMA_OK
+#define MDICT_NOTFOUND PMA_NOTFOUND
+#define mdict_free pma_free
+#define mdict_n_entries pma_n_entries
+#define MDICT_ITERATE PMA_ITERATE
+#define mdict_insert pma_insert
+#define mdict_create pma_create
+#define mdict_delete pma_delete
+#define mdict_lookup pma_lookup
+#define mdict_random_pick pma_random_pick
+#define mdict_iterate pma_iterate
+
+#elif defined(USEHASH)
+#include "hashtable.h"
+#define MDICT HASHTABLE
+#define MDICT_OK 0
+#define MDICT_NOTFOUND -1
+#define MDICT_ALREADY_THERE -2
+#define mdict_free hashtable_free
+#define mdict_n_entries hashtable_n_entries
+#define MDICT_ITERATE HASHTABLE_ITERATE
+#define mdict_insert hash_insert
+#define mdict_create hashtable_create
+#define mdict_delete hash_delete
+#define mdict_lookup hash_find
+#define mdict_random_pick hashtable_random_pick
+#define mdict_iterate hashtable_iterate
+#define mdict_find_last hashtable_find_last
+
+#else
+/* In-memory dictionary.  */
+
+
+enum mdict_errors { MDICT_OK=0, MDICT_NOTFOUND = -1, MDICT_ALREADY_THERE = -2 };
+
+typedef struct mdict *MDICT;
+
+int mdict_create (MDICT*);
+void mdict_free (MDICT);
+
+int  mdict_n_entries (MDICT);
+
+/* Returns an error if the key is already present. */
+/* The values returned should not be modified. */
+/* May damage the cursor. */
+int mdict_insert (MDICT, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
+/* This returns an error if the key is NOT present. */
+int mdict_replace (MDICT, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
+/* This returns an error if the key is NOT present. */
+int mdict_delete (MDICT, bytevec key, ITEMLEN keylen);
+
+/* Exposes internals of the MDICT by returning a pointer to the guts.
+ * Don't modify the returned data.  Don't free it. */
+int mdict_lookup (MDICT, bytevec key, ITEMLEN keylen, bytevec*data, ITEMLEN *datalen);
+
+
+int mdict_random_pick(MDICT, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
+
+void mdict_iterate (MDICT, void(*)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), void*);
+
+#define MDICT_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,body) ({ \
+  void __do_iterate(bytevec keyvar, ITEMLEN keylenvar, bytevec datavar, ITEMLEN datalenvar, void *__ignore __attribute__((__unused__))) { \
+      body; \
+  } \
+  mdict_iterate(table,__do_iterate, 0);		\
+})
+
+#endif
+#endif

newbrt/memory.c

+#include "memory.h"
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+
+int memory_check=1;
+
+#define WHEN_MEM_DEBUG(x) ({if (memory_check) ({x});})
+
+
+long long n_items_malloced=0;
+
+/* Memory checking */
+enum { items_limit = 1000 };
+int overflowed=0;
+static void *items[items_limit];
+static long sizes[items_limit];
+
+void note_did_malloc (void *p, long size) {
+    WHEN_MEM_DEBUG(
+		   if (n_items_malloced<items_limit) { items[n_items_malloced]=p; sizes[n_items_malloced]=size; }
+		   else overflowed=1;
+		   //printf("%s:%d %p=malloc(%ld)\n", __FILE__, __LINE__, r, size);
+		   );
+    n_items_malloced++;
+}
+
+void note_did_free(void *p) {
+    WHEN_MEM_DEBUG(
+		   if (!overflowed) {
+		       int i;
+		       //printf("not overflowed\n");
+		       for (i=0; i<n_items_malloced; i++) {
+			   if (items[i]==p) {
+			       items[i]=items[n_items_malloced-1];
+			       sizes[i]=sizes[n_items_malloced-1];
+			       // printf("items[%d] replaced, now %p\n", i, items[i]); 
+			       goto ok;
+			   }
+		       }
+		       printf("%s:%d freed something (%p) not alloced\n", __FILE__, __LINE__, p);
+		       abort();
+		   ok:;
+		   }
+		   //printf("%s:%d free(%p)\n", __FILE__, __LINE__, p);
+		   );
+    n_items_malloced--;
+}
+
+
+//#define BUFFERED_MALLOC
+#ifdef BUFFERED_MALLOC
+
+enum { BUFFERING = 4096 };
+void mark_buffer (char *p, int size) {
+    unsigned int *pl = (unsigned int*)p;
+    int i;
+    for (i=0; i<BUFFERING/4; i++) {
+	pl[i] = 0xdeadbeef;
+    }
+    pl[BUFFERING/8] = size;
+}
+int check_buffer (char *p) {
+    unsigned int *pl = (unsigned int*)p;
+    int i;
+    for (i=0; i<BUFFERING/4; i++) {
+	if (i!=BUFFERING/8) {
+	    assert(pl[i] == 0xdeadbeef);
+	}
+    }
+    return pl[BUFFERING/8];
+}
+
+void check_all_buffers (void) {
+    int i;
+    if (!overflowed) {
+	for (i=0; i<n_items_malloced; i++) {
+	    int size = check_buffer(((char*)items[i])-BUFFERING);
+	    check_buffer(((char*)items[i])+size);
+	}
+    }
+}
+
+void *actual_malloc(long size) {
+    char *r = malloc(size+BUFFERING*2);
+    mark_buffer(r, size);
+    mark_buffer(r+size+BUFFERING, size);
+    check_all_buffers();
+    return r+BUFFERING;
+}
+
+void actual_free(void *pv) {
+    char *p = pv;
+    int size=check_buffer(p-BUFFERING);
+    check_buffer(p+size);
+    check_all_buffers();
+    //free(p-BUFFERING);
+}
+void *actual_realloc(void *pv, long size) {
+    check_all_buffers();
+    {
+	char *p = pv;
+	char *r = realloc(p-BUFFERING, size+BUFFERING*2);
+	mark_buffer(r, size);
+	mark_buffer(r+size+BUFFERING, size);
+	return r+BUFFERING;
+    }
+}
+void *actual_calloc (long nmemb, long size) {
+    return actual_malloc(nmemb*size);
+}
+
+void do_memory_check (void) {
+    check_all_buffers();
+}
+#else
+#define actual_malloc malloc
+#define actual_free free
+#define actual_realloc realloc
+#define actual_calloc calloc
+#endif
+
+
+void *my_calloc(long nmemb, long size) {
+    void *r;
+    errno=0;
+    r = actual_calloc(nmemb, size);
+    //printf("%s:%d calloc(%ld,%ld)->%p\n", __FILE__, __LINE__, nmemb, size, r);
+    note_did_malloc(r, nmemb*size);
+    //if ((long)r==0x80523f8) { printf("%s:%d %p\n", __FILE__, __LINE__, r);  }
+    return r;
+}
+void *my_malloc(long size) {
+    void * r;
+    errno=0;
+    r=actual_malloc(size);
+    //printf("%s:%d malloc(%ld)->%p\n", __FILE__, __LINE__, size,r);
+    note_did_malloc(r, size);
+    //if ((long)r==0x80523f8) { printf("%s:%d %p size=%ld\n", __FILE__, __LINE__, r, size);   }
+    return r;
+}
+void *tagmalloc(unsigned long size, int typtag) {
+    void *r = my_malloc(size);
+    assert(size>sizeof(int));
+    ((int*)r)[0] = typtag;
+    return r;
+}
+
+void *my_realloc(void *p, long size) {
+    void *newp;
+    note_did_free(p);
+    errno=0;
+    newp = actual_realloc(p, size);
+    //printf("%s:%d realloc(%p,%ld)-->%p\n", __FILE__, __LINE__, p, size, newp);
+    note_did_malloc(newp, size);
+    return newp;
+}
+
+void my_free(void* p) {
+    //printf("%s:%d free(%p)\n", __FILE__, __LINE__, p);
+    note_did_free(p);
+    actual_free(p);
+}
+
+void *memdup (const void *v, unsigned int len) {
+    void *r=my_malloc(len);
+    memcpy(r,v,len);
+    return r;
+}
+char *mystrdup (const char *s) {
+    return memdup(s, strlen(s)+1);
+}
+
+void memory_check_all_free (void) {
+    if (n_items_malloced>0) {
+	printf("n_items_malloced=%lld\n", n_items_malloced);
+	if (memory_check)
+	    printf(" one item is %p size=%ld\n", items[0], sizes[0]);
+    }
+    assert(n_items_malloced==0);
+}
+
+int get_n_items_malloced (void) { return n_items_malloced; }
+void print_malloced_items (void) {
+    int i;
+    for (i=0; i<n_items_malloced; i++) {
+	printf(" %p size=%ld\n", items[i], sizes[i]);
+    }
+}

newbrt/memory.h

+//#include <stdlib.h>
+
+/* errno is set to 0 or a value to indicate problems. */
+void *my_calloc(long nmemb, long size);
+void *my_malloc(long size);
+void *tagmalloc(unsigned long size, int typ);
+void my_free(void*);
+void *my_realloc(void *, long size);
+
+#define MALLOC(v) v = my_malloc(sizeof(*v))
+#define MALLOC_N(n,v) v = my_malloc((n)*sizeof(*v))
+
+#define TAGMALLOC(t,v) t v = tagmalloc(sizeof(*v), TYP_ ## t);
+
+void *memdup (const void *v, unsigned int len);
+char *mystrdup (const char *s);
+
+void memory_check_all_free (void);
+void do_memory_check(void);
+
+extern int memory_check; // Set to nonzero to get a (much) slower version of malloc that does (much) more checking.
+
+int get_n_items_malloced(void);
+void print_malloced_items(void);

newbrt/myassert.c

+#include "myassert.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef TESTER
+void my_assert(int a, const char *f, int l) {
+    if (!a) { fprintf(stderr, "Assertion failed at %s:%d\n", f, l); abort(); }
+}
+#endif

newbrt/myassert.h

+#ifndef MYASSERT_H
+#define MYASSERT_H
+
+#ifndef TESTER
+#include <assert.h>
+#else
+extern void my_assert(int, const char *, int);
+#define assert(x) my_assert(x, __FILE__, __LINE__)
+#endif
+
+#endif

newbrt/pma-internal.h

+#include "pma.h"
+
+struct pair {
+    bytevec key; /* NULL for empty slots */
+    int keylen;
+    bytevec val;
+    int vallen;
+};
+
+struct pma_cursor {
+    PMA pma;
+    int position; /* -1 if the position is undefined. */
+    PMA_CURSOR next,prev;
+    void *skey, *sval; /* used in dbts. */ 
+};
+
+struct pma {
+    enum typ_tag tag;
+    int N;               /* How long is the array? Always a power of two >= 4. */
+    int n_pairs_present; /* How many array elements are non-null.         */
+    struct pair *pairs;
+    int uplgN;           /* The smallest power of two >= lg(N)            */
+    double densitystep;  /* Each doubling decreases the density by densitystep.
+			  * For example if array_len=256 and uplgN=8 then there are 5 doublings.
+			  * Regions of size 8 are full.  Regions of size 16 are 90% full.
+			  *  Regions of size 32 are 80% full.  Regions of size 64 are 70% full.
+			  *  Regions of size 128 are 60% full.  Regions of size 256 are 50% full.
+			  *  The densitystep is 0.10. */
+    PMA_CURSOR cursors_head, cursors_tail;
+};
+
+int pmainternal_count_region (struct pair *pairs, int lo, int hi);
+void pmainternal_calculate_parameters (PMA pma);
+int pmainternal_smooth_region (struct pair *pairs, int n, int idx);
+int pmainternal_printpairs (struct pair *pairs, int N);
+int pmainternal_make_space_at (PMA pma, int idx);
+int pmainternal_find (PMA pma, bytevec key, int keylen);
+void print_pma (PMA pma); /* useful for debugging, so keep the name short. I.e., not pmainternal_print_pma() */

newbrt/ybt.h

+#ifndef YBT_H
+#define YBT_H
+
+// brttypes.h must be first to make 64-bit file mode work right in linux.
+#include "brttypes.h"
+#include "../include/db.h"
+
+
+int ybt_init (DBT *);
+int ybt_set_value (DBT *, bytevec val, ITEMLEN vallen, void **staticptrp);
+
+#endif

newbrt/yerror.h

+enum pma_errors { BRT_OK=0, BRT_ALREADY_THERE = -2, BRT_KEYEMPTY=-3 };
+
+enum typ_tag { TYP_BRTNODE = 0xdead0001, TYP_CACHETABLE, TYP_PMA };

pma/Makefile

+CFLAGS = -Wall -W -Werror -g
+pma: LDFLAGS=-lm
+pma:
+pma.o:
+