Skip to content

M
mariadb
Commit 85d55da2 authored by John Esmet's avatar John Esmet
Browse files
Options

FT-293 Remove examples/, which hasn't been built or maintained in years

parent e91c66cc
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 2093 deletions
CMakeLists.txt
View file @ 85d55da2
...@@ -77,9 +77,6 @@ add_subdirectory(locktree) ...@@ -77,9 +77,6 @@ add_subdirectory(locktree)
add_subdirectory(src) add_subdirectory(src)
add_subdirectory(tools) add_subdirectory(tools)
## subdirectories that just install things
add_subdirectory(examples)
install( install(
FILES README.md README-TOKUDB FILES README.md README-TOKUDB
DESTINATION . DESTINATION .
......
examples/CMakeLists.txt deleted 100644 → 0
View file @ e91c66cc
# detect when we are being built as a subproject
if (NOT DEFINED MYSQL_PROJECT_NAME_DOCSTRING)
install(
FILES
db-insert.c
db-insert-multiple.c
db-scan.c
db-update.c
Makefile
README.examples
DESTINATION
examples
COMPONENT
tokukv_examples
)
endif ()
\ No newline at end of file
examples/Makefile deleted 100644 → 0
View file @ e91c66cc
SRCS = $(wildcard *.c)
TARGETS = $(patsubst %.c,%,$(SRCS)) $(patsubst %.c,%-bdb,$(SRCS))
CPPFLAGS = -I../include -D_GNU_SOURCE
CFLAGS = -g -std=c99 -Wall -Wextra -Werror -Wno-missing-field-initializers
ifeq ($(USE_STATIC_LIBS),1)
LIBTOKUDB = tokufractaltree_static
LIBTOKUPORTABILITY = tokuportability_static
else
LIBTOKUDB = tokufractaltree
LIBTOKUPORTABILITY = tokuportability
endif
LDFLAGS = -L../lib -l$(LIBTOKUDB) -l$(LIBTOKUPORTABILITY) -Wl,-rpath,../lib -lpthread -lz -ldl
default local: $(TARGETS)
%: %.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -o $@ $(LDFLAGS)
%-bdb: %.c
$(CC) -D_GNU_SOURCE -DBDB $(CFLAGS) $^ -o $@ -ldb
check: $(TARGETS)
./db-insert -x && ./db-scan --lwc --prelock --prelockflag
checknox: $(TARGETS)
./db-insert && ./db-scan --nox --lwc --prelock --prelockflag
clean:
rm -rf $(TARGETS) bench.* update.env.* insertm.env.*
examples/README.examples deleted 100644 → 0
View file @ e91c66cc
The examples includes a pair of programs that can be compiled to use either the Berkeley DB library or the Tokutek Fractal Tree index library.
Note: The file formats are different from TokuDB and Berkley DB. Thus
you cannot access a database created by Berkeley DB using the Tokutek
DB, or vice-versa.
db-insert is a program that inserts random key-value pairs into a database.
db-scan is a program that scans through the key-value pairs, reading every row, from a database.
db-update is a program that upserts key-value pairs into a database. If the key already exists it increment a count in the value.
db-insert-multiple is a program and inserts key-value pairs into multiple databases. This is is now TokuDB maintains consistent
secondary databases.
To build it and run it (it's been tested on Fedora 10):
$ make (Makes the binaries)
Run the insertion workload under TokuDB:
$ ./db-insert
Run the insertion workload under BDB:
$ ./db-insert-bdb
Here is what the output looks like (this on a Thinkpad X61s laptop
running Fedora 10). BDB is a little faster for sequential insertions
(the first three columns), but much much slower for random insertions
(the next 3 columns), so that TokuDB is faster on combined workload.
$ ./db-insert
serial and random insertions of 1048576 per batch
serial 2.609965s 401759/s random 10.983798s 95466/s cumulative 13.593869s 154272/s
serial 3.053433s 343409/s random 12.008670s 87318/s cumulative 28.656115s 146367/s
serial 5.198312s 201715/s random 15.087426s 69500/s cumulative 48.954605s 128516/s
serial 6.096396s 171999/s random 13.550688s 77382/s cumulative 68.638321s 122215/s
Shutdown 4.025110s
Total time 72.677498s for 8388608 insertions = 115422/s
$ ./db-insert-bdb
serial and random insertions of 1048576 per batch
serial 2.623888s 399627/s random 8.770850s 119552/s cumulative 11.394805s 184045/s
serial 3.081946s 340232/s random 21.046589s 49822/s cumulative 35.523434s 118071/s
serial 14.160498s 74049/s random 497.117523s 2109/s cumulative 546.804504s 11506/s
serial 1.534212s 683462/s random 1128.525146s 929/s cumulative 1676.863892s 5003/s
Shutdown 195.879242s
Total time 1872.746582s for 8388608 insertions = 4479/s
The files are smaller for TokuDB than BDB.
$ ls -lh bench.tokudb/
total 39M
-rwxrwxr-x 1 bradley bradley 39M 2009-07-28 15:36 bench.db
$ ls -lh bench.bdb/
total 322M
-rw-r--r-- 1 bradley bradley 322M 2009-07-28 16:14 bench.db
When scanning the table, one can run out of locks with BDB. There are ways around it (increase the lock table size).
$ ./db-scan-bdb --nox
Lock table is out of available object entries
db-scan-bdb: db-scan.c:177: scanscan_hwc: Assertion `r==(-30988)' failed.
Aborted
TokuDB is fine on a big table scan.
$ ./db-scan --nox
Scan 33162304 bytes (2072644 rows) in 7.924463s at 4.184801MB/s
Scan 33162304 bytes (2072644 rows) in 3.062239s at 10.829431MB/s
0:3 1:53 2:56
miss=3 hit=53 wait_reading=0 wait=0
VmPeak: 244668 kB
VmHWM: 68096 kB
VmRSS: 1232 kB
The update-bdb program upserts 1B rows into a BDB database. When the database gets larger than memory, the throughput
should tank since every update needs to read a block from the storage system. The storage system becomes the performance
bottleneck. The program uses 1 1GB cache in front of the kernel's file system buffer cache. The program should hit the wall
at about 300M rows on a machine with 16GB of memory since keys are 8 bytes and values are 8 bytes in size.
$ ./db-update-bdb
The update program upserts 1B rows into a TokuDB database. Throughput should be not degrade significantly since the cost
of the storage system reads is amortized over 1000's of update operations. One should expect TokuDB to be at least 50 times
faster than BDB.
$ ./db-update
There isn't much documentation for the Tokutek Fractal Tree index library, but most of the API is like Berkeley DB's.
examples/db-insert-multiple.c deleted 100644 → 0
View file @ e91c66cc
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// measure the performance of insertions into multiple dictionaries using ENV->put_multiple
// the table schema is t(a bigint, b bigint, c bigint, d bigint, primary key(a), key(b), key(c,d), clustering key(d))
// the primary key(a) is represented with key=a and value=b,c,d
// the key(b) index is represented with key=b,a and no value
// the key(c,d) index is represented with key=c,d,a and no value
// the clustering key(d) is represented with key=d,a and value=b,c
// a is auto increment
// b, c and d are random
#include "../include/toku_config.h"
#include <stdio.h>
#include <stdbool.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/time.h>
#if defined(HAVE_BYTESWAP_H)
# include <byteswap.h>
#elif defined(HAVE_LIBKERN_OSBYTEORDER_H)
# include <libkern/OSByteOrder.h>
# define bswap_64 OSSwapInt64
#endif
#include <arpa/inet.h>
#include "db.h"
static int force_multiple = 1;
struct table {
int ndbs;
DB **dbs;
#if defined(TOKUDB)
DBT *mult_keys;
DBT *mult_vals;
uint32_t *mult_flags;
#endif
};
#if defined(TOKUDB)
static void table_init_dbt(DBT *dbt, size_t length) {
dbt->flags = DB_DBT_USERMEM;
dbt->data = malloc(length);
dbt->ulen = length;
dbt->size = 0;
}
static void table_destroy_dbt(DBT *dbt) {
free(dbt->data);
}
#endif
static void table_init(struct table *t, int ndbs, DB **dbs, size_t key_length __attribute__((unused)), size_t val_length __attribute__((unused))) {
t->ndbs = ndbs;
t->dbs = dbs;
#if defined(TOKUDB)
t->mult_keys = calloc(ndbs, sizeof (DBT));
int i;
for (i = 0; i < ndbs; i++)
table_init_dbt(&t->mult_keys[i], key_length);
t->mult_vals = calloc(ndbs, sizeof (DBT));
for (i = 0; i < ndbs; i++)
table_init_dbt(&t->mult_vals[i], val_length);
t->mult_flags = calloc(ndbs, sizeof (uint32_t));
for (i = 0; i < ndbs; i++)
t->mult_flags[i] = 0;
#endif
}
static void table_destroy(struct table *t) {
#if defined(TOKUDB)
int i;
for (i = 0; i < t->ndbs; i++)
table_destroy_dbt(&t->mult_keys[i]);
free(t->mult_keys);
for (i = 0; i < t->ndbs; i++)
table_destroy_dbt(&t->mult_vals[i]);
free(t->mult_vals);
free(t->mult_flags);
#else
assert(t);
#endif
}
static int verbose = 0;
static long random64(void) {
return ((long)random() << 32LL) + (long)random();
}
static long htonl64(long x) {
#if BYTE_ORDER == LITTLE_ENDIAN
return bswap_64(x);
#else
#error
#endif
}
#if defined(TOKUDB)
static int my_generate_row_for_put(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
assert(src_db);
assert(dest_key->flags == DB_DBT_USERMEM && dest_key->ulen >= 4 * 8);
assert(dest_val->flags == DB_DBT_USERMEM && dest_val->ulen >= 4 * 8);
int index_num;
assert(dest_db->descriptor->dbt.size == sizeof index_num);
memcpy(&index_num, dest_db->descriptor->dbt.data, sizeof index_num);
switch (htonl(index_num) % 4) {
case 0:
// dest_key = src_key
dest_key->size = src_key->size;
memcpy(dest_key->data, src_key->data, src_key->size);
// dest_val = src_val
dest_val->size = src_val->size;
memcpy(dest_val->data, src_val->data, src_val->size);
break;
case 1:
// dest_key = b,a
dest_key->size = 2 * 8;
memcpy((char *)dest_key->data + 0, (char *)src_val->data + 0, 8);
memcpy((char *)dest_key->data + 8, (char *)src_key->data + 0, 8);
// dest_val = null
dest_val->size = 0;
break;
case 2:
// dest_key = c,d,a
dest_key->size = 3 * 8;
memcpy((char *)dest_key->data + 0, (char *)src_val->data + 8, 8);
memcpy((char *)dest_key->data + 8, (char *)src_val->data + 16, 8);
memcpy((char *)dest_key->data + 16, (char *)src_key->data + 0, 8);
// dest_val = null
dest_val->size = 0;
break;
case 3:
// dest_key = d,a
dest_key->size = 2 * 8;
memcpy((char *)dest_key->data + 0, (char *)src_val->data + 16, 8);
memcpy((char *)dest_key->data + 8, (char *)src_key->data + 0, 8);
// dest_val = b,c
dest_val->size = 2 * 8;
memcpy((char *)dest_val->data + 0, (char *)src_val->data + 0, 8);
memcpy((char *)dest_val->data + 8, (char *)src_val->data + 8, 8);
break;
default:
assert(0);
}
return 0;
}
#else
static int my_secondary_key(DB *db, const DBT *src_key, const DBT *src_val, DBT *dest_key) {
assert(dest_key->flags == 0 && dest_key->data == NULL);
dest_key->flags = DB_DBT_APPMALLOC;
dest_key->data = malloc(4 * 8); assert(dest_key->data);
switch ((intptr_t)db->app_private % 4) {
case 0:
// dest_key = src_key
dest_key->size = src_key->size;
memcpy(dest_key->data, src_key->data, src_key->size);
break;
case 1:
// dest_key = b,a
dest_key->size = 2 * 8;
memcpy((char *)dest_key->data + 0, (char *)src_val->data + 0, 8);
memcpy((char *)dest_key->data + 8, (char *)src_key->data + 0, 8);
break;
case 2:
// dest_key = c,d,a
dest_key->size = 3 * 8;
memcpy((char *)dest_key->data + 0, (char *)src_val->data + 8, 8);
memcpy((char *)dest_key->data + 8, (char *)src_val->data + 16, 8);
memcpy((char *)dest_key->data + 16, (char *)src_key->data + 0, 8);
break;
case 3:
// dest_key = d,a,b,c
dest_key->size = 4 * 8;
memcpy((char *)dest_key->data + 0, (char *)src_val->data + 16, 8);
memcpy((char *)dest_key->data + 8, (char *)src_key->data + 0, 8);
memcpy((char *)dest_key->data + 16, (char *)src_val->data + 0, 8);
memcpy((char *)dest_key->data + 24, (char *)src_val->data + 8, 8);
break;
default:
assert(0);
}
return 0;
}
#endif
static void insert_row(DB_ENV *db_env, struct table *t, DB_TXN *txn, long a, long b, long c, long d) {
int r;
// generate the primary key
char key_buffer[8];
a = htonl64(a);
memcpy(key_buffer, &a, sizeof a);
// generate the primary value
char val_buffer[3*8];
b = htonl64(b);
memcpy(val_buffer+0, &b, sizeof b);
c = htonl64(c);
memcpy(val_buffer+8, &c, sizeof c);
d = htonl64(d);
memcpy(val_buffer+16, &d, sizeof d);
DBT key = { .data = key_buffer, .size = sizeof key_buffer };
DBT value = { .data = val_buffer, .size = sizeof val_buffer };
#if defined(TOKUDB)
if (!force_multiple && t->ndbs == 1) {
r = t->dbs[0]->put(t->dbs[0], txn, &key, &value, t->mult_flags[0]); assert(r == 0);
} else {
r = db_env->put_multiple(db_env, t->dbs[0], txn, &key, &value, t->ndbs, &t->dbs[0], t->mult_keys, t->mult_vals, t->mult_flags); assert(r == 0);
}
#else
assert(db_env);
r = t->dbs[0]->put(t->dbs[0], txn, &key, &value, 0); assert(r == 0);
#endif
}
static inline float tdiff (struct timeval *a, struct timeval *b) {
return (a->tv_sec - b->tv_sec) +1e-6*(a->tv_usec - b->tv_usec);
}
static void insert_all(DB_ENV *db_env, struct table *t, long nrows, long max_rows_per_txn, long key_range, long rows_per_report, bool do_txn) {
int r;
struct timeval tstart;
r = gettimeofday(&tstart, NULL); assert(r == 0);
struct timeval tlast = tstart;
DB_TXN *txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
}
long n_rows_per_txn = 0;
long rowi;
for (rowi = 0; rowi < nrows; rowi++) {
long a = rowi;
long b = random64() % key_range;
long c = random64() % key_range;
long d = random64() % key_range;
insert_row(db_env, t, txn, a, b, c, d);
n_rows_per_txn++;
// maybe commit
if (do_txn && n_rows_per_txn == max_rows_per_txn) {
r = txn->commit(txn, 0); assert(r == 0);
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
n_rows_per_txn = 0;
}
// maybe report performance
if (((rowi + 1) % rows_per_report) == 0) {
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
float last_time = tdiff(&tnow, &tlast);
float total_time = tdiff(&tnow, &tstart);
printf("%ld %.3f %.0f/s %.0f/s\n", rowi + 1, last_time, rows_per_report/last_time, rowi/total_time); fflush(stdout);
tlast = tnow;
}
}
if (do_txn) {
r = txn->commit(txn, 0); assert(r == 0);
}
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
printf("total %ld %.3f %.0f/s\n", nrows, tdiff(&tnow, &tstart), nrows/tdiff(&tnow, &tstart)); fflush(stdout);
}
int main(int argc, char *argv[]) {
#if defined(TOKDUB)
char *db_env_dir = "insertm.env.tokudb";
#else
char *db_env_dir = "insertm.env.bdb";
#endif
int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG;
long rows = 100000000;
long rows_per_txn = 1000;
long rows_per_report = 100000;
long key_range = 100000;
bool do_txn = true;
u_int32_t pagesize = 0;
u_int64_t cachesize = 1000000000;
int ndbs = 4;
#if defined(TOKUDB)
u_int32_t checkpoint_period = 60;
#endif
int i;
for (i = 1; i < argc; i++) {
char *arg = argv[i];
if (strcmp(arg, "--verbose") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "--ndbs") == 0 && i+1 < argc) {
ndbs = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--rows") == 0 && i+1 < argc) {
rows = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_txn") == 0 && i+1 < argc) {
rows_per_txn = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_report") == 0 && i+1 < argc) {
rows_per_report = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--key_range") == 0 && i+1 < argc) {
key_range = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--txn") == 0 && i+1 < argc) {
do_txn = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--pagesize") == 0 && i+1 < argc) {
pagesize = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--cachesize") == 0 && i+1 < argc) {
cachesize = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--force_multiple") == 0 && i+1 < argc) {
force_multiple = atoi(argv[++i]);
continue;
}
#if defined(TOKUDB)
if (strcmp(arg, "--checkpoint_period") == 0 && i+1 < argc) {
checkpoint_period = atoi(argv[++i]);
continue;
}
#endif
assert(0);
}
int r;
char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
r = system(rm_cmd); assert(r == 0);
r = mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
// create and open the env
DB_ENV *db_env = NULL;
r = db_env_create(&db_env, 0); assert(r == 0);
if (!do_txn)
db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG);
if (cachesize) {
const u_int64_t gig = 1 << 30;
r = db_env->set_cachesize(db_env, cachesize / gig, cachesize % gig, 1); assert(r == 0);
}
#if defined(TOKUDB)
r = db_env->set_generate_row_callback_for_put(db_env, my_generate_row_for_put); assert(r == 0);
#endif
r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
#if defined(TOKUDB)
if (checkpoint_period) {
r = db_env->checkpointing_set_period(db_env, checkpoint_period); assert(r == 0);
u_int32_t period;
r = db_env->checkpointing_get_period(db_env, &period); assert(r == 0 && period == checkpoint_period);
}
#endif
// create the db
DB *dbs[ndbs];
for (i = 0; i < ndbs; i++) {
DB *db = NULL;
r = db_create(&db, db_env, 0); assert(r == 0);
DB_TXN *create_txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &create_txn, 0); assert(r == 0);
}
if (pagesize) {
r = db->set_pagesize(db, pagesize); assert(r == 0);
}
char db_filename[32]; sprintf(db_filename, "test%d", i);
r = db->open(db, create_txn, db_filename, NULL, DB_BTREE, DB_CREATE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
#if defined(TOKUDB)
DESCRIPTOR_S new_descriptor;
int index_num = htonl(i);
new_descriptor.dbt.data = &index_num;
new_descriptor.dbt.size = sizeof i;
r = db->change_descriptor(db, create_txn, &new_descriptor.dbt, 0); assert(r == 0);
#else
db->app_private = (void *) (intptr_t) i;
if (i > 0) {
r = dbs[0]->associate(dbs[0], create_txn, db, my_secondary_key, 0); assert(r == 0);
}
#endif
if (do_txn) {
r = create_txn->commit(create_txn, 0); assert(r == 0);
}
dbs[i] = db;
}
// insert all rows
struct table table;
table_init(&table, ndbs, dbs, 4 * 8, 4 * 8);
insert_all(db_env, &table, rows, rows_per_txn, key_range, rows_per_report, do_txn);
table_destroy(&table);
// shutdown
for (i = 0; i < ndbs; i++) {
DB *db = dbs[i];
r = db->close(db, 0); assert(r == 0); db = NULL;
}
r = db_env->close(db_env, 0); assert(r == 0); db_env = NULL;
return 0;
}
examples/db-insert.c deleted 100644 → 0
View file @ e91c66cc
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
// Define BDB if you want to compile this to use Berkeley DB
#include <stdint.h>
#include <inttypes.h>
#ifdef BDB
#include <sys/types.h>
#include <db.h>
#define DIRSUF bdb
#else
#include <tokudb.h>
#define DIRSUF tokudb
#endif
#include <assert.h>
#include <errno.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/time.h>
static inline float toku_tdiff (struct timeval *a, struct timeval *b) {
return (a->tv_sec - b->tv_sec) +1e-6*(a->tv_usec - b->tv_usec);
}
#if !defined(DB_PRELOCKED_WRITE)
#define NO_DB_PRELOCKED
#define DB_PRELOCKED_WRITE 0
#endif
int verbose=1;
enum { SERIAL_SPACING = 1<<6 };
enum { DEFAULT_ITEMS_TO_INSERT_PER_ITERATION = 1<<20 };
enum { DEFAULT_ITEMS_PER_TRANSACTION = 1<<14 };
static void insert (long long v);
#define CKERR(r) ({ int __r = r; if (__r!=0) fprintf(stderr, "%s:%d error %d %s\n", __FILE__, __LINE__, __r, db_strerror(r)); assert(__r==0); })
#define CKERR2(r,rexpect) if (r!=rexpect) fprintf(stderr, "%s:%d error %d %s\n", __FILE__, __LINE__, r, db_strerror(r)); assert(r==rexpect);
/* default test parameters */
int keysize = sizeof (long long);
int valsize = sizeof (long long);
int pagesize = 0;
long long cachesize = 1000000000; // 1GB
int dupflags = 0;
int noserial = 0; // Don't do the serial stuff
int norandom = 0; // Don't do the random stuff
int prelock = 0;
int prelockflag = 0;
int items_per_transaction = DEFAULT_ITEMS_PER_TRANSACTION;
int items_per_iteration = DEFAULT_ITEMS_TO_INSERT_PER_ITERATION;
int finish_child_first = 0; // Commit or abort child first (before doing so to the parent). No effect if child does not exist.
int singlex_child = 0; // Do a single transaction, but do all work with a child
int singlex = 0; // Do a single transaction
int singlex_create = 0; // Create the db using the single transaction (only valid if singlex)
int insert1first = 0; // insert 1 before doing the rest
int do_transactions = 0;
int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
int do_abort = 0;
int n_insertions_since_txn_began=0;
int env_open_flags = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL;
u_int32_t put_flags = 0;
double compressibility = -1; // -1 means make it very compressible. 1 means use random bits everywhere. 2 means half the bits are random.
int do_append = 0;
u_int32_t checkpoint_period = 60;
static void do_prelock(DB* db, DB_TXN* txn) {
if (prelock) {
#if !defined(NO_DB_PRELOCKED)
int r = db->pre_acquire_table_lock(db, txn);
assert(r==0);
#else
(void) db; (void) txn;
#endif
}
}
#define STRINGIFY2(s) #s
#define STRINGIFY(s) STRINGIFY2(s)
const char *dbdir = "./bench." STRINGIFY(DIRSUF);
char *dbfilename = "bench.db";
char *dbname;
DB_ENV *dbenv;
DB *db;
DB_TXN *parenttid=0;
DB_TXN *tid=0;
static void benchmark_setup (void) {
int r;
if (!do_append) {
char unlink_cmd[strlen(dbdir) + strlen("rm -rf ") + 1];
snprintf(unlink_cmd, sizeof(unlink_cmd), "rm -rf %s", dbdir);
//printf("unlink_cmd=%s\n", unlink_cmd);
system(unlink_cmd);
if (strcmp(dbdir, ".") != 0) {
r = mkdir(dbdir,S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH);
assert(r == 0);
}
}
r = db_env_create(&dbenv, 0);
assert(r == 0);
#if !defined(TOKUDB)
#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR <= 4
if (dbenv->set_lk_max) {
r = dbenv->set_lk_max(dbenv, items_per_transaction*2);
assert(r==0);
}
#elif (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR <= 7) || DB_VERSION_MAJOR >= 5
if (dbenv->set_lk_max_locks) {
r = dbenv->set_lk_max_locks(dbenv, items_per_transaction*2);
assert(r==0);
}
if (dbenv->set_lk_max_lockers) {
r = dbenv->set_lk_max_lockers(dbenv, items_per_transaction*2);
assert(r==0);
}
if (dbenv->set_lk_max_objects) {
r = dbenv->set_lk_max_objects(dbenv, items_per_transaction*2);
assert(r==0);
}
#else
#error
#endif
#endif
if (dbenv->set_cachesize) {
r = dbenv->set_cachesize(dbenv, cachesize / (1024*1024*1024), cachesize % (1024*1024*1024), 1);
if (r != 0)
printf("WARNING: set_cachesize %d\n", r);
}
{
r = dbenv->open(dbenv, dbdir, env_open_flags, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
assert(r == 0);
}
#if defined(TOKUDB)
if (checkpoint_period) {
printf("set checkpoint_period %u\n", checkpoint_period);
r = dbenv->checkpointing_set_period(dbenv, checkpoint_period); assert(r == 0);
u_int32_t period;
r = dbenv->checkpointing_get_period(dbenv, &period); assert(r == 0 && period == checkpoint_period);
}
#endif
r = db_create(&db, dbenv, 0);
assert(r == 0);
if (do_transactions) {
r=dbenv->txn_begin(dbenv, 0, &tid, 0); CKERR(r);
}
if (pagesize && db->set_pagesize) {
r = db->set_pagesize(db, pagesize);
assert(r == 0);
}
if (dupflags) {
r = db->set_flags(db, dupflags);
assert(r == 0);
}
r = db->open(db, tid, dbfilename, NULL, DB_BTREE, DB_CREATE, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
if (r!=0) fprintf(stderr, "errno=%d, %s\n", errno, strerror(errno));
assert(r == 0);
if (insert1first) {
if (do_transactions) {
r=tid->commit(tid, 0);
assert(r==0);
tid = NULL;
r=dbenv->txn_begin(dbenv, 0, &tid, 0); CKERR(r);
}
insert(-1);
if (singlex) {
r=tid->commit(tid, 0);
assert(r==0);
tid = NULL;
r=dbenv->txn_begin(dbenv, 0, &tid, 0); CKERR(r);
}
}
else if (singlex && !singlex_create) {
r=tid->commit(tid, 0);
assert(r==0);
tid = NULL;
r=dbenv->txn_begin(dbenv, 0, &tid, 0); CKERR(r);
}
if (do_transactions) {
if (singlex)
do_prelock(db, tid);
else {
r=tid->commit(tid, 0);
assert(r==0);
tid = NULL;
}
}
if (singlex_child) {
parenttid = tid;
tid = NULL;
r=dbenv->txn_begin(dbenv, parenttid, &tid, 0); CKERR(r);
}
}
static void benchmark_shutdown (void) {
int r;
if (do_transactions && singlex && !insert1first && (singlex_create || prelock)) {
#if defined(TOKUDB)
//There should be a single 'truncate' in the rollback instead of many 'insert' entries.
struct txn_stat *s;
r = tid->txn_stat(tid, &s);
assert(r==0);
//TODO: #1125 Always do the test after performance testing is done.
if (singlex_child) fprintf(stderr, "SKIPPED 'small rollback' test for child txn\n");
else
assert(s->rollback_raw_count < 100); // gross test, not worth investigating details
free(s);
//system("ls -l bench.tokudb");
#endif
}
if (do_transactions && singlex) {
if (!singlex_child || finish_child_first) {
assert(tid);
r = (do_abort ? tid->abort(tid) : tid->commit(tid, 0)); assert(r==0);
tid = NULL;
}
if (singlex_child) {
assert(parenttid);
r = (do_abort ? parenttid->abort(parenttid) : parenttid->commit(parenttid, 0)); assert(r==0);
parenttid = NULL;
}
else
assert(!parenttid);
}
assert(!tid);
assert(!parenttid);
r = db->close(db, 0);
assert(r == 0);
r = dbenv->close(dbenv, 0);
assert(r == 0);
}
static void long_long_to_array (unsigned char *a, int array_size, unsigned long long l) {
int i;
for (i=0; i<8 && i<array_size; i++)
a[i] = (l>>(56-8*i))&0xff;
}
static DBT *fill_dbt(DBT *dbt, const void *data, int size) {
memset(dbt, 0, sizeof *dbt);
dbt->size = size;
dbt->data = (void *) data;
return dbt;
}
// Fill array with 0's if compressibilty==-1, otherwise fill array with data that is likely to compress by a factor of compressibility.
static void fill_array (unsigned char *data, int size) {
memset(data, 0, size);
if (compressibility>0) {
int i;
for (i=0; i<size/compressibility; i++) {
data[i] = (unsigned char) random();
}
}
}
static void insert (long long v) {
unsigned char kc[keysize], vc[valsize];
DBT kt, vt;
fill_array(kc, sizeof kc);
long_long_to_array(kc, keysize, v); // Fill in the array first, then write the long long in.
fill_array(vc, sizeof vc);
long_long_to_array(vc, valsize, v);
int r = db->put(db, tid, fill_dbt(&kt, kc, keysize), fill_dbt(&vt, vc, valsize), put_flags);
CKERR(r);
if (do_transactions) {
if (n_insertions_since_txn_began>=items_per_transaction && !singlex) {
n_insertions_since_txn_began=0;
r = tid->commit(tid, 0); assert(r==0);
tid = NULL;
r=dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
do_prelock(db, tid);
n_insertions_since_txn_began=0;
}
n_insertions_since_txn_began++;
}
}
static void serial_insert_from (long long from) {
long long i;
if (do_transactions && !singlex) {
int r = dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
do_prelock(db, tid);
{
DBT k,v;
r=db->put(db, tid, fill_dbt(&k, "a", 1), fill_dbt(&v, "b", 1), put_flags);
CKERR(r);
}
}
for (i=0; i<items_per_iteration; i++) {
insert((from+i)*SERIAL_SPACING);
}
if (do_transactions && !singlex) {
int r= tid->commit(tid, 0); assert(r==0);
tid=NULL;
}
}
static long long llrandom (void) {
return (((long long)(random()))<<32) + random();
}
static void random_insert_below (long long below) {
long long i;
if (do_transactions && !singlex) {
int r = dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
do_prelock(db, tid);
}
for (i=0; i<items_per_iteration; i++) {
insert(llrandom()%below);
}
if (do_transactions && !singlex) {
int r= tid->commit(tid, 0); assert(r==0);
tid=NULL;
}
}
static void biginsert (long long n_elements, struct timeval *starttime) {
long long i;
struct timeval t1,t2;
int iteration;
for (i=0, iteration=0; i<n_elements; i+=items_per_iteration, iteration++) {
if (verbose) {
printf("%d ", iteration);
fflush(stdout);
}
if (!noserial) {
gettimeofday(&t1,0);
serial_insert_from(i);
gettimeofday(&t2,0);
if (verbose) {
printf("serial %9.6fs %8.0f/s ", toku_tdiff(&t2, &t1), items_per_iteration/toku_tdiff(&t2, &t1));
fflush(stdout);
}
}
if (!norandom) {
gettimeofday(&t1,0);
random_insert_below((i+items_per_iteration)*SERIAL_SPACING);
gettimeofday(&t2,0);
if (verbose) {
printf("random %9.6fs %8.0f/s ", toku_tdiff(&t2, &t1), items_per_iteration/toku_tdiff(&t2, &t1));
fflush(stdout);
}
}
if (verbose) {
printf("cumulative %9.6fs %8.0f/s\n", toku_tdiff(&t2, starttime), (((float)items_per_iteration*(!noserial+!norandom))/toku_tdiff(&t2, starttime))*(iteration+1));
fflush(stdout);
}
}
}
const long long default_n_items = 1LL<<22;
static int print_usage (const char *argv0) {
fprintf(stderr, "Usage:\n");
fprintf(stderr, " %s [-x] [--keysize KEYSIZE] [--valsize VALSIZE] [--noserial] [--norandom] [ n_iterations ]\n", argv0);
fprintf(stderr, " where\n");
fprintf(stderr, " -x do transactions (XCOUNT transactions per iteration) (default: no transactions at all)\n");
fprintf(stderr, " --keysize KEYSIZE sets the key size (default 8)\n");
fprintf(stderr, " --valsize VALSIZE sets the value size (default 8)\n");
fprintf(stderr, " --noserial causes the serial insertions to be skipped\n");
fprintf(stderr, " --norandom causes the random insertions to be skipped\n");
fprintf(stderr, " --cachesize CACHESIZE set the database cache size\n");
fprintf(stderr, " --pagesize PAGESIZE sets the database page size\n");
fprintf(stderr, " --compressibility C creates data that should compress by about a factor C. Default C is large. C is an float.\n");
fprintf(stderr, " --xcount N how many insertions per transaction (default=%d)\n", DEFAULT_ITEMS_PER_TRANSACTION);
fprintf(stderr, " --singlex (implies -x) Run the whole job as a single transaction. (Default don't run as a single transaction.)\n");
fprintf(stderr, " --singlex-child (implies -x) Run the whole job as a single transaction, do all work a child of that transaction.\n");
fprintf(stderr, " --finish-child-first Commit/abort child before doing so to parent (no effect if no child).\n");
fprintf(stderr, " --singlex-create (implies --singlex) Create the file using the single transaction (Default is to use a different transaction to create.)\n");
fprintf(stderr, " --prelock Prelock the database.\n");
fprintf(stderr, " --prelockflag Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
fprintf(stderr, " --abort Abort the singlex after the transaction is over. (Requires --singlex.)\n");
fprintf(stderr, " --nolog If transactions are used, then don't write the recovery log\n");
fprintf(stderr, " --periter N how many insertions per iteration (default=%d)\n", DEFAULT_ITEMS_TO_INSERT_PER_ITERATION);
fprintf(stderr, " --env DIR\n");
fprintf(stderr, " --append append to an existing file\n");
fprintf(stderr, " --checkpoint-period %" PRIu32 " checkpoint period\n", checkpoint_period);
fprintf(stderr, " n_iterations how many iterations (default %lld)\n", default_n_items/DEFAULT_ITEMS_TO_INSERT_PER_ITERATION);
return 1;
}
#define UU(x) x __attribute__((__unused__))
int main (int argc, const char *argv[]) {
struct timeval t1,t2,t3;
long long total_n_items = default_n_items;
char *endptr;
int i;
for (i=1; i<argc; i++) {
const char *arg = argv[i];
if (arg[0] != '-')
break;
if (strcmp(arg, "-q") == 0) {
verbose--; if (verbose<0) verbose=0;
} else if (strcmp(arg, "-x") == 0) {
do_transactions = 1;
} else if (strcmp(arg, "--noserial") == 0) {
noserial=1;
} else if (strcmp(arg, "--norandom") == 0) {
norandom=1;
} else if (strcmp(arg, "--compressibility") == 0) {
compressibility = atof(argv[++i]);
} else if (strcmp(arg, "--nolog") == 0) {
if_transactions_do_logging = 0;
} else if (strcmp(arg, "--singlex-create") == 0) {
do_transactions = 1;
singlex = 1;
singlex_create = 1;
} else if (strcmp(arg, "--finish-child-first") == 0) {
finish_child_first = 1;
} else if (strcmp(arg, "--singlex-child") == 0) {
do_transactions = 1;
singlex = 1;
singlex_child = 1;
} else if (strcmp(arg, "--singlex") == 0) {
do_transactions = 1;
singlex = 1;
} else if (strcmp(arg, "--insert1first") == 0) {
insert1first = 1;
} else if (strcmp(arg, "--xcount") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
items_per_transaction = strtoll(argv[++i], &endptr, 10); assert(*endptr == 0);
} else if (strcmp(arg, "--abort") == 0) {
do_abort = 1;
} else if (strcmp(arg, "--periter") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
items_per_iteration = strtoll(argv[++i], &endptr, 10); assert(*endptr == 0);
} else if (strcmp(arg, "--cachesize") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
cachesize = strtoll(argv[++i], &endptr, 10); assert(*endptr == 0);
} else if (strcmp(arg, "--keysize") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
keysize = atoi(argv[++i]);
} else if (strcmp(arg, "--valsize") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
valsize = atoi(argv[++i]);
} else if (strcmp(arg, "--pagesize") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
pagesize = atoi(argv[++i]);
} else if (strcmp(arg, "--env") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
dbdir = argv[++i];
} else if (strcmp(arg, "--prelock") == 0) {
prelock=1;
} else if (strcmp(arg, "--prelockflag") == 0) {
prelock=1;
prelockflag=1;
} else if (strcmp(arg, "--srandom") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
srandom(atoi(argv[++i]));
} else if (strcmp(arg, "--append") == 0) {
do_append = 1;
} else if (strcmp(arg, "--checkpoint-period") == 0) {
if (i+1 >= argc) return print_usage(argv[9]);
checkpoint_period = (u_int32_t) atoi(argv[++i]);
} else if (strcmp(arg, "--unique_checks") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
int unique_checks = atoi(argv[++i]);
if (unique_checks)
put_flags = DB_NOOVERWRITE;
else
put_flags = 0;
} else {
return print_usage(argv[0]);
}
}
if (do_transactions) {
env_open_flags |= DB_INIT_TXN | if_transactions_do_logging | DB_INIT_LOCK;
}
if (do_transactions && prelockflag) {
put_flags |= DB_PRELOCKED_WRITE;
}
if (i<argc) {
/* if it looks like a number */
char *end;
errno=0;
long n_iterations = strtol(argv[i], &end, 10);
if (errno!=0 || *end!=0 || end==argv[i]) {
print_usage(argv[0]);
return 1;
}
total_n_items = items_per_iteration * (long long)n_iterations;
}
if (verbose) {
if (!noserial) printf("serial ");
if (!noserial && !norandom) printf("and ");
if (!norandom) printf("random ");
printf("insertions of %d per batch%s\n", items_per_iteration, do_transactions ? " (with transactions)" : "");
}
benchmark_setup();
gettimeofday(&t1,0);
biginsert(total_n_items, &t1);
gettimeofday(&t2,0);
benchmark_shutdown();
gettimeofday(&t3,0);
if (verbose) {
printf("Shutdown %9.6fs\n", toku_tdiff(&t3, &t2));
printf("Total time %9.6fs for %lld insertions = %8.0f/s\n", toku_tdiff(&t3, &t1),
(!noserial+!norandom)*total_n_items, (!noserial+!norandom)*total_n_items/toku_tdiff(&t3, &t1));
}
return 0;
}
examples/db-scan.c deleted 100644 → 0
View file @ e91c66cc
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Scan the bench.tokudb/bench.db over and over. */
#define DONT_DEPRECATE_MALLOC
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdint.h>
#include <inttypes.h>
#ifdef BDB
#include <db.h>
#define DIRSUF bdb
#else
#include <tokudb.h>
#define DIRSUF tokudb
#endif
#include <assert.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/time.h>
static const char *pname;
static enum run_mode { RUN_HWC, RUN_LWC, RUN_VERIFY, RUN_RANGE} run_mode = RUN_HWC;
static int do_txns=1, prelock=0, prelockflag=0;
static u_int32_t lock_flag = 0;
static long limitcount=-1;
static u_int32_t cachesize = 127*1024*1024;
static u_int64_t start_range = 0, end_range = 0;
static int n_experiments = 2;
static int bulk_fetch = 1;
static int print_usage (const char *argv0) {
fprintf(stderr, "Usage:\n%s [--verify-lwc | --lwc | --nohwc] [--prelock] [--prelockflag] [--prelockwriteflag] [--env DIR]\n", argv0);
fprintf(stderr, " --verify-lwc means to run the light weight cursor and the heavyweight cursor to verify that they get the same answer.\n");
fprintf(stderr, " --lwc run light weight cursors instead of heavy weight cursors\n");
fprintf(stderr, " --prelock acquire a read lock on the entire table before running\n");
fprintf(stderr, " --prelockflag pass DB_PRELOCKED to the the cursor get operation whenever the locks have been acquired\n");
fprintf(stderr, " --prelockwriteflag pass DB_PRELOCKED_WRITE to the cursor get operation\n");
fprintf(stderr, " --nox no transactions (no locking)\n");
fprintf(stderr, " --count COUNT read the first COUNT rows and then stop.\n");
fprintf(stderr, " --cachesize N set the env cachesize to N bytes\n");
fprintf(stderr, " --srandom N srandom(N)\n");
fprintf(stderr, " --env DIR put db files in DIR instead of default\n");
fprintf(stderr, " --bulk_fetch 0|1 do bulk fetch on lwc operations (default: 1)\n");
return 1;
}
static DB_ENV *env;
static DB *db;
static DB_TXN *tid=0;
#define STRINGIFY2(s) #s
#define STRINGIFY(s) STRINGIFY2(s)
static const char *dbdir = "./bench." STRINGIFY(DIRSUF); /* DIRSUF is passed in as a -D argument to the compiler. */
static int env_open_flags_yesx = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOG|DB_INIT_LOCK;
static int env_open_flags_nox = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL;
static char *dbfilename = "bench.db";
static void parse_args (int argc, const char *argv[]) {
pname=argv[0];
argc--; argv++;
int specified_run_mode=0;
while (argc>0) {
if (strcmp(*argv,"--verify-lwc")==0) {
if (specified_run_mode && run_mode!=RUN_VERIFY) { two_modes: fprintf(stderr, "You specified two run modes\n"); exit(1); }
run_mode = RUN_VERIFY;
} else if (strcmp(*argv, "--lwc")==0) {
if (specified_run_mode && run_mode!=RUN_LWC) goto two_modes;
run_mode = RUN_LWC;
} else if (strcmp(*argv, "--hwc")==0) {
if (specified_run_mode && run_mode!=RUN_VERIFY) goto two_modes;
run_mode = RUN_HWC;
} else if (strcmp(*argv, "--prelock")==0) prelock=1;
#ifdef TOKUDB
else if (strcmp(*argv, "--prelockflag")==0) { prelockflag=1; lock_flag = DB_PRELOCKED; }
else if (strcmp(*argv, "--prelockwriteflag")==0) { prelockflag=1; lock_flag = DB_PRELOCKED_WRITE; }
#endif
else if (strcmp(*argv, "--nox")==0) { do_txns=0; }
else if (strcmp(*argv, "--count")==0) {
char *end;
argc--; argv++;
errno=0; limitcount=strtol(*argv, &end, 10); assert(errno==0);
printf("Limiting count to %ld\n", limitcount);
} else if (strcmp(*argv, "--cachesize")==0 && argc>0) {
char *end;
argc--; argv++;
cachesize=(u_int32_t)strtol(*argv, &end, 10);
} else if (strcmp(*argv, "--env") == 0) {
argc--; argv++;
if (argc==0) exit(print_usage(pname));
dbdir = *argv;
} else if (strcmp(*argv, "--range") == 0 && argc > 2) {
run_mode = RUN_RANGE;
argc--; argv++;
start_range = strtoll(*argv, NULL, 10);
argc--; argv++;
end_range = strtoll(*argv, NULL, 10);
} else if (strcmp(*argv, "--experiments") == 0 && argc > 1) {
argc--; argv++;
n_experiments = strtol(*argv, NULL, 10);
} else if (strcmp(*argv, "--srandom") == 0 && argc > 1) {
argc--; argv++;
srandom(atoi(*argv));
} else if (strcmp(*argv, "--bulk_fetch") == 0 && argc > 1) {
argc--; argv++;
bulk_fetch = atoi(*argv);
} else {
exit(print_usage(pname));
}
argc--; argv++;
}
//Prelocking is meaningless without transactions
if (do_txns==0) {
prelockflag=0;
lock_flag=0;
prelock=0;
}
}
static void scanscan_setup (void) {
int r;
r = db_env_create(&env, 0); assert(r==0);
r = env->set_cachesize(env, 0, cachesize, 1); assert(r==0);
r = env->open(env, dbdir, do_txns? env_open_flags_yesx : env_open_flags_nox, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); assert(r==0);
r = db_create(&db, env, 0); assert(r==0);
if (do_txns) {
r = env->txn_begin(env, 0, &tid, 0); assert(r==0);
}
r = db->open(db, tid, dbfilename, NULL, DB_BTREE, 0, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); assert(r==0);
#ifdef TOKUDB
if (prelock) {
r = db->pre_acquire_table_lock(db, tid);
assert(r==0);
}
#endif
}
static void scanscan_shutdown (void) {
int r;
r = db->close(db, 0); assert(r==0);
if (do_txns) {
r = tid->commit(tid, 0); assert(r==0);
}
r = env->close(env, 0); assert(r==0);
}
static double gettime (void) {
struct timeval tv;
int r = gettimeofday(&tv, 0);
assert(r==0);
return tv.tv_sec + 1e-6*tv.tv_usec;
}
static void scanscan_hwc (void) {
int r;
int counter=0;
for (counter=0; counter<n_experiments; counter++) {
long long totalbytes=0;
int rowcounter=0;
double prevtime = gettime();
DBT k,v;
DBC *dbc;
r = db->cursor(db, tid, &dbc, 0); assert(r==0);
memset(&k, 0, sizeof(k));
memset(&v, 0, sizeof(v));
u_int32_t c_get_flags = DB_NEXT;
if (prelockflag && (counter || prelock)) {
c_get_flags |= lock_flag;
}
while (0 == (r = dbc->c_get(dbc, &k, &v, c_get_flags))) {
//printf("r=%d\n", r);
totalbytes += k.size + v.size;
rowcounter++;
if (limitcount>0 && rowcounter>=limitcount) break;
}
assert(r==DB_NOTFOUND);
r = dbc->c_close(dbc); assert(r==0);
double thistime = gettime();
double tdiff = thistime-prevtime;
printf("Scan %lld bytes (%d rows) in %9.6fs at %9fMB/s\n", totalbytes, rowcounter, tdiff, 1e-6*totalbytes/tdiff);
}
}
#ifdef TOKUDB
struct extra_count {
long long totalbytes;
int rowcounter;
};
static int counttotalbytes (DBT const *key, DBT const *data, void *extrav) {
struct extra_count *e=extrav;
e->totalbytes += key->size + data->size;
e->rowcounter++;
return bulk_fetch ? TOKUDB_CURSOR_CONTINUE : 0;
}
static void scanscan_lwc (void) {
int r;
int counter=0;
for (counter=0; counter<n_experiments; counter++) {
struct extra_count e = {0,0};
double prevtime = gettime();
DBC *dbc;
r = db->cursor(db, tid, &dbc, 0); assert(r==0);
u_int32_t f_flags = 0;
if (prelockflag && (counter || prelock)) {
f_flags |= lock_flag;
}
long rowcounter=0;
while (0 == (r = dbc->c_getf_next(dbc, f_flags, counttotalbytes, &e))) {
rowcounter++;
if (limitcount>0 && rowcounter>=limitcount) break;
}
r = dbc->c_close(dbc); assert(r==0);
double thistime = gettime();
double tdiff = thistime-prevtime;
printf("LWC Scan %lld bytes (%d rows) in %9.6fs at %9fMB/s\n", e.totalbytes, e.rowcounter, tdiff, 1e-6*e.totalbytes/tdiff);
}
}
#endif
static void scanscan_range (void) {
int r;
double texperiments[n_experiments];
u_int64_t k = 0;
char kv[8];
DBT key, val;
int counter;
for (counter = 0; counter < n_experiments; counter++) {
if (1) { //if ((counter&1) == 0) {
makekey:
// generate a random key in the key range
k = (start_range + (random() % (end_range - start_range))) * (1<<6);
int i;
for (i = 0; i < 8; i++)
kv[i] = k >> (56-8*i);
}
memset(&key, 0, sizeof key); key.data = &kv, key.size = sizeof kv;
memset(&val, 0, sizeof val);
double tstart = gettime();
DBC *dbc;
r = db->cursor(db, tid, &dbc, 0); assert(r==0);
// set the cursor to the random key
r = dbc->c_get(dbc, &key, &val, DB_SET_RANGE+lock_flag);
if (r != 0) {
assert(r == DB_NOTFOUND);
printf("%s:%d %" PRIu64 "\n", __FUNCTION__, __LINE__, k);
goto makekey;
}
#ifdef TOKUDB
// do the range scan
long rowcounter = 0;
struct extra_count e = {0,0};
while (limitcount > 0 && rowcounter < limitcount) {
r = dbc->c_getf_next(dbc, prelockflag ? lock_flag : 0, counttotalbytes, &e);
if (r != 0)
break;
rowcounter++;
}
#endif
r = dbc->c_close(dbc);
assert(r==0);
texperiments[counter] = gettime() - tstart;
printf("%" PRIu64 " %f\n", k, texperiments[counter]); fflush(stdout);
}
// print the times
double tsum = 0.0, tmin = 0.0, tmax = 0.0;
for (counter = 0; counter < n_experiments; counter++) {
if (counter==0 || texperiments[counter] < tmin)
tmin = texperiments[counter];
if (counter==0 || texperiments[counter] > tmax)
tmax = texperiments[counter];
tsum += texperiments[counter];
}
printf("%f %f %f/%d = %f\n", tmin, tmax, tsum, n_experiments, tsum / n_experiments);
}
#ifdef TOKUDB
struct extra_verify {
long long totalbytes;
int rowcounter;
DBT k,v; // the k and v are gotten using the old cursor
};
static int
checkbytes (DBT const *key, DBT const *data, void *extrav) {
struct extra_verify *e=extrav;
e->totalbytes += key->size + data->size;
e->rowcounter++;
assert(e->k.size == key->size);
assert(e->v.size == data->size);
assert(memcmp(e->k.data, key->data, key->size)==0);
assert(memcmp(e->v.data, data->data, data->size)==0);
assert(e->k.data != key->data);
assert(e->v.data != data->data);
return 0;
}
static void scanscan_verify (void) {
int r;
int counter=0;
for (counter=0; counter<n_experiments; counter++) {
struct extra_verify v;
v.totalbytes=0;
v.rowcounter=0;
double prevtime = gettime();
DBC *dbc1, *dbc2;
r = db->cursor(db, tid, &dbc1, 0); assert(r==0);
r = db->cursor(db, tid, &dbc2, 0); assert(r==0);
memset(&v.k, 0, sizeof(v.k));
memset(&v.v, 0, sizeof(v.v));
u_int32_t f_flags = 0;
u_int32_t c_get_flags = DB_NEXT;
if (prelockflag && (counter || prelock)) {
f_flags |= lock_flag;
c_get_flags |= lock_flag;
}
while (1) {
int r1,r2;
r2 = dbc1->c_get(dbc1, &v.k, &v.v, c_get_flags);
r1 = dbc2->c_getf_next(dbc2, f_flags, checkbytes, &v);
assert(r1==r2);
if (r1) break;
}
r = dbc1->c_close(dbc1); assert(r==0);
r = dbc2->c_close(dbc2); assert(r==0);
double thistime = gettime();
double tdiff = thistime-prevtime;
printf("verify %lld bytes (%d rows) in %9.6fs at %9fMB/s\n", v.totalbytes, v.rowcounter, tdiff, 1e-6*v.totalbytes/tdiff);
}
}
#endif
int main (int argc, const char *argv[]) {
parse_args(argc,argv);
scanscan_setup();
switch (run_mode) {
case RUN_HWC: scanscan_hwc(); break;
#ifdef TOKUDB
case RUN_LWC: scanscan_lwc(); break;
case RUN_VERIFY: scanscan_verify(); break;
#endif
case RUN_RANGE: scanscan_range(); break;
default: assert(0); break;
}
scanscan_shutdown();
return 0;
}
examples/db-update.c deleted 100644 → 0
View file @ e91c66cc
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// measure the performance of a simulated "insert on duplicate key update" operation
// the table schema is t(a int, b int, c int, d int, primary key(a, b))
// a and b are random
// c is the sum of the observations
// d is the first observation
#include <stdio.h>
#include <stdbool.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <arpa/inet.h>
#include "db.h"
static size_t key_size = 8;
static size_t val_size = 8;
static int verbose = 0;
static void db_error(const DB_ENV *env, const char *prefix, const char *msg) {
printf("%s: %p %s %s\n", __FUNCTION__, env, prefix, msg);
}
static int get_int(void *p) {
int v;
memcpy(&v, p, sizeof v);
return htonl(v);
}
#if defined(TOKUDB)
static int my_update_callback(DB *db, const DBT *key, const DBT *old_val, const DBT *extra, void (*set_val)(const DBT *new_val, void *set_extra), void *set_extra) {
assert(db);
assert(key);
if (old_val == NULL) {
// insert new_val = extra
set_val(extra, set_extra);
} else {
if (verbose) printf("u");
// update new_val = old_val + extra
assert(old_val->size == val_size && extra->size == val_size);
char new_val_buffer[val_size];
memcpy(new_val_buffer, old_val->data, sizeof new_val_buffer);
int newc = htonl(get_int(old_val->data) + get_int(extra->data)); // newc = oldc + newc
memcpy(new_val_buffer, &newc, sizeof newc);
DBT new_val = { .data = new_val_buffer, .size = sizeof new_val_buffer };
set_val(&new_val, set_extra);
}
return 0;
}
#endif
static void insert_and_update(DB *db, DB_TXN *txn, int a, int b, int c, int d, bool do_update_callback) {
#if !defined(TOKUDB)
assert(!do_update_callback);
#endif
int r;
// generate the key
assert(key_size >= 8);
char key_buffer[key_size];
int newa = htonl(a);
memcpy(key_buffer, &newa, sizeof newa);
int newb = htonl(b);
memcpy(key_buffer+4, &newb, sizeof newb);
// generate the value
assert(val_size >= 8);
char val_buffer[val_size];
int newc = htonl(c);
memcpy(val_buffer, &newc, sizeof newc);
int newd = htonl(d);
memcpy(val_buffer+4, &newd, sizeof newd);
#if defined(TOKUDB)
if (do_update_callback) {
// extra = value_buffer, implicit combine column c update function
DBT key = { .data = key_buffer, .size = sizeof key_buffer };
DBT extra = { .data = val_buffer, .size = sizeof val_buffer };
r = db->update(db, txn, &key, &extra, 0); assert(r == 0);
} else
#endif
{
DBT key = { .data = key_buffer, .size = sizeof key_buffer };
DBT value = { .data = val_buffer, .size = sizeof val_buffer };
DBT oldvalue = { };
r = db->get(db, txn, &key, &oldvalue, 0);
assert(r == 0 || r == DB_NOTFOUND);
if (r == 0) {
// update it
if (verbose) printf("U");
int oldc = get_int(oldvalue.data);
newc = htonl(oldc + c); // newc = oldc + newc
memcpy(val_buffer, &newc, sizeof newc);
r = db->put(db, txn, &key, &value, 0);
assert(r == 0);
} else if (r == DB_NOTFOUND) {
r = db->put(db, txn, &key, &value, 0);
assert(r == 0);
}
}
}
static inline float tdiff (struct timeval *a, struct timeval *b) {
return (a->tv_sec - b->tv_sec) +1e-6*(a->tv_usec - b->tv_usec);
}
static void insert_and_update_all(DB_ENV *db_env, DB *db, long nrows, long max_rows_per_txn, int key_range, long rows_per_report, bool do_update_callback, bool do_txn) {
int r;
struct timeval tstart;
r = gettimeofday(&tstart, NULL); assert(r == 0);
struct timeval tlast = tstart;
DB_TXN *txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
}
long n_rows_per_txn = 0;
long rowi;
for (rowi = 0; rowi < nrows; rowi++) {
int a = random() % key_range;
int b = random() % key_range;
int c = 1;
int d = 0; // timestamp
insert_and_update(db, txn, a, b, c, d, do_update_callback);
n_rows_per_txn++;
// maybe commit
if (do_txn && n_rows_per_txn == max_rows_per_txn) {
r = txn->commit(txn, 0); assert(r == 0);
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
n_rows_per_txn = 0;
}
// maybe report performance
if (((rowi + 1) % rows_per_report) == 0) {
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
float last_time = tdiff(&tnow, &tlast);
float total_time = tdiff(&tnow, &tstart);
printf("%ld %.3f %.0f/s %.0f/s\n", rowi + 1, last_time, rows_per_report/last_time, rowi/total_time); fflush(stdout);
tlast = tnow;
}
}
if (do_txn) {
r = txn->commit(txn, 0); assert(r == 0);
}
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
printf("total %ld %.3f %.0f/s\n", nrows, tdiff(&tnow, &tstart), nrows/tdiff(&tnow, &tstart)); fflush(stdout);
}
int main(int argc, char *argv[]) {
#if defined(TOKUDB)
char *db_env_dir = "update.env.tokudb";
#else
char *db_env_dir = "update.env.bdb";
#endif
int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG;
char *db_filename = "update.db";
long rows = 1000000000;
long rows_per_txn = 100;
long rows_per_report = 100000;
int key_range = 1000000;
#if defined(TOKUDB)
bool do_update_callback = true;
#else
bool do_update_callback = false;
#endif
bool do_txn = false;
u_int64_t cachesize = 1000000000;
u_int32_t pagesize = 0;
#if defined(TOKUDB)
u_int32_t checkpoint_period = 60;
#endif
int i;
for (i = 1; i < argc; i++) {
char *arg = argv[i];
if (strcmp(arg, "--verbose") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "--rows") == 0 && i+1 < argc) {
rows = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_txn") == 0 && i+1 < argc) {
rows_per_txn = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_report") == 0 && i+1 < argc) {
rows_per_report = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--key_range") == 0 && i+1 < argc) {
key_range = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--txn") == 0 && i+1 < argc) {
do_txn = atoi(argv[++i]) != 0;
continue;
}
if (strcmp(arg, "--pagesize") == 0 && i+1 < argc) {
pagesize = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--cachesize") == 0 && i+1 < argc) {
cachesize = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--update_callback") == 0 && i+1 < argc) {
do_update_callback = atoi(argv[++i]) != 0;
continue;
}
if (strcmp(arg, "--key_size") == 0 && i+1 < argc) {
key_size = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--val_size") == 0 && i+1 < argc) {
val_size = atoi(argv[++i]);
continue;
}
#if defined(TOKUDB)
if (strcmp(arg, "--checkpoint_period") == 0 && i+1 < argc) {
checkpoint_period = atoi(argv[++i]);
continue;
}
#endif
assert(0);
}
int r;
char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
r = system(rm_cmd); assert(r == 0);
r = mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
// create and open the env
DB_ENV *db_env = NULL;
r = db_env_create(&db_env, 0); assert(r == 0);
#if defined(TOKUDB)
db_env->set_update(db_env, my_update_callback);
#endif
if (cachesize) {
if (verbose) printf("cachesize %llu\n", (unsigned long long)cachesize);
const u_int64_t gig = 1 << 30;
r = db_env->set_cachesize(db_env, cachesize / gig, cachesize % gig, 1); assert(r == 0);
}
if (!do_txn)
db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG);
db_env->set_errcall(db_env, db_error);
if (verbose) printf("env %s\n", db_env_dir);
r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
#if defined(TOKUDB)
if (checkpoint_period) {
r = db_env->checkpointing_set_period(db_env, checkpoint_period); assert(r == 0);
u_int32_t period;
r = db_env->checkpointing_get_period(db_env, &period); assert(r == 0 && period == checkpoint_period);
}
#endif
// create the db
DB *db = NULL;
r = db_create(&db, db_env, 0); assert(r == 0);
DB_TXN *create_txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &create_txn, 0); assert(r == 0);
}
if (pagesize) {
r = db->set_pagesize(db, pagesize); assert(r == 0);
}
r = db->open(db, create_txn, db_filename, NULL, DB_BTREE, DB_CREATE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
if (do_txn) {
r = create_txn->commit(create_txn, 0); assert(r == 0);
}
// insert on duplicate key update
insert_and_update_all(db_env, db, rows, rows_per_txn, key_range, rows_per_report, do_update_callback, do_txn);
// shutdown
r = db->close(db, 0); assert(r == 0); db = NULL;
r = db_env->close(db_env, 0); assert(r == 0); db_env = NULL;
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7