Skip to content

M
mariadb
Commit 1bf7a7a4 authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

closes[t:2449] [t:2484] Merge #2449 changes to main. 

Rollback logs are now checkpointed.  There are no rolltmp files.


git-svn-id: file:///svn/toku/tokudb@19167 c7de825b-a66e-492c-adef-691d508d4ae1
parent bf8e181e
Expand all Show whitespace changes
Inline Side-by-side
Showing with 2035 additions and 1717 deletions
buildheader/db.h_4_1
View file @ 1bf7a7a4
......@@ -379,7 +379,7 @@ typedef struct __toku_txn_progress {
} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
struct txn_stat {
u_int64_t rolltmp_raw_count;
u_int64_t rollback_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
......
buildheader/db.h_4_3
View file @ 1bf7a7a4
......@@ -395,7 +395,7 @@ typedef struct __toku_txn_progress {
} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
struct txn_stat {
u_int64_t rolltmp_raw_count;
u_int64_t rollback_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
......
buildheader/db.h_4_4
View file @ 1bf7a7a4
......@@ -403,7 +403,7 @@ typedef struct __toku_txn_progress {
} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
struct txn_stat {
u_int64_t rolltmp_raw_count;
u_int64_t rollback_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
......
buildheader/db.h_4_5
View file @ 1bf7a7a4
......@@ -403,7 +403,7 @@ typedef struct __toku_txn_progress {
} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
struct txn_stat {
u_int64_t rolltmp_raw_count;
u_int64_t rollback_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
......
buildheader/db.h_4_6
View file @ 1bf7a7a4
......@@ -407,7 +407,7 @@ typedef struct __toku_txn_progress {
} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
struct txn_stat {
u_int64_t rolltmp_raw_count;
u_int64_t rollback_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
......
buildheader/make_db_h.c
View file @ 1bf7a7a4
......@@ -585,7 +585,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
printf("} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;\n");
printf("typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);\n");
printf("struct txn_stat {\n u_int64_t rolltmp_raw_count;\n};\n");
printf("struct txn_stat {\n u_int64_t rollback_raw_count;\n};\n");
const char *extra[] = {
"int (*txn_stat)(DB_TXN *, struct txn_stat **)",
"struct { void *next, *prev; } open_txns",
......
buildheader/tdb.h
View file @ 1bf7a7a4
......@@ -354,7 +354,7 @@ typedef struct __toku_txn_progress {
} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
struct txn_stat {
u_int64_t rolltmp_raw_count;
u_int64_t rollback_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/;
......
db-benchmark-test/Makefile
View file @ 1bf7a7a4
......@@ -64,7 +64,7 @@ build: build.tdb build.bdb
build.bdb: $(TARGET_BDB) $(SCANSCAN_BDB) $(WINDOWS_BDB_LIB_NAME)
build.tdb: $(TARGET_TDB) $(SCANSCAN_TDB)
check: check-default check-rowsize-dup check-rowsize check-xfast check-x check-no-rolltmp check-4G
check: check-default check-rowsize-dup check-rowsize check-xfast check-x check-no-rollback check-4G child.benchmark.dir
SUPPORT_KEYSIZE=$$((3*1024)) # at least 3KiB
SUPPORT_ROWSIZE=$$((80*1024)) # at least 80KiB
......@@ -96,8 +96,8 @@ check-xfast: $(TARGET_TDB)
./$(TARGET_TDB) $(VERBVERBOSE) --noserial -x --valsize 1000 --cachesize 8000000 --xcount 1000 --periter 20000 --env xfast.dir 1 $(SUMMARIZE_CMD)
# A relatively fast test that detects #853 (don't log changes to a dictionary created in the same txn)
check-no-rolltmp: $(TARGET_TDB)
./$(TARGET_TDB) $(VERBVERBOSE) --env no-rolltmp.dir --singlex --nolog --check_small_rolltmp $(SUMMARIZE_CMD)
check-no-rollback: $(TARGET_TDB)
./$(TARGET_TDB) $(VERBVERBOSE) --env no-rollback.dir --singlex --nolog --check_small_rollback $(SUMMARIZE_CMD)
# Check to make sure that if we make a file that's bigger than 4GB that we can read the file back out and get all the rows.
ifeq ($(TOKU_SKIP_4G),1)
......
db-benchmark-test/db-benchmark-test.c
View file @ 1bf7a7a4
......@@ -53,7 +53,7 @@ int singlex_child = 0; // Do a single transaction, but do all work with a child
int singlex = 0; // Do a single transaction
int singlex_create = 0; // Create the db using the single transaction (only valid if singlex)
int insert1first = 0; // insert 1 before doing the rest
int check_small_rolltmp = 0; // verify that the rollback logs are small (only valid if singlex)
int check_small_rollback = 0; // verify that the rollback logs are small (only valid if singlex)
int do_transactions = 0;
int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
int do_abort = 0;
......@@ -294,14 +294,14 @@ static void benchmark_shutdown (void) {
#endif
if (do_transactions && singlex && !insert1first && (singlex_create || prelock)) {
#if defined(TOKUDB)
//There should be a single 'truncate' in the rolltmp instead of many 'insert' entries.
//There should be a single 'truncate' in the rollback instead of many 'insert' entries.
struct txn_stat *s;
r = tid->txn_stat(tid, &s);
assert(r==0);
//TODO: #1125 Always do the test after performance testing is done.
if (singlex_child) fprintf(stderr, "SKIPPED 'small rolltmp' test for child txn\n");
if (singlex_child) fprintf(stderr, "SKIPPED 'small rollback' test for child txn\n");
else
assert(s->rolltmp_raw_count < 100); // gross test, not worth investigating details
assert(s->rollback_raw_count < 100); // gross test, not worth investigating details
os_free(s);
//system("ls -l bench.tokudb");
#endif
......@@ -487,7 +487,7 @@ static int print_usage (const char *argv0) {
fprintf(stderr, " --singlex-child (implies -x) Run the whole job as a single transaction, do all work a child of that transaction.\n");
fprintf(stderr, " --finish-child-first Commit/abort child before doing so to parent (no effect if no child).\n");
fprintf(stderr, " --singlex-create (implies --singlex) Create the file using the single transaction (Default is to use a different transaction to create.)\n");
fprintf(stderr, " --check_small_rolltmp (Only valid in --singlex mode) Verify that very little data was saved in the rollback logs.\n");
fprintf(stderr, " --check_small_rollback (Only valid in --singlex mode) Verify that very little data was saved in the rollback logs.\n");
fprintf(stderr, " --prelock Prelock the database.\n");
fprintf(stderr, " --prelockflag Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
fprintf(stderr, " --abort Abort the singlex after the transaction is over. (Requires --singlex.)\n");
......@@ -589,8 +589,8 @@ int main (int argc, const char *const argv[]) {
singlex = 1;
} else if (strcmp(arg, "--insert1first") == 0) {
insert1first = 1;
} else if (strcmp(arg, "--check_small_rolltmp") == 0) {
check_small_rolltmp = 1;
} else if (strcmp(arg, "--check_small_rollback") == 0) {
check_small_rollback = 1;
} else if (strcmp(arg, "--xcount") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
items_per_transaction = strtoll(argv[++i], &endptr, 10); assert(*endptr == 0);
......@@ -685,8 +685,8 @@ int main (int argc, const char *const argv[]) {
fprintf(stderr, "--insert_multiple only works on the TokuDB (not BDB)\n");
return print_usage(argv[0]);
}
if (check_small_rolltmp) {
fprintf(stderr, "--check_small_rolltmp only works on the TokuDB (not BDB)\n");
if (check_small_rollback) {
fprintf(stderr, "--check_small_rollback only works on the TokuDB (not BDB)\n");
return print_usage(argv[0]);
}
#endif
......@@ -697,8 +697,8 @@ int main (int argc, const char *const argv[]) {
put_flagss[i] = put_flags;
}
}
if (check_small_rolltmp && !singlex) {
fprintf(stderr, "--check_small_rolltmp requires --singlex\n");
if (check_small_rollback && !singlex) {
fprintf(stderr, "--check_small_rollback requires --singlex\n");
return print_usage(argv[0]);
}
if (!do_transactions && insert_multiple) {
......
include/db.h
View file @ 1bf7a7a4
......@@ -354,7 +354,7 @@ typedef struct __toku_txn_progress {
} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
struct txn_stat {
u_int64_t rolltmp_raw_count;
u_int64_t rollback_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/;
......
newbrt/Makefile
View file @ 1bf7a7a4
......@@ -41,7 +41,6 @@ local: bins libs $(TEST_NEWBRT);
BRT_SOURCES = \
block_allocator \
block_table \
bread \
brt-serialize \
brt-verify \
brt \
......
newbrt/block_table.c
View file @ 1bf7a7a4
......@@ -596,6 +596,19 @@ toku_block_verify_no_free_blocknums(BLOCK_TABLE bt) {
assert(bt->current.blocknum_freelist_head.b == freelist_null.b);
}
//Verify there are no data blocks except root.
void
toku_block_verify_no_data_blocks_except_root_unlocked(BLOCK_TABLE bt, BLOCKNUM root) {
//Relies on checkpoint having used optimize_translation
assert(root.b >= RESERVED_BLOCKNUMS);
assert(bt->current.smallest_never_used_blocknum.b == root.b + 1);
int64_t i;
for (i=RESERVED_BLOCKNUMS; i < root.b; i++) {
BLOCKNUM b = make_blocknum(i);
assert(bt->current.block_translation[b.b].size == size_is_free);
}
}
//Verify a blocknum is currently allocated.
void
toku_verify_blocknum_allocated(BLOCK_TABLE bt, BLOCKNUM b) {
......
newbrt/block_table.h
View file @ 1bf7a7a4
......@@ -35,6 +35,7 @@ void toku_allocate_blocknum(BLOCK_TABLE bt, BLOCKNUM *res, struct brt_header * h
void toku_allocate_blocknum_unlocked(BLOCK_TABLE bt, BLOCKNUM *res, struct brt_header * h);
void toku_free_blocknum(BLOCK_TABLE bt, BLOCKNUM *b, struct brt_header * h);
void toku_verify_blocknum_allocated(BLOCK_TABLE bt, BLOCKNUM b);
void toku_block_verify_no_data_blocks_except_root_unlocked(BLOCK_TABLE bt, BLOCKNUM root);
void toku_block_verify_no_free_blocknums(BLOCK_TABLE bt);
void toku_realloc_descriptor_on_disk(BLOCK_TABLE bt, DISKOFF size, DISKOFF *offset, struct brt_header * h);
void toku_get_descriptor_offset_size(BLOCK_TABLE bt, DISKOFF *offset, DISKOFF *size);
......
newbrt/bread.c deleted 100644 → 0
View file @ bf8e181e
/* Buffered read. */
#ident "$Id$"
#ident "Copyright (c) 2007, 2008, 2009 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "includes.h"
struct bread {
int64_t fileoff; // The byte before this offset is the next byte we will read (since we are reading backward)
int fd;
int bufoff; // The current offset in the buf. The next byte we will read is buf[bufoff-1] (assuming that bufoff>0).
char *buf; // A buffer with at least bufoff bytes in it.
};
BREAD create_bread_from_fd_initialize_at(int fd) {
BREAD XMALLOC(result);
int r = toku_os_get_file_size(fd, &result->fileoff);
assert(r==0);
result->fd=fd;
result->bufoff=0;
result->buf = 0;
return result;
}
int close_bread_without_closing_fd(BREAD br) {
toku_free(br->buf);
toku_free(br);
return 0;
}
ssize_t bread_backwards(BREAD br, void *vbuf, size_t nbytes) {
char *buf=vbuf;
ssize_t result=0;
const int i4 = sizeof(u_int32_t);
while (nbytes > 0) {
// read whatever we can out of the buffer.
if (br->bufoff>0) {
size_t to_copy = ((size_t)br->bufoff >= nbytes) ? nbytes : (size_t)br->bufoff;
memcpy(buf+nbytes-to_copy, &br->buf[br->bufoff-to_copy], to_copy);
nbytes -= to_copy;
result += to_copy;
br->bufoff -= to_copy;
}
if (nbytes>0) {
assert(br->bufoff==0);
u_int32_t compressed_length_n, uncompressed_length_n;
assert(br->fileoff>=i4); // there better be the three lengths plus the compressed data.
{ ssize_t r = pread(br->fd, &compressed_length_n, i4, br->fileoff- i4); assert(r==i4); }
u_int32_t compressed_length = toku_dtoh32(compressed_length_n);
assert(br->fileoff >= compressed_length + 3*i4);
{ ssize_t r = pread(br->fd, &uncompressed_length_n, i4, br->fileoff-2*i4); assert(r==i4); }
u_int32_t uncompressed_length = toku_dtoh32(uncompressed_length_n);
char *XMALLOC_N(compressed_length, zbuf);
{
ssize_t r = pread(br->fd, zbuf, compressed_length, br->fileoff- compressed_length -2*i4);
assert(r==(ssize_t)compressed_length);
}
{
u_int32_t compressed_length_n_again;
ssize_t r = pread(br->fd, &compressed_length_n_again, i4, br->fileoff-compressed_length-3*i4); assert(r==i4);
assert(compressed_length_n_again == compressed_length_n);
}
uLongf destlen = uncompressed_length;
XREALLOC_N(uncompressed_length, br->buf);
uncompress((Bytef*)br->buf, &destlen, (Bytef*)zbuf, compressed_length);
assert(destlen==uncompressed_length);
toku_free(zbuf);
br->bufoff = uncompressed_length;
br->fileoff -= (compressed_length + 3*i4);
}
}
return result;
}
int bread_has_more(BREAD br) {
return (br->fileoff>0) || (br->bufoff>0);
}
newbrt/bread.h deleted 100644 → 0
View file @ bf8e181e
#ifndef BREAD_H
#define BREAD_H
#ident "$Id$"
#ident "Copyright (c) 2007, 2008, 2009 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// A BREAD reads a file backwards using buffered I/O. BREAD stands for Buffered Read or Backwards Read.
// Conceivably, we could read forward too.
// The buffered I/O is buffered using a large buffer (e.g., something like a megabyte).
// Furthermore, data is compressed into blocks. Each block is a 4-byte compressed length (in network order), followed by compressed data, followed by a 4-byte uncompressed-length (in network order), followed by a 4-byte compressed length
// The compressed-length appears twice so that the file can be read backward or forward.
// If not for the large-buffer requirement, as well as compression, as well as reading backward, we could have used a FILE.
#include <sys/types.h>
typedef struct bread *BREAD;
BREAD create_bread_from_fd_initialize_at(int fd);
// Effect: Given a file descriptor, fd, create a BREAD.
// Requires: The fd must be an open fd.
int close_bread_without_closing_fd(BREAD);
// Effect: Close the BREAD, but don't close the underlying fd.
ssize_t bread_backwards(BREAD, void *buf, size_t nbytes);
// Read nbytes into buf, reading backwards.
int bread_has_more(BREAD);
// Is there more to read?
#endif
newbrt/brt-internal.h
View file @ 1bf7a7a4
......@@ -217,6 +217,10 @@ int toku_serialize_brtnode_to_memory (BRTNODE node, int n_workitems, int n_threa
/*out*/ size_t *n_bytes_to_write,
/*out*/ char **bytes_to_write);
int toku_serialize_brtnode_to(int fd, BLOCKNUM, BRTNODE node, struct brt_header *h, int n_workitems, int n_threads, BOOL for_checkpoint);
int toku_serialize_rollback_log_to (int fd, BLOCKNUM blocknum, ROLLBACK_LOG_NODE log,
struct brt_header *h, int n_workitems, int n_threads,
BOOL for_checkpoint);
int toku_deserialize_rollback_log_from (int fd, BLOCKNUM blocknum, u_int32_t fullhash, ROLLBACK_LOG_NODE *logp, TOKUTXN txn, struct brt_header *h);
int toku_deserialize_brtnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/, BRTNODE *brtnode, struct brt_header *h);
unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
......
newbrt/brt-serialize.c
View file @ 1bf7a7a4
This diff is collapsed.
newbrt/brt.c
View file @ 1bf7a7a4
This diff is collapsed.
newbrt/brt.h
View file @ 1bf7a7a4
......@@ -52,7 +52,7 @@ int brt_set_cachetable(BRT, CACHETABLE);
int toku_brt_open(BRT, const char *fname_in_env,
int is_create, int only_create, CACHETABLE ct, TOKUTXN txn, DB *db);
int toku_brt_open_recovery(BRT, const char *fname_in_env,
int is_create, int only_create, CACHETABLE ct, TOKUTXN txn, DB *db, int recovery_force_fcreate, FILENUM use_filenum);
int is_create, int only_create, CACHETABLE ct, TOKUTXN txn, DB *db, FILENUM use_filenum);
int toku_brt_remove_subdb(BRT brt, const char *dbname, u_int32_t flags);
......@@ -206,12 +206,14 @@ void toku_maybe_truncate_cachefile (CACHEFILE cf, int fd, u_int64_t size_used);
int maybe_preallocate_in_file (int fd, u_int64_t size);
// Effect: If file size is less than SIZE, make it bigger by either doubling it or growing by 16MB whichever is less.
int toku_brt_note_table_lock (BRT brt, TOKUTXN txn);
int toku_brt_note_table_lock (BRT brt, TOKUTXN txn, BOOL ignore_not_empty);
// Effect: Record the fact that the BRT has a table lock (and thus no other txn will modify it until this txn completes. As a result, we can limit the amount of information in the rollback data structure.
int toku_brt_zombie_needed (BRT brt);
int toku_brt_get_fragmentation(BRT brt, TOKU_DB_FRAGMENTATION report);
int toku_brt_header_set_panic(struct brt_header *h, int panic, char *panic_string);
BOOL toku_brt_is_empty (BRT brt);
double get_tdiff(void) __attribute__((__visibility__("default")));
......
newbrt/brttypes.h
View file @ 1bf7a7a4
......@@ -33,6 +33,7 @@ typedef u_int64_t TXNID;
#define TXNID_NONE ((TXNID)0)
typedef struct s_blocknum { int64_t b; } BLOCKNUM; // make a struct so that we will notice type problems.
#define ROLLBACK_NONE ((BLOCKNUM){0})
static inline BLOCKNUM make_blocknum(int64_t b) { BLOCKNUM result={b}; return result; }
......@@ -70,6 +71,7 @@ typedef enum __toku_bool { FALSE=0, TRUE=1} BOOL;
typedef struct tokulogger *TOKULOGGER;
#define NULL_LOGGER ((TOKULOGGER)0)
typedef struct tokutxn *TOKUTXN;
typedef struct txninfo *TXNINFO;
#define NULL_TXN ((TOKUTXN)0)
struct logged_btt_pair {
......@@ -121,5 +123,8 @@ typedef int (*generate_row_for_del_func)(DB *dest_db, DB *src_db, DBT *dest_val,
#define UU(x) x __attribute__((__unused__))
typedef struct memarena *MEMARENA;
typedef struct rollback_log_node *ROLLBACK_LOG_NODE;
#endif
newbrt/cachetable.c
View file @ 1bf7a7a4
This diff is collapsed.
newbrt/cachetable.h
View file @ 1bf7a7a4
......@@ -123,6 +123,7 @@ typedef int (*CACHETABLE_FETCH_CALLBACK)(CACHEFILE, int fd, CACHEKEY key, u_int3
void toku_cachefile_set_userdata(CACHEFILE cf, void *userdata,
int (*log_fassociate_during_checkpoint)(CACHEFILE, void*),
int (*log_suppress_rollback_during_checkpoint)(CACHEFILE, void*),
int (*close_userdata)(CACHEFILE, int, void*, char **/*error_string*/, BOOL, LSN),
int (*checkpoint_userdata)(CACHEFILE, int, void*),
int (*begin_checkpoint_userdata)(CACHEFILE, int, LSN, void*),
......
newbrt/checkpoint.c
View file @ 1bf7a7a4
......@@ -218,7 +218,6 @@ toku_checkpoint(CACHETABLE ct, TOKULOGGER logger,
checkpoint_footprint = 40;
time_last_checkpoint_begin = time(NULL);
r = toku_cachetable_begin_checkpoint(ct, logger);
LSN oldest_live_lsn = toku_logger_get_oldest_living_lsn(logger);
multi_operation_checkpoint_unlock();
ydb_unlock();
......@@ -230,7 +229,7 @@ toku_checkpoint(CACHETABLE ct, TOKULOGGER logger,
r = toku_cachetable_end_checkpoint(ct, logger, ydb_lock, ydb_unlock, callback2_f, extra2);
}
if (r==0 && logger) {
LSN trim_lsn = (oldest_live_lsn.lsn < logger->checkpoint_lsn.lsn) ? oldest_live_lsn : logger->checkpoint_lsn;
LSN trim_lsn = logger->last_completed_checkpoint_lsn;
r = toku_logger_maybe_trim_log(logger, trim_lsn);
}
......
newbrt/log-internal.h
View file @ 1bf7a7a4
......@@ -85,7 +85,7 @@ struct tokulogger {
// To access these, you must have the output condition lock.
LSN written_lsn; // the last lsn written
LSN fsynced_lsn; // What is the LSN of the highest fsynced log entry (accessed only while holding the output lock, and updated only when the output lock and output permission are held)
LSN checkpoint_lsn; // What is the LSN of the most recent completed checkpoint.
LSN last_completed_checkpoint_lsn; // What is the LSN of the most recent completed checkpoint.
long long next_log_file_number;
struct logbuf outbuf; // data being written to the file
int n_in_file; // The amount of data in the current file
......@@ -101,6 +101,7 @@ struct tokulogger {
u_int64_t swap_ctr; // how many times have input/output log buffers been swapped
void (*remove_finalize_callback) (DICTIONARY_ID, void*); // ydb-level callback to be called when a transaction that ...
void * remove_finalize_callback_extra; // ... deletes a file is committed or when one that creates a file is aborted.
CACHEFILE rollback_cachefile;
};
int toku_logger_find_next_unused_log_file(const char *directory, long long *result);
......@@ -116,25 +117,36 @@ struct tokutxn {
u_int64_t txnid64; /* this happens to be the first lsn */
TOKULOGGER logger;
TOKUTXN parent;
LSN last_lsn; /* Everytime anything is logged, update the LSN. (We need to atomically record the LSN along with writing into the log.) */
LSN first_lsn; /* The first lsn in the transaction. */
struct roll_entry *oldest_logentry,*newest_logentry; /* Only logentries with rollbacks are here. There is a list going from newest to oldest. */
MEMARENA rollentry_arena;
size_t rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory.
char *rollentry_filename;
int rollentry_fd; // If we spill the roll_entries, we write them into this fd.
toku_off_t rollentry_filesize; // How many bytes are in the rollentry file (this is the uncompressed bytes. If the file is compressed it may actually be smaller (or even larger with header information))
u_int64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children.
OMT open_brts; // a collection of the brts that we touched. Indexed by filenum.
XIDS xids; //Represents the xid list
BOOL force_fsync_on_commit; //This transaction NEEDS an fsync once (if) it commits. (commit means root txn)
BOOL has_done_work; //If this transaction has not done work, there is no need to fsync.
TXN_PROGRESS_POLL_FUNCTION progress_poll_fun;
void * progress_poll_fun_extra;
uint64_t num_rollback_nodes;
uint64_t num_rollentries;
uint64_t num_rollentries_processed;
BLOCKNUM spilled_rollback_head;
uint32_t spilled_rollback_head_hash;
BLOCKNUM spilled_rollback_tail;
uint32_t spilled_rollback_tail_hash;
BLOCKNUM current_rollback;
uint32_t current_rollback_hash;
BOOL recovered_from_checkpoint;
ROLLBACK_LOG_NODE pinned_inprogress_rollback_log;
};
struct txninfo {
uint64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children.
uint32_t num_brts;
BRT *open_brts;
BOOL force_fsync_on_commit; //This transaction NEEDS an fsync once (if) it commits. (commit means root txn)
uint64_t num_rollback_nodes;
uint64_t num_rollentries;
BLOCKNUM spilled_rollback_head;
BLOCKNUM spilled_rollback_tail;
BLOCKNUM current_rollback;
};
static inline int toku_logsizeof_u_int8_t (u_int32_t v __attribute__((__unused__))) {
......@@ -180,5 +192,4 @@ static inline char *fixup_fname(BYTESTRING *f) {
return fname;
}
int toku_read_rollback_backwards(BREAD, struct roll_entry **item, MEMARENA);
#endif
newbrt/log.h
View file @ 1bf7a7a4
......@@ -11,7 +11,6 @@
#include "../include/db.h"
#include "brttypes.h"
#include "memory.h"
#include "bread.h"
#include "x1764.h"
typedef void(*voidfp)(void);
......
newbrt/logformat.c
View file @ 1bf7a7a4
This diff is collapsed.
newbrt/logger.c
View file @ 1bf7a7a4
......@@ -55,7 +55,7 @@ int toku_logger_create (TOKULOGGER *resultp) {
result->outbuf = (struct logbuf) {0, LOGGER_MIN_BUF_SIZE, toku_xmalloc(LOGGER_MIN_BUF_SIZE), ZERO_LSN};
// written_lsn is uninitialized
// fsynced_lsn is uninitialized
result->checkpoint_lsn = ZERO_LSN;
result->last_completed_checkpoint_lsn = ZERO_LSN;
// next_log_file_number is uninitialized
// n_in_file is uninitialized
result->write_block_size = BRT_DEFAULT_NODE_SIZE; // default logging size is the same as the default brt block size
......@@ -68,6 +68,7 @@ int toku_logger_create (TOKULOGGER *resultp) {
result->input_lock_ctr = 0;
result->output_condition_lock_ctr = 0;
result->swap_ctr = 0;
result->rollback_cachefile = NULL;
result->output_is_available = TRUE;
return 0;
......@@ -136,6 +137,68 @@ int toku_logger_open (const char *directory, TOKULOGGER logger) {
return 0;
}
int
toku_logger_open_rollback(TOKULOGGER logger, CACHETABLE cachetable, BOOL create) {
assert(logger->is_open);
assert(!logger->is_panicked);
assert(!logger->rollback_cachefile);
int r;
BRT t = NULL; // Note, there is no DB associated with this BRT.
r = toku_brt_create(&t);
assert(r==0);
r = toku_brt_open(t, ROLLBACK_CACHEFILE_NAME, create, create, cachetable, NULL_TXN, NULL);
assert(r==0);
logger->rollback_cachefile = t->cf;
toku_brtheader_lock(t->h);
//Verify it is empty
assert(!t->h->panic);
//Must have no data blocks (rollback logs or otherwise).
toku_block_verify_no_data_blocks_except_root_unlocked(t->h->blocktable, t->h->root);
toku_brtheader_unlock(t->h);
assert(toku_brt_is_empty(t));
return r;
}
// Requires: Rollback cachefile can only be closed immediately after a checkpoint,
// so it will always be clean (!h->dirty) when about to be closed.
// Rollback log can only be closed when there are no open transactions,
// so it will always be empty (no data blocks) when about to be closed.
int
toku_logger_close_rollback(TOKULOGGER logger, BOOL recovery_failed) {
int r = 0;
CACHEFILE cf = logger->rollback_cachefile; // stored in logger at rollback cachefile open
if (!logger->is_panicked && cf) {
BRT brt_to_close;
{ //Find "brt"
struct brt_header *h = toku_cachefile_get_userdata(cf);
toku_brtheader_lock(h);
if (!h->panic && recovery_failed) {
toku_brt_header_set_panic(h, EINVAL, "Recovery failed");
}
//Verify it is safe to close it.
if (!h->panic) { //If paniced, it is safe to close.
assert(!h->dirty); //Must not be dirty.
//Must have no data blocks (rollback logs or otherwise).
toku_block_verify_no_data_blocks_except_root_unlocked(h->blocktable, h->root);
}
assert(!toku_list_empty(&h->live_brts)); // there is always one brt associated with the header
brt_to_close = toku_list_struct(toku_list_head(&h->live_brts), struct brt, live_brt_link);
assert(brt_to_close);
toku_brtheader_unlock(h);
assert(toku_brt_is_empty(brt_to_close));
}
char *error_string_ignore = NULL;
r = toku_close_brt(brt_to_close, &error_string_ignore);
//Set as dealt with already.
logger->rollback_cachefile = NULL;
}
return r;
}
// No locks held on entry
// No locks held on exit.
// No locks are needed, since you cannot legally close the log concurrently with doing anything else.
......@@ -183,7 +246,8 @@ int toku_logger_shutdown(TOKULOGGER logger) {
if (logger->is_open) {
if (toku_omt_size(logger->live_txns) == 0) {
BYTESTRING comment = { strlen("shutdown"), "shutdown" };
r = toku_log_comment(logger, NULL, TRUE, 0, comment);
int r2 = toku_log_comment(logger, NULL, TRUE, 0, comment);
if (!r) r = r2;
}
}
return r;
......@@ -787,6 +851,10 @@ int toku_fread_LSN (FILE *f, LSN *lsn, struct x1764 *checksum, u_int32_t *le
return toku_fread_u_int64_t (f, &lsn->lsn, checksum, len);
}
int toku_fread_BLOCKNUM (FILE *f, BLOCKNUM *b, struct x1764 *checksum, u_int32_t *len) {
return toku_fread_u_int64_t (f, (u_int64_t*)&b->b, checksum, len);
}
int toku_fread_FILENUM (FILE *f, FILENUM *filenum, struct x1764 *checksum, u_int32_t *len) {
return toku_fread_u_int32_t (f, &filenum->fileid, checksum, len);
}
......@@ -903,6 +971,11 @@ int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, stru
return 0;
}
int toku_logprint_BLOCKNUM (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format) {
return toku_logprint_u_int64_t(outf, inf, fieldname, checksum, len, format);
}
int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format) {
return toku_logprint_u_int32_t(outf, inf, fieldname, checksum, len, format);
......@@ -982,11 +1055,6 @@ TXNID toku_txn_get_txnid (TOKUTXN txn) {
else return txn->txnid64;
}
LSN toku_txn_get_last_lsn (TOKUTXN txn) {
if (txn==0) return (LSN){0};
return txn->last_lsn;
}
LSN toku_logger_last_lsn(TOKULOGGER logger) {
return logger->lsn;
}
......@@ -1083,32 +1151,20 @@ int toku_logger_log_archive (TOKULOGGER logger, char ***logs_p, int flags) {
// get them into increasing order
qsort(all_logs, all_n_logs, sizeof(all_logs[0]), logfilenamecompare);
LSN oldest_live_txn_lsn;
{
TXNID oldest_living_xid = toku_logger_get_oldest_living_xid(logger);
if (oldest_living_xid == TXNID_NONE_LIVING)
oldest_live_txn_lsn = MAX_LSN;
else
oldest_live_txn_lsn.lsn = oldest_living_xid;
}
//printf("%s:%d Oldest txn is %lld\n", __FILE__, __LINE__, (long long)oldest_live_txn_lsn.lsn);
LSN save_lsn = logger->last_completed_checkpoint_lsn;
// Now starting at the last one, look for archivable ones.
// Count the total number of bytes, because we have to return a single big array. (That's the BDB interface. Bleah...)
LSN earliest_lsn_in_logfile={(unsigned long long)(-1LL)};
r = peek_at_log(logger, all_logs[all_n_logs-1], &earliest_lsn_in_logfile); // try to find the lsn that's in the most recent log
if ((earliest_lsn_in_logfile.lsn <= logger->checkpoint_lsn.lsn)&&
(earliest_lsn_in_logfile.lsn <= oldest_live_txn_lsn.lsn)) {
if (earliest_lsn_in_logfile.lsn <= save_lsn.lsn) {
i=all_n_logs-1;
} else {
for (i=all_n_logs-2; i>=0; i--) { // start at all_n_logs-2 because we never archive the most recent log
r = peek_at_log(logger, all_logs[i], &earliest_lsn_in_logfile);
if (r!=0) continue; // In case of error, just keep going
//printf("%s:%d file=%s firstlsn=%lld checkpoint_lsns={%lld %lld}\n", __FILE__, __LINE__, all_logs[i], (long long)earliest_lsn_in_logfile.lsn, (long long)logger->checkpoint_lsns[0].lsn, (long long)logger->checkpoint_lsns[1].lsn);
if ((earliest_lsn_in_logfile.lsn <= logger->checkpoint_lsn.lsn)&&
(earliest_lsn_in_logfile.lsn <= oldest_live_txn_lsn.lsn)) {
if (earliest_lsn_in_logfile.lsn <= save_lsn.lsn) {
break;
}
}
......@@ -1148,7 +1204,7 @@ TOKUTXN toku_logger_txn_parent (TOKUTXN txn) {
}
void toku_logger_note_checkpoint(TOKULOGGER logger, LSN lsn) {
logger->checkpoint_lsn = lsn;
logger->last_completed_checkpoint_lsn = lsn;
}
TXNID toku_logger_get_oldest_living_xid(TOKULOGGER logger) {
......@@ -1158,17 +1214,6 @@ TXNID toku_logger_get_oldest_living_xid(TOKULOGGER logger) {
return rval;
}
LSN toku_logger_get_oldest_living_lsn(TOKULOGGER logger) {
LSN lsn = {0};
if (logger) {
if (logger->oldest_living_xid == TXNID_NONE_LIVING)
lsn = MAX_LSN;
else
lsn.lsn = logger->oldest_living_xid;
}
return lsn;
}
LSN
toku_logger_get_next_lsn(TOKULOGGER logger) {
return logger->lsn;
......
newbrt/logger.h
View file @ 1bf7a7a4
......@@ -5,12 +5,20 @@
#ident "Copyright (c) 2007, 2008, 2009 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
enum { TOKU_LOG_VERSION = 1 };
enum {
TOKU_LOG_VERSION_1 = 1,
TOKU_LOG_VERSION_2 = 2,
TOKU_LOG_NEXT_VERSION, // the version after the current version
TOKU_LOG_VERSION = TOKU_LOG_NEXT_VERSION-1, // A hack so I don't have to change this line.
};
#define ROLLBACK_CACHEFILE_NAME "tokudb.rollback"
int toku_logger_create (TOKULOGGER *resultp);
int toku_logger_open (const char *directory, TOKULOGGER logger);
int toku_logger_shutdown(TOKULOGGER logger);
int toku_logger_close(TOKULOGGER *loggerp);
int toku_logger_open_rollback(TOKULOGGER logger, CACHETABLE cachetable, BOOL create);
int toku_logger_close_rollback(TOKULOGGER logger, BOOL recovery_failed);
int toku_logger_fsync (TOKULOGGER logger);
void toku_logger_panic (TOKULOGGER logger, int err);
......@@ -49,6 +57,7 @@ int toku_fread_u_int32_t_nocrclen (FILE *f, u_int32_t *v);
int toku_fread_u_int32_t (FILE *f, u_int32_t *v, struct x1764 *checksum, u_int32_t *len);
int toku_fread_u_int64_t (FILE *f, u_int64_t *v, struct x1764 *checksum, u_int32_t *len);
int toku_fread_LSN (FILE *f, LSN *lsn, struct x1764 *checksum, u_int32_t *len);
int toku_fread_BLOCKNUM (FILE *f, BLOCKNUM *lsn, struct x1764 *checksum, u_int32_t *len);
int toku_fread_FILENUM (FILE *f, FILENUM *filenum, struct x1764 *checksum, u_int32_t *len);
int toku_fread_TXNID (FILE *f, TXNID *txnid, struct x1764 *checksum, u_int32_t *len);
int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, struct x1764 *checksum, u_int32_t *len);
......@@ -58,6 +67,7 @@ int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, struct x176
int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format __attribute__((__unused__)));
int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
int toku_logprint_BLOCKNUM (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
int toku_logprint_u_int64_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
void toku_print_BYTESTRING (FILE *outf, u_int32_t len, char *data);
int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format __attribute__((__unused__)));
......@@ -67,7 +77,6 @@ int toku_read_and_print_logmagic (FILE *f, u_int32_t *versionp);
int toku_read_logmagic (FILE *f, u_int32_t *versionp);
TXNID toku_txn_get_txnid (TOKUTXN txn);
LSN toku_txn_get_last_lsn (TOKUTXN txn);
LSN toku_logger_last_lsn(TOKULOGGER logger);
TOKULOGGER toku_txn_logger (TOKUTXN txn);
......@@ -81,7 +90,6 @@ TOKUTXN toku_logger_txn_parent (TOKUTXN txn);
void toku_logger_note_checkpoint(TOKULOGGER logger, LSN lsn);
TXNID toku_logger_get_oldest_living_xid(TOKULOGGER logger);
LSN toku_logger_get_oldest_living_lsn(TOKULOGGER logger);
LSN toku_logger_get_next_lsn(TOKULOGGER logger);
void toku_logger_set_remove_finalize_callback(TOKULOGGER logger, void (*funcp)(DICTIONARY_ID, void *), void * extra);
void toku_logger_call_remove_finalize_callback(TOKULOGGER logger, DICTIONARY_ID dict_id);
......
newbrt/memarena.c
View file @ 1bf7a7a4
......@@ -12,9 +12,9 @@ struct memarena {
int n_other_bufs;
};
MEMARENA memarena_create (void) {
MEMARENA MALLOC(result); assert(result);
result->buf_size = 1024;
MEMARENA memarena_create_presized (size_t initial_size) {
MEMARENA XMALLOC(result);
result->buf_size = initial_size;
result->buf_used = 0;
result->other_bufs = NULL;
result->size_of_other_bufs = 0;
......@@ -23,6 +23,10 @@ MEMARENA memarena_create (void) {
return result;
}
MEMARENA memarena_create (void) {
return memarena_create_presized(1024);
}
void memarena_clear (MEMARENA ma) {
// Free the other bufs.
int i;
......
newbrt/memarena.h
View file @ 1bf7a7a4
......@@ -19,10 +19,11 @@
#include <sys/types.h>
typedef struct memarena *MEMARENA;
MEMARENA memarena_create_presized (size_t initial_size);
// Effect: Create a memarena with initial size. In case of ENOMEM, aborts.
MEMARENA memarena_create (void);
// Effect: Create a memarena. In case of ENOMEM, aborts.
// Effect: Create a memarena with default initial size. In case of ENOMEM, aborts.
void memarena_clear (MEMARENA ma);
// Effect: Reset the internal state so that the allocated memory can be used again.
......
newbrt/rbuf.h
View file @ 1bf7a7a4
......@@ -100,6 +100,18 @@ static inline BLOCKNUM rbuf_blocknum (struct rbuf *r) {
BLOCKNUM result = make_blocknum(rbuf_longlong(r));
return result;
}
static inline void rbuf_ma_BLOCKNUM (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), BLOCKNUM *blocknum) {
*blocknum = rbuf_blocknum(r);
}
static inline void rbuf_ma_u_int32_t (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), u_int32_t *num) {
*num = rbuf_int(r);
}
static inline void rbuf_ma_u_int64_t (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), u_int64_t *num) {
*num = rbuf_ulonglong(r);
}
static inline void rbuf_TXNID (struct rbuf *r, TXNID *txnid) {
*txnid = rbuf_ulonglong(r);
......@@ -119,7 +131,7 @@ static inline void rbuf_ma_FILENUM (struct rbuf *r, MEMARENA ma __attribute__((_
static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) {
bs->len = rbuf_int(r);
u_int32_t newndone = r->ndone + bs->len;
assert(newndone < r->size);
assert(newndone <= r->size);
bs->data = toku_memdup(&r->buf[r->ndone], (size_t)bs->len);
assert(bs->data);
r->ndone = newndone;
......@@ -128,7 +140,7 @@ static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) {
static inline void rbuf_ma_BYTESTRING (struct rbuf *r, MEMARENA ma, BYTESTRING *bs) {
bs->len = rbuf_int(r);
u_int32_t newndone = r->ndone + bs->len;
assert(newndone < r->size);
assert(newndone <= r->size);
bs->data = memarena_memdup(ma, &r->buf[r->ndone], (size_t)bs->len);
assert(bs->data);
r->ndone = newndone;
......
newbrt/recover.c
View file @ 1bf7a7a4
This diff is collapsed.
newbrt/recover.h
View file @ 1bf7a7a4
......@@ -11,7 +11,6 @@
#include "../include/db.h"
#include "brttypes.h"
#include "memory.h"
#include "bread.h"
#include "x1764.h"
// Run tokudb recovery from the log
......@@ -29,10 +28,6 @@ int tokudb_recover (const char *env_dir, const char *log_dir,
// Returns: TRUE if we need recovery, otherwise FALSE.
int tokudb_needs_recovery(const char *logdir, BOOL ignore_empty_log);
// Delete the rolltmp files
// Ruturns 0 if success
int tokudb_recover_delete_rolltmp_files(const char *datadir, const char *logdir);
// Return 0 if recovery log exists, ENOENT if log is missing
int tokudb_recover_log_exists(const char * log_dir);
......
newbrt/roll.c
View file @ 1bf7a7a4
This diff is collapsed.
newbrt/rollback.c
View file @ 1bf7a7a4
This diff is collapsed.
newbrt/rollback.h
View file @ 1bf7a7a4
......@@ -13,19 +13,24 @@ void toku_poll_txn_progress_function(TOKUTXN txn, uint8_t is_commit, uint8_t sta
int toku_rollback_commit(TOKUTXN txn, YIELDF yield, void*yieldv, LSN lsn);
int toku_rollback_abort(TOKUTXN txn, YIELDF yield, void*yieldv, LSN lsn);
void toku_rollback_txn_close (TOKUTXN txn);
int toku_get_and_pin_rollback_log_for_new_entry (TOKUTXN txn, ROLLBACK_LOG_NODE *result);
int toku_get_and_pin_rollback_log(TOKUTXN txn, TXNID xid, uint64_t sequence, BLOCKNUM name, uint32_t hash, ROLLBACK_LOG_NODE *result);
int toku_rollback_log_unpin(TOKUTXN txn, ROLLBACK_LOG_NODE log);
int toku_delete_rollback_log(TOKUTXN txn, ROLLBACK_LOG_NODE log);
typedef int(*apply_rollback_item)(TOKUTXN txn, struct roll_entry *item, YIELDF yield, void*yieldv, LSN lsn);
int toku_commit_rollback_item (TOKUTXN txn, struct roll_entry *item, YIELDF yield, void*yieldv, LSN lsn);
int toku_abort_rollback_item (TOKUTXN txn, struct roll_entry *item, YIELDF yield, void*yieldv, LSN lsn);
void *toku_malloc_in_rollback(TOKUTXN txn, size_t size);
void *toku_memdup_in_rollback(TOKUTXN txn, const void *v, size_t len);
char *toku_strdup_in_rollback(TOKUTXN txn, const char *s);
int toku_maybe_spill_rollbacks (TOKUTXN txn);
void *toku_malloc_in_rollback(ROLLBACK_LOG_NODE log, size_t size);
void *toku_memdup_in_rollback(ROLLBACK_LOG_NODE log, const void *v, size_t len);
int toku_maybe_spill_rollbacks (TOKUTXN txn, ROLLBACK_LOG_NODE log);
int toku_txn_note_brt (TOKUTXN txn, BRT brt);
int toku_txn_note_swap_brt (BRT live, BRT zombie);
int toku_txn_note_close_brt (BRT brt);
int toku_logger_txn_rolltmp_raw_count(TOKUTXN txn, u_int64_t *raw_count);
int toku_logger_txn_rollback_raw_count(TOKUTXN txn, u_int64_t *raw_count);
int toku_txn_find_by_xid (BRT brt, TXNID xid, TOKUTXN *txnptr);
......@@ -36,4 +41,23 @@ int toku_commit_fileentries (int fd, TOKUTXN txn, YIELDF yield,void *yieldv, LSN
//Heaviside function to find a TOKUTXN by TOKUTXN (used to find the index)
int find_xid (OMTVALUE v, void *txnv);
struct rollback_log_node {
enum typ_tag tag;
int layout_version;
int layout_version_original;
int layout_version_read_from_disk;
int dirty;
TXNID txnid; // Which transaction made this?
uint64_t sequence; // Which rollback log in the sequence is this?
BLOCKNUM thislogname; // Which block number is this chunk of the log?
uint32_t thishash;
BLOCKNUM older; // Which block number is the next oldest chunk of the log?
uint32_t older_hash;
struct roll_entry *oldest_logentry;
struct roll_entry *newest_logentry;
MEMARENA rollentry_arena;
size_t rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory.
};
#endif
newbrt/tests/Makefile
View file @ 1bf7a7a4
......@@ -58,15 +58,6 @@ check_ok:
test 0 = 0 $(SUMMARIZE_CMD)
ifeq ($(TOKU_SKIP_1305),1)
check_test1305:
@echo SKIPPED SLOW TEST $@
else
# Don't run 1305 under valgrind. It takes way too long.
check_test1305$(BINSUF): test1305$(BINSUF)
./$< $(VERBVERBOSE) $(SUMMARIZE_CMD)
endif
ifeq ($(TOKU_SKIP_MINICRON),1)
check_minicron-test$(BINSUF):
@echo "SKIPPED TEST $@ (Fails in win64 vm due to timing issues)"
......
newbrt/tests/bread-test.c deleted 100644 → 0
View file @ bf8e181e
/* Test bread by writing random data and then reading it using bread_backwards() to see if it gives the right answer.
* See test_1305 for another bread test (testing to see if it can read 1GB files) */
#include "test.h"
#include <toku_portability.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <zlib.h>
#include "../brttypes.h"
#include "../bread.h"
#define FNAME "bread-test.data"
#define RECORDS 20
#define RECORDLEN 100
char buf[RECORDS][RECORDLEN];
int sizes[RECORDS];
int sizesn[RECORDS];
int nwrote=0;
char wrotedata[RECORDS*RECORDLEN];
static void
test (int seed) {
srandom(seed);
unlink(FNAME);
int i;
{
int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
assert(fd>=0);
for (i=0; i<RECORDS; i++) {
sizes[i] = 1+ random()%RECORDLEN;
sizesn[i] = toku_htod32(sizes[i]);
int j;
for (j=0; j<sizes[i]; j++) {
buf[i][j] = wrotedata[nwrote++] = (char)random();
}
uLongf compressed_size = compressBound(sizes[i]);
Bytef compressed_buf[compressed_size];
{ int r = compress2(compressed_buf, &compressed_size, (Bytef*)(buf[i]), sizes[i], 1); assert(r==Z_OK); }
u_int32_t compressed_size_n = toku_htod32(compressed_size);
{ toku_os_full_write(fd, &compressed_size_n, 4); }
{ toku_os_full_write(fd, compressed_buf, compressed_size); }
{ toku_os_full_write(fd, &sizesn[i], 4); } // the uncompressed size
{ toku_os_full_write(fd, &compressed_size_n, 4); }
}
{ int r=close(fd); assert(r==0); }
}
int fd = open(FNAME, O_RDONLY+O_BINARY); assert(fd>=0);
// Now read it all backward
BREAD br = create_bread_from_fd_initialize_at(fd);
while (bread_has_more(br)) {
assert(nwrote>0);
int to_read = 1+(random()%RECORDLEN); // read from 1 to 100 (if RECORDLEN is 100)
if (to_read>nwrote) to_read=nwrote;
char rbuf[to_read];
int r = bread_backwards(br, rbuf, to_read);
assert(r==to_read);
assert(memcmp(rbuf, &wrotedata[nwrote-to_read], to_read)==0);
nwrote-=to_read;
}
assert(nwrote==0);
{ int r=close_bread_without_closing_fd(br); assert(r==0); }
{ int r=close(fd); assert(r==0); }
unlink(FNAME);
}
int
test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
int i;
for (i=0; i<10; i++) test(i);
return 0;
}
newbrt/tests/cachetable-checkpoint-pending.c
View file @ 1bf7a7a4
......@@ -102,7 +102,7 @@ static void checkpoint_pending(void) {
char fname1[] = __FILE__ "test1.dat";
r = unlink(fname1); if (r!=0) CKERR2(errno, ENOENT);
r = toku_cachetable_openf(&cf, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
toku_cachefile_set_userdata(cf, NULL, NULL, NULL, NULL, NULL, NULL,
toku_cachefile_set_userdata(cf, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
dummy_pin_unpin, dummy_pin_unpin);
// Insert items into the cachetable. All dirty.
......
newbrt/tests/cachetable-checkpoint-test.c
View file @ 1bf7a7a4
......@@ -60,7 +60,7 @@ static void cachetable_checkpoint_test(int n, enum cachetable_dirty dirty) {
unlink(fname1);
CACHEFILE f1;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL,
toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
dummy_pin_unpin, dummy_pin_unpin);
// insert items into the cachetable. all should be dirty
......
newbrt/tests/cachetable-prefetch-checkpoint-test.c
View file @ 1bf7a7a4
......@@ -46,7 +46,7 @@ static void cachetable_prefetch_checkpoint_test(int n, enum cachetable_dirty dir
unlink(fname1);
CACHEFILE f1;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL,
toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
dummy_pin_unpin, dummy_pin_unpin);
// prefetch block n+1. this will take 10 seconds.
......
newbrt/tests/recovery-bad-last-entry.c
View file @ 1bf7a7a4
......@@ -5,7 +5,7 @@
#define TESTDIR "dir." __FILE__
static const int magic_begin_end_checkpoint_sz = 77; // leave this many bytes in file
static const int magic_begin_end_checkpoint_sz = 85; // leave this many bytes in file
static int
run_test(void) {
......@@ -28,12 +28,12 @@ run_test(void) {
LSN beginlsn;
// all logs must contain a valid checkpoint
r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
r = toku_log_comment(logger, NULL, TRUE, 0, world); assert(r == 0);
r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
r = toku_log_comment(logger, NULL, TRUE, 0, there); assert(r == 0);
r = toku_logger_close(&logger); assert(r == 0);
......
newbrt/tests/recovery-cbegin-cend-hello.c
View file @ 1bf7a7a4
......@@ -21,7 +21,7 @@ run_test(void) {
// add begin checkpoint, end checkpoint
LSN beginlsn;
r = toku_log_begin_checkpoint(logger, &beginlsn, FALSE, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
r = toku_logger_close(&logger); assert(r == 0);
// add hello
......
newbrt/tests/recovery-cbegin-cend.c
View file @ 1bf7a7a4
......@@ -21,7 +21,7 @@ run_test(void) {
r = toku_logger_create(&logger); assert(r == 0);
r = toku_logger_open(TESTDIR, logger); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
r = toku_logger_close(&logger); assert(r == 0);
// run recovery
......
newbrt/tests/recovery-cbegin.c
View file @ 1bf7a7a4
......@@ -17,7 +17,7 @@ run_test(void) {
r = toku_logger_open(TESTDIR, logger); assert(r == 0);
LSN beginlsn;
r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
r = toku_logger_close(&logger); assert(r == 0);
......
newbrt/tests/recovery-cend-cbegin.c
View file @ 1bf7a7a4
......@@ -20,7 +20,7 @@ run_test(void) {
LSN firstbegin = ZERO_LSN;
r = toku_log_begin_checkpoint(logger, &firstbegin, TRUE, 0); assert(r == 0);
assert(firstbegin.lsn != ZERO_LSN.lsn);
r = toku_log_end_checkpoint(logger, NULL, FALSE, firstbegin.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, FALSE, firstbegin.lsn, 0, 0, 0); assert(r == 0);
r = toku_log_begin_checkpoint(logger, NULL, TRUE, 0); assert(r == 0);
r = toku_logger_close(&logger); assert(r == 0);
......
newbrt/tests/recovery-fopen-missing-file.c
View file @ 1bf7a7a4
......@@ -19,7 +19,7 @@ run_test(void) {
r = toku_logger_open(TESTDIR, logger); assert(r == 0);
LSN beginlsn;
r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
BYTESTRING iname = { strlen("missing_tokudb_file"), "missing_tokudb_file" };
FILENUM filenum = {42};
......
newbrt/tests/recovery-hello.c
View file @ 1bf7a7a4
......@@ -21,7 +21,7 @@ run_test(void) {
r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
LSN beginlsn;
r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
BYTESTRING there = { strlen("there"), "there" };
r = toku_log_comment(logger, NULL, TRUE, 0, there); assert(r == 0);
......
newbrt/tests/recovery-lsn-error-during-forward-scan.c
View file @ 1bf7a7a4
......@@ -30,7 +30,7 @@ run_test(void) {
LSN beginlsn;
r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
r = toku_logger_close(&logger); assert(r == 0);
......
newbrt/tests/test1305.c deleted 100644 → 0
View file @ bf8e181e
/* Test bread_backwards to make sure it can read backwards even for large files. */
#include <toku_portability.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <zlib.h>
#include "../brttypes.h"
#include "../bread.h"
#include "test.h"
#define FNAME "test1305.data"
// THe buffer size in units of 64-bit integers.
#define N_BIGINTS (1<<20)
#define BIGINT_SIZE (sizeof(u_int64_t))
// How big is the readback buffer (in 8-bit integers)?
#define READBACK_BUFSIZE (1<<20)
static void
test (u_int64_t fsize) {
unlink(FNAME);
// Create a file of size fsize. Fill it with 8-byte values which are integers, in order)
assert(fsize%(N_BIGINTS*sizeof(u_int64_t)) == 0); // Make sure the fsize is a multiple of the buffer size.
u_int64_t i = 0;
{
int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
assert(fd>=0);
static u_int64_t buf[N_BIGINTS]; //windows cannot handle this on the stack
static char compressed_buf[N_BIGINTS*2 + 1000]; // this is more than compressbound returns
uLongf compressed_len;
while (i*BIGINT_SIZE < fsize) {
if (verbose>0 && i % (1<<25) == 0) {
printf(" %s:test (%"PRIu64") forwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize);
fflush(stdout);
}
int j;
for (j=0; j<N_BIGINTS; j++) {
buf[j] = i++;
}
assert(sizeof(buf) == N_BIGINTS * BIGINT_SIZE);
{
compressed_len = sizeof(compressed_buf);
int r = compress2((Bytef*)compressed_buf, &compressed_len, (Bytef*)buf, sizeof(buf), 1);
assert(r==Z_OK);
}
{
u_int32_t v = toku_htod32(compressed_len);
toku_os_full_write(fd, &v, sizeof(v));
}
{
toku_os_full_write(fd, compressed_buf, compressed_len);
}
{
u_int32_t v = toku_htod32(sizeof(buf));
toku_os_full_write(fd, &v, sizeof(v));
}
{
u_int32_t v = toku_htod32(compressed_len);
toku_os_full_write(fd, &v, sizeof(v));
}
}
{ int r = close(fd); assert(r==0); }
}
assert(i*BIGINT_SIZE == fsize);
// Now read it all backward
{
int fd = open(FNAME, O_RDONLY+O_BINARY); assert(fd>=0);
BREAD br = create_bread_from_fd_initialize_at(fd);
while (bread_has_more(br)) {
if (verbose>0 && (fsize/BIGINT_SIZE - i) % (1<<25) == 0) {
printf(" %s:test (%"PRIu64") backwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize);
fflush(stdout);
}
assert(i>0);
i--;
u_int64_t storedi;
{ int r = bread_backwards(br, &storedi, sizeof(storedi)); assert(r==sizeof(storedi)); }
assert(storedi==i);
}
assert(i==0);
{ int r=close_bread_without_closing_fd(br); assert(r==0); }
{ int r=close(fd); assert(r==0); }
}
//printf("Did %" PRIu64 "\n", fsize);
//system("ls -l " FNAME);
unlink(FNAME);
}
int
test_main (int argc, const char *argv[]) {
default_parse_args(argc, argv);
test(1LL<<23);
test(1LL<<30);
test(1LL<<31);
test(1LL<<32);
test(1LL<<33);
return 0;
}
newbrt/tests/test_logcursor.c
View file @ 1bf7a7a4
......@@ -154,6 +154,9 @@ int create_logfiles() {
TXNID txnid = 0;
TXNID cp_txnid = 0;
u_int32_t num_fassociate = 0;
u_int32_t num_xstillopen = 0;
bs_aname.len = 4; bs_aname.data="a.db";
bs_bname.len = 4; bs_bname.data="b.db";
bs_a.len = 2; bs_a.data="a";
......@@ -171,31 +174,40 @@ int create_logfiles() {
//fcreate 'F': lsn=2 txnid=1 filenum=0 fname={len=4 data="a.db"} mode=0777 treeflags=0 crc=18a3d525 len=49
r = toku_log_fcreate(logger, &lsn, NO_FSYNC, txnid, fn_aname, bs_aname, 0x0777, 0, 0, bs_empty); assert(r==0);
//commit 'C': lsn=3 txnid=1 crc=00001f1e len=29
r = toku_log_commit(logger, &lsn, FSYNC, txnid); assert(r==0);
r = toku_log_xcommit(logger, &lsn, FSYNC, txnid); assert(r==0);
//xbegin 'b': lsn=4 parenttxnid=0 crc=00000a1f len=29
r = toku_log_xbegin(logger, &lsn, NO_FSYNC, 0); assert(r==0); txnid = lsn.lsn;
//fcreate 'F': lsn=5 txnid=4 filenum=1 fname={len=4 data="b.db"} mode=0777 treeflags=0 crc=14a47925 len=49
r = toku_log_fcreate(logger, &lsn, NO_FSYNC, txnid, fn_bname, bs_bname, 0x0777, 0, 0, bs_empty); assert(r==0);
//commit 'C': lsn=6 txnid=4 crc=0000c11e len=29
r = toku_log_commit(logger, &lsn, FSYNC, txnid); assert(r==0);
r = toku_log_xcommit(logger, &lsn, FSYNC, txnid); assert(r==0);
//xbegin 'b': lsn=7 parenttxnid=0 crc=0000f91f len=29
r = toku_log_xbegin(logger, &lsn, NO_FSYNC, 0); assert(r==0); txnid = lsn.lsn;
//enq_insert 'I': lsn=8 filenum=0 xid=7 key={len=2 data="a\000"} value={len=2 data="b\000"} crc=40b863e4 len=45
r = toku_log_enq_insert(logger, &lsn, NO_FSYNC, fn_aname, txnid, bs_a, bs_b); assert(r==0);
//begin_checkpoint 'x': lsn=9 timestamp=1251309957584197 crc=cd067878 len=29
r = toku_log_begin_checkpoint(logger, &lsn, NO_FSYNC, 1251309957584197); assert(r==0); cp_txnid = lsn.lsn;
//xstillopen 's': lsn=10 txnid=7 parent=0 crc=00061816 len=37
r = toku_log_xstillopen(logger, &lsn, NO_FSYNC, txnid, 0); assert(r==0);
//fassociate 'f': lsn=11 filenum=1 fname={len=4 data="b.db"} crc=a7126035 len=33
r = toku_log_fassociate(logger, &lsn, NO_FSYNC, fn_bname, 0, bs_bname); assert(r==0);
num_fassociate++;
//fassociate 'f': lsn=12 filenum=0 fname={len=4 data="a.db"} crc=a70c5f35 len=33
r = toku_log_fassociate(logger, &lsn, NO_FSYNC, fn_aname, 0, bs_aname); assert(r==0);
num_fassociate++;
//xstillopen 's': lsn=10 txnid=7 parent=0 crc=00061816 len=37 <- obsolete
{
FILENUMS filenums = {0, NULL};
r = toku_log_xstillopen(logger, &lsn, NO_FSYNC, txnid, 0,
0, filenums, 0, 0, 0,
ROLLBACK_NONE, ROLLBACK_NONE, ROLLBACK_NONE);
assert(r==0);
}
num_xstillopen++;
//end_checkpoint 'X': lsn=13 txnid=9 timestamp=1251309957586872 crc=cd285c30 len=37
r = toku_log_end_checkpoint(logger, &lsn, FSYNC, cp_txnid, 1251309957586872); assert(r==0);
r = toku_log_end_checkpoint(logger, &lsn, FSYNC, cp_txnid, 1251309957586872, num_fassociate, num_xstillopen); assert(r==0);
//enq_insert 'I': lsn=14 filenum=1 xid=7 key={len=2 data="b\000"} value={len=2 data="a\000"} crc=40388be4 len=45
r = toku_log_enq_insert(logger, &lsn, NO_FSYNC, fn_bname, txnid, bs_b, bs_a); assert(r==0);
//commit 'C': lsn=15 txnid=7 crc=00016d1e len=29
r = toku_log_commit(logger, &lsn, FSYNC, txnid); assert(r==0);
r = toku_log_xcommit(logger, &lsn, FSYNC, txnid); assert(r==0);
// close logger
r = toku_logger_close(&logger); assert(r==0);
......
newbrt/txn.c
View file @ 1bf7a7a4
......@@ -12,18 +12,20 @@ int toku_txn_begin_txn (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER log
int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER logger, TXNID xid) {
if (logger->is_panicked) return EINVAL;
assert(logger->rollback_cachefile);
TAGMALLOC(TOKUTXN, result);
if (result==0)
return errno;
int r;
LSN first_lsn;
if (xid == 0) {
r = toku_log_xbegin(logger, &result->first_lsn, 0, parent_tokutxn ? parent_tokutxn->txnid64 : 0);
r = toku_log_xbegin(logger, &first_lsn, 0, parent_tokutxn ? parent_tokutxn->txnid64 : 0);
if (r!=0) goto died;
} else
result->first_lsn.lsn = xid;
first_lsn.lsn = xid;
r = toku_omt_create(&result->open_brts);
if (r!=0) goto died;
result->txnid64 = result->first_lsn.lsn;
result->txnid64 = first_lsn.lsn;
XIDS parent_xids;
if (parent_tokutxn==NULL)
parent_xids = xids_get_root_xids();
......@@ -33,13 +35,19 @@ int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGE
goto died;
result->logger = logger;
result->parent = parent_tokutxn;
result->oldest_logentry = result->newest_logentry = 0;
result->rollentry_arena = memarena_create();
result->num_rollentries = 0;
result->num_rollentries_processed = 0;
result->progress_poll_fun = NULL;
result->progress_poll_fun_extra = NULL;
result->spilled_rollback_head = ROLLBACK_NONE;
result->spilled_rollback_tail = ROLLBACK_NONE;
result->spilled_rollback_head_hash = 0;
result->spilled_rollback_tail_hash = 0;
result->current_rollback = ROLLBACK_NONE;
result->current_rollback_hash = 0;
result->num_rollback_nodes = 0;
result->pinned_inprogress_rollback_log = NULL;
if (toku_omt_size(logger->live_txns) == 0) {
assert(logger->oldest_living_xid == TXNID_NONE_LIVING);
......@@ -59,13 +67,9 @@ int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGE
assert(idx > 0);
}
result->rollentry_resident_bytecount=0;
result->rollentry_raw_count = 0;
result->rollentry_filename = 0;
result->rollentry_fd = -1;
result->rollentry_filesize = 0;
result->force_fsync_on_commit = FALSE;
result->has_done_work = FALSE;
result->recovered_from_checkpoint = FALSE;
*tokutxn = result;
return 0;
......@@ -75,6 +79,36 @@ died:
return r;
}
//Used on recovery to recover a transaction.
int
toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info) {
#define COPY_FROM_INFO(field) txn->field = info->field
COPY_FROM_INFO(rollentry_raw_count);
uint32_t i;
for (i = 0; i < info->num_brts; i++) {
BRT brt = info->open_brts[i];
int r = toku_txn_note_brt(txn, brt);
assert(r==0);
}
COPY_FROM_INFO(force_fsync_on_commit );
COPY_FROM_INFO(num_rollback_nodes);
COPY_FROM_INFO(num_rollentries);
CACHEFILE rollback_cachefile = txn->logger->rollback_cachefile;
COPY_FROM_INFO(spilled_rollback_head);
txn->spilled_rollback_head_hash = toku_cachetable_hash(rollback_cachefile,
txn->spilled_rollback_head);
COPY_FROM_INFO(spilled_rollback_tail);
txn->spilled_rollback_tail_hash = toku_cachetable_hash(rollback_cachefile,
txn->spilled_rollback_tail);
COPY_FROM_INFO(current_rollback);
txn->current_rollback_hash = toku_cachetable_hash(rollback_cachefile,
txn->current_rollback);
#undef COPY_FROM_INFO
txn->recovered_from_checkpoint = TRUE;
return 0;
}
// Doesn't close the txn, just performs the commit operations.
int toku_txn_commit_txn(TOKUTXN txn, int nosync, YIELDF yield, void *yieldv,
......@@ -92,13 +126,13 @@ int toku_txn_commit_with_lsn(TOKUTXN txn, int nosync, YIELDF yield, void *yieldv
// panic handled in log_commit
//Child transactions do not actually 'commit'. They promote their changes to parent, so no need to fsync if this txn has a parent.
int do_fsync = !txn->parent && (txn->force_fsync_on_commit || (!nosync && txn->has_done_work));
int do_fsync = !txn->parent && (txn->force_fsync_on_commit || (!nosync && txn->num_rollentries>0));
txn->progress_poll_fun = poll;
txn->progress_poll_fun_extra = poll_extra;
if (release_locks) release_locks(locks_thunk);
r = toku_log_commit(txn->logger, (LSN*)0, do_fsync, txn->txnid64); // exits holding neither of the tokulogger locks.
r = toku_log_xcommit(txn->logger, (LSN*)0, do_fsync, txn->txnid64); // exits holding neither of the tokulogger locks.
if (reacquire_locks) reacquire_locks(locks_thunk);
if (r!=0)
return r;
......
newbrt/txn.h
View file @ 1bf7a7a4
......@@ -7,6 +7,7 @@
int toku_txn_begin_txn (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER logger);
int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER logger, TXNID xid);
int toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info);
int toku_txn_commit_txn (TOKUTXN txn, int nosync, YIELDF yield, void *yieldv,
TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra,
......
newbrt/wbuf.h
View file @ 1bf7a7a4
......@@ -170,7 +170,6 @@ static inline void wbuf_DISKOFF (struct wbuf *w, DISKOFF off) {
static inline void wbuf_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
wbuf_ulonglong(w, b.b);
}
static inline void wbuf_nocrc_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
wbuf_nocrc_ulonglong(w, b.b);
}
......
release/examples/db-insert.c
View file @ 1bf7a7a4
......@@ -60,7 +60,7 @@ int singlex_child = 0; // Do a single transaction, but do all work with a child
int singlex = 0; // Do a single transaction
int singlex_create = 0; // Create the db using the single transaction (only valid if singlex)
int insert1first = 0; // insert 1 before doing the rest
int check_small_rolltmp = 0; // verify that the rollback logs are small (only valid if singlex)
int check_small_rollback = 0; // verify that the rollback logs are small (only valid if singlex)
int do_transactions = 0;
int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
int do_abort = 0;
......@@ -210,14 +210,14 @@ static void benchmark_shutdown (void) {
#endif
if (do_transactions && singlex && !insert1first && (singlex_create || prelock)) {
#if defined(TOKUDB)
//There should be a single 'truncate' in the rolltmp instead of many 'insert' entries.
//There should be a single 'truncate' in the rollback instead of many 'insert' entries.
struct txn_stat *s;
r = tid->txn_stat(tid, &s);
assert(r==0);
//TODO: #1125 Always do the test after performance testing is done.
if (singlex_child) fprintf(stderr, "SKIPPED 'small rolltmp' test for child txn\n");
if (singlex_child) fprintf(stderr, "SKIPPED 'small rollback' test for child txn\n");
else
assert(s->rolltmp_raw_count < 100); // gross test, not worth investigating details
assert(s->rollback_raw_count < 100); // gross test, not worth investigating details
free(s);
//system("ls -l bench.tokudb");
#endif
......@@ -374,7 +374,7 @@ static int print_usage (const char *argv0) {
fprintf(stderr, " --singlex-child (implies -x) Run the whole job as a single transaction, do all work a child of that transaction.\n");
fprintf(stderr, " --finish-child-first Commit/abort child before doing so to parent (no effect if no child).\n");
fprintf(stderr, " --singlex-create (implies --singlex) Create the file using the single transaction (Default is to use a different transaction to create.)\n");
fprintf(stderr, " --check_small_rolltmp (Only valid in --singlex mode) Verify that very little data was saved in the rollback logs.\n");
fprintf(stderr, " --check_small_rollback (Only valid in --singlex mode) Verify that very little data was saved in the rollback logs.\n");
fprintf(stderr, " --prelock Prelock the database.\n");
fprintf(stderr, " --prelockflag Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
fprintf(stderr, " --abort Abort the singlex after the transaction is over. (Requires --singlex.)\n");
......@@ -463,8 +463,8 @@ int main (int argc, const char *argv[]) {
singlex = 1;
} else if (strcmp(arg, "--insert1first") == 0) {
insert1first = 1;
} else if (strcmp(arg, "--check_small_rolltmp") == 0) {
check_small_rolltmp = 1;
} else if (strcmp(arg, "--check_small_rollback") == 0) {
check_small_rollback = 1;
} else if (strcmp(arg, "--xcount") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
items_per_transaction = strtoll(argv[++i], &endptr, 10); assert(*endptr == 0);
......@@ -560,13 +560,13 @@ int main (int argc, const char *argv[]) {
printf("insertions of %d per batch%s\n", items_per_iteration, do_transactions ? " (with transactions)" : "");
}
#if !defined TOKUDB
if (check_small_rolltmp) {
fprintf(stderr, "--check_small_rolltmp only works on the TokuDB (not BDB)\n");
if (check_small_rollback) {
fprintf(stderr, "--check_small_rollback only works on the TokuDB (not BDB)\n");
return print_usage(argv[0]);
}
#endif
if (check_small_rolltmp && !singlex) {
fprintf(stderr, "--check_small_rolltmp requires --singlex\n");
if (check_small_rollback && !singlex) {
fprintf(stderr, "--check_small_rollback requires --singlex\n");
return print_usage(argv[0]);
}
benchmark_setup();
......
src/tests/Makefile
View file @ 1bf7a7a4
......@@ -332,6 +332,7 @@ endif
mkdir dir.$*.c.tdb.recover && \
cp dir.$*.c.tdb/tokudb.directory dir.$*.c.tdb.recover/ && \
cp dir.$*.c.tdb/tokudb.environment dir.$*.c.tdb.recover/ && \
cp dir.$*.c.tdb/tokudb.rollback dir.$*.c.tdb.recover/ && \
cp dir.$*.c.tdb/*.tokulog dir.$*.c.tdb.recover/ && \
echo doing recovery &&\
$(VGRIND) ../../newbrt/tdb-recover dir.$*.c.tdb.recover dir.$*.c.tdb.recover && \
......
src/tests/bug1381.c
View file @ 1bf7a7a4
......@@ -60,13 +60,13 @@ static void do_1381_maybe_lock (int do_table_lock, u_int64_t *raw_count) {
}
r = txn->txn_stat(txn, &s2); CKERR(r);
//printf("Raw counts = %" PRId64 ", %" PRId64 "\n", s1->rolltmp_raw_count, s2->rolltmp_raw_count);
//printf("Raw counts = %" PRId64 ", %" PRId64 "\n", s1->rollback_raw_count, s2->rollback_raw_count);
*raw_count = s2->rolltmp_raw_count - s1->rolltmp_raw_count;
*raw_count = s2->rollback_raw_count - s1->rollback_raw_count;
if (do_table_lock) {
assert(s1->rolltmp_raw_count == s2->rolltmp_raw_count);
assert(s1->rollback_raw_count == s2->rollback_raw_count);
} else {
assert(s1->rolltmp_raw_count < s2->rolltmp_raw_count);
assert(s1->rollback_raw_count < s2->rollback_raw_count);
}
toku_free(s1); toku_free(s2);
......
src/tests/diskfull.c
View file @ 1bf7a7a4
......@@ -91,7 +91,7 @@ do_db_work(void) {
}
if (did_fail) goto shutdown2;
// Put an extra item in so that the rolltmp file will be created.
// Put an extra item in
r=env->txn_begin(env, 0, &tid, 0); assert(r==0);
r=db->put(db, tid, dbt_init(&key, "a", 2), dbt_init(&data, "b", 2), 0); DOERR(r);
if (did_fail) {
......
src/tests/stat64.c
View file @ 1bf7a7a4
......@@ -21,7 +21,7 @@ test_stat64 (unsigned int N) {
DB_TXN *txn;
r = db_env_create(&env, 0); CKERR(r);
r = env->set_cachesize(env, 0, 10*1000000, 1);
r = env->set_cachesize(env, 0, 20*1000000, 1);
r = env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_create(&db, env, 0); CKERR(r);
......@@ -38,6 +38,10 @@ test_stat64 (unsigned int N) {
unsigned int i;
u_int64_t dsize=0;
for (i=0; i<N; i++) {
if (verbose>1 && i % (1<<14) == 0) {
printf("%s(total=%u) inserted %u so far\n", __FILE__, N, i);
fflush(stdout);
}
char hello[30], there[30];
snprintf(hello, sizeof(hello), "hello%8d", i);
snprintf(there, sizeof(there), "there%d", i);
......
src/tests/test1324.c deleted 100644 → 0
View file @ bf8e181e
This diff is collapsed.
src/ydb.c
View file @ 1bf7a7a4
This diff is collapsed.
toku_include/memory.h
View file @ 1bf7a7a4
......@@ -21,7 +21,8 @@ enum typ_tag { TYP_BRTNODE = 0xdead0001,
TYP_GPMA,
TYP_TOKULOGGER,
TYP_TOKUTXN,
TYP_LEAFENTRY
TYP_LEAFENTRY,
TYP_ROLLBACK_LOG_NODE
};
/* Everything should call toku_malloc() instead of malloc(), and toku_calloc() instead of calloc() */
......
toku_include/toku_list.h
View file @ 1bf7a7a4
......@@ -78,7 +78,7 @@ static inline void toku_list_move(struct toku_list *newhead, struct toku_list *o
// Note: Need the extra level of parens in these macros so that
// toku_list_struct(h, foo, b)->zot
// will work right. Otherwise the type cast will try to include ->zot, and it will be all messed up.
#if defined(__GNUC__) && __GNUC__ >= 4
#if (defined(__GNUC__) && __GNUC__ >= 4) || defined(__builtin_offsetof)
#define toku_list_struct(p, t, f) ((t*)((char*)(p) - __builtin_offsetof(t, f)))
#else
#define toku_list_struct(p, t, f) ((t*)((char*)(p) - ((char*)&((t*)0)->f)))
......
windows/misc.h
View file @ 1bf7a7a4
......@@ -7,6 +7,7 @@ extern "C" {
#include "toku_os.h"
#include <sys/stat.h>
#include <stddef.h>
//These are functions that really exist in windows but are named
//something else.
......@@ -82,6 +83,9 @@ int mkstemp(char * ttemplate);
toku_off_t ftello(FILE *stream);
#define __builtin_offsetof(type, member) offsetof(type, member)
#if defined(__cplusplus)
};
#endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7