Commit 576d0976 authored by Christian Rober's avatar Christian Rober Committed by Yoni Fogel

[t:4570] Merging verification tool and partial removal of deserialization custom errors.

git-svn-id: file:///svn/toku/tokudb@44286 c7de825b-a66e-492c-adef-691d508d4ae1
parent 7c394a4e
...@@ -119,6 +119,7 @@ enum { ...@@ -119,6 +119,7 @@ enum {
TOKUDB_TRY_AGAIN = -100012, TOKUDB_TRY_AGAIN = -100012,
TOKUDB_NEEDS_REPAIR = -100013, TOKUDB_NEEDS_REPAIR = -100013,
TOKUDB_CURSOR_CONTINUE = -100014, TOKUDB_CURSOR_CONTINUE = -100014,
TOKUDB_BAD_CHECKSUM = -100015,
DONTUSE_I_JUST_PUT_THIS_HERE_SO_I_COULD_HAVE_A_COMMA_AFTER_EACH_ITEM DONTUSE_I_JUST_PUT_THIS_HERE_SO_I_COULD_HAVE_A_COMMA_AFTER_EACH_ITEM
}; };
...@@ -269,6 +270,7 @@ static void print_defines (void) { ...@@ -269,6 +270,7 @@ static void print_defines (void) {
dodefine(TOKUDB_TRY_AGAIN); dodefine(TOKUDB_TRY_AGAIN);
dodefine(TOKUDB_NEEDS_REPAIR); dodefine(TOKUDB_NEEDS_REPAIR);
dodefine(TOKUDB_CURSOR_CONTINUE); dodefine(TOKUDB_CURSOR_CONTINUE);
dodefine(TOKUDB_BAD_CHECKSUM);
/* LOADER flags */ /* LOADER flags */
printf("/* LOADER flags */\n"); printf("/* LOADER flags */\n");
......
...@@ -37,6 +37,7 @@ set(FT_SOURCES ...@@ -37,6 +37,7 @@ set(FT_SOURCES
ftloader-callback.c ftloader-callback.c
ft_msg.c ft_msg.c
ft_node-serialize.c ft_node-serialize.c
ft-node-deserialize.c
ft-ops.c ft-ops.c
ft-pwrite.c ft-pwrite.c
ft-serialize.c ft-serialize.c
...@@ -111,6 +112,7 @@ set(bins ...@@ -111,6 +112,7 @@ set(bins
ftdump ftdump
tdb_logprint tdb_logprint
tdb-recover tdb-recover
ftverify
) )
foreach(bin ${bins}) foreach(bin ${bins})
add_executable(${bin} ${bin}.c) add_executable(${bin} ${bin}.c)
...@@ -122,6 +124,10 @@ foreach(bin ${bins}) ...@@ -122,6 +124,10 @@ foreach(bin ${bins})
target_link_libraries(${bin}_static ft_static ${LIBTOKUPORTABILITY_STATIC}) target_link_libraries(${bin}_static ft_static ${LIBTOKUPORTABILITY_STATIC})
endforeach(bin) endforeach(bin)
# link in math.h library just for this tool.
target_link_libraries(ftverify m)
target_link_libraries(ftverify_static m)
install( install(
TARGETS ftdump_static TARGETS ftdump_static
DESTINATION bin DESTINATION bin
......
...@@ -518,9 +518,68 @@ int toku_serialize_rollback_log_to (int fd, BLOCKNUM blocknum, ROLLBACK_LOG_NODE ...@@ -518,9 +518,68 @@ int toku_serialize_rollback_log_to (int fd, BLOCKNUM blocknum, ROLLBACK_LOG_NODE
FT h, int n_workitems, int n_threads, FT h, int n_workitems, int n_threads,
BOOL for_checkpoint); BOOL for_checkpoint);
int toku_deserialize_rollback_log_from (int fd, BLOCKNUM blocknum, u_int32_t fullhash, ROLLBACK_LOG_NODE *logp, FT h); int toku_deserialize_rollback_log_from (int fd, BLOCKNUM blocknum, u_int32_t fullhash, ROLLBACK_LOG_NODE *logp, FT h);
enum deserialize_error_code toku_deserialize_bp_from_disk(FTNODE node, FTNODE_DISK_DATA ndd, int childnum, int fd, struct ftnode_fetch_extra* bfe); int toku_deserialize_bp_from_disk(FTNODE node, FTNODE_DISK_DATA ndd, int childnum, int fd, struct ftnode_fetch_extra* bfe);
enum deserialize_error_code toku_deserialize_bp_from_compressed(FTNODE node, int childnum, DESCRIPTOR desc, ft_compare_func cmp); int toku_deserialize_bp_from_compressed(FTNODE node, int childnum, DESCRIPTOR desc, ft_compare_func cmp);
enum deserialize_error_code toku_deserialize_ftnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/, FTNODE *ftnode, FTNODE_DISK_DATA* ndd, struct ftnode_fetch_extra* bfe); int toku_deserialize_ftnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/, FTNODE *ftnode, FTNODE_DISK_DATA* ndd, struct ftnode_fetch_extra* bfe);
//////////////// <CER> TODO: Move these function declarations
int
deserialize_ft_from_fd_into_rbuf(int fd,
toku_off_t offset_of_header,
struct rbuf *rb,
u_int64_t *checkpoint_count,
LSN *checkpoint_lsn,
u_int32_t * version_p,
enum deserialize_error_code *e);
enum deserialize_error_code
deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ft, uint32_t version);
int
read_block_from_fd_into_rbuf(
int fd,
BLOCKNUM blocknum,
FT h,
struct rbuf *rb
);
int
read_compressed_sub_block(struct rbuf *rb, struct sub_block *sb);
int
verify_ftnode_sub_block (struct sub_block *sb);
void
just_decompress_sub_block(struct sub_block *sb);
/* Beginning of ft-node-deserialize.c helper functions. */
//
inline void
initialize_ftnode(FTNODE node, BLOCKNUM blocknum);
//
inline int
read_and_check_magic(struct rbuf *rb);
//
inline int
read_and_check_version(FTNODE node, struct rbuf *rb);
//
inline void
read_node_info(FTNODE node, struct rbuf *rb, int version);
//
inline void
allocate_and_read_partition_offsets(FTNODE node, struct rbuf *rb, FTNODE_DISK_DATA *ndd);
//
inline int
check_node_info_checksum(struct rbuf *rb);
//////////////// <CER>
unsigned int toku_serialize_ftnode_size(FTNODE node); /* How much space will it take? */ unsigned int toku_serialize_ftnode_size(FTNODE node); /* How much space will it take? */
int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len); int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
...@@ -983,6 +1042,6 @@ toku_ft_node_put_cmd ( ...@@ -983,6 +1042,6 @@ toku_ft_node_put_cmd (
void toku_flusher_thread_set_callback(void (*callback_f)(int, void*), void* extra); void toku_flusher_thread_set_callback(void (*callback_f)(int, void*), void* extra);
enum deserialize_error_code toku_upgrade_subtree_estimates_to_stat64info(int fd, FT h); int toku_upgrade_subtree_estimates_to_stat64info(int fd, FT h);
#endif #endif
/* -*- mode: C; c-basic-offset: 4 -*- */
// vim: expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id: ft-serialize.c 43686 2012-05-18 23:21:00Z leifwalsh $"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <ft-internal.h>
#include <db.h>
////////////////////////////////////////
// 1. Sets initial values
//
inline void
initialize_ftnode(FTNODE node, BLOCKNUM blocknum)
{
node->fullhash = 0xDEADBEEF; // <CER> Is this 'spoof' ok?
node->thisnodename = blocknum;
node->dirty = 0;
node->bp = NULL;
// <CER> Can we use this initialization as a correctness assert in
// a later function?
node->layout_version_read_from_disk = 0;
}
// TODO:
/************************
* In other deserialization code, we check the rb size member. We
* verify that it is greater than or equal to 24. Ignoring this magic
* number for a moment, should we put this check in its own function? *
*************************/
/////////////////////////////////////////////////////////////////////
// 2. Read and check the 'magic' bytes on disk. Returns an error if
// the magic does not match.
//
inline int
read_and_check_magic(struct rbuf *rb)
{
int r = 0;
bytevec magic;
rbuf_literal_bytes(rb, &magic, 8);
if (memcmp(magic, "tokuleaf", 8)!=0 &&
memcmp(magic, "tokunode", 8)!=0) {
r = DB_BADFORMAT; // TODO: Return more meaningful error.
}
return r;
}
////////////////////
// 3.
inline int
read_and_check_version(FTNODE node, struct rbuf *rb)
{
int r = 0;
int version = rbuf_int(rb);
node->layout_version_read_from_disk = version;
if (version < FT_LAYOUT_MIN_SUPPORTED_VERSION) {
r = 1; // TODO: Better error reporting.
}
return r;
}
////////////////////
// 4.
inline void
read_node_info(FTNODE node, struct rbuf *rb, int version)
{
node->layout_version = version;
node->layout_version_original = rbuf_int(rb);
node->build_id = rbuf_int(rb);
node->n_children = rbuf_int(rb);
}
////////////////////
// 5.
// <CER> Should these be two seperate functions?
inline void
allocate_and_read_partition_offsets(FTNODE node, struct rbuf *rb, FTNODE_DISK_DATA *ndd)
{
XMALLOC_N(node->n_children, node->bp);
// TODO: Fix this to use xmalloc_n
*ndd = toku_xmalloc(node->n_children * sizeof(**ndd));
// Read the partition locations.
for (int i = 0; i < node->n_children; i++) {
BP_START(*ndd, i) = rbuf_int(rb);
BP_SIZE (*ndd, i) = rbuf_int(rb);
}
}
////////////////////
// 6. Check the node info checksum.
inline int
check_node_info_checksum(struct rbuf *rb)
{
int r = 0;
// Verify checksum of header stored.
u_int32_t checksum = x1764_memory(rb->buf, rb->ndone);
u_int32_t stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
// TODO: dump_bad_block(rb->buf, rb->size);
r = TOKUDB_BAD_CHECKSUM;
}
return r;
}
// Two functions, at this point, SHOULD be split into sperate read and
// checksum check calls:
// 1. read_and_decompress_sub_block - this is generic, used elsewhere.
// So....
// a. call read_compressed_sub_block() directly
// then....
//
// 2. deserialize_ftnode_info() - this actually reads in the node
// 'info' fields, such as height, nodesize, etc.
/////////////////////////////////
// ?. ----
// setup_ftnode_paritions() -
// calls :
// a. update_bfe_using_ftnode
// b. setup_partitions_using_bfe()
/////////////////////////////////
// ?. partition from sub-block deserialization.
// A.decompress_and_deserialize_worker()
// calls:
// a. read_and_decompress_sub_block (SEE ABOVE) -
// -calls:
// -ii. read_compressed_sub_block() - Returns checksum error AND reads out buffer.
// b. deserialize_ftnode_partition()
// -calls:
// -ii. verify_ftnode_sub_block() - JUST verifies checksum.
//
// OR
//
// B. check_and_copy_compressed_sub_block_worker
// calls:
// a. read_compressed_sub_block() - Returns checksum AND reads out of buffer.
/////////////////////////////////
// 1. first calls verify_ftnode_sub_block() which must be refactored
// into two seperate calls.
...@@ -138,7 +138,7 @@ exit: ...@@ -138,7 +138,7 @@ exit:
} }
// We only deserialize brt header once and then share everything with all the brts. // We only deserialize brt header once and then share everything with all the brts.
static enum deserialize_error_code enum deserialize_error_code
deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version) deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
{ {
enum deserialize_error_code e = DS_OK; enum deserialize_error_code e = DS_OK;
...@@ -423,7 +423,7 @@ serialize_ft_min_size (u_int32_t version) { ...@@ -423,7 +423,7 @@ serialize_ft_min_size (u_int32_t version) {
// //
// TOKUDB_DICTIONARY_NO_HEADER means we can overwrite everything in the // TOKUDB_DICTIONARY_NO_HEADER means we can overwrite everything in the
// file AND the header is useless // file AND the header is useless
static int int
deserialize_ft_from_fd_into_rbuf(int fd, deserialize_ft_from_fd_into_rbuf(int fd,
toku_off_t offset_of_header, toku_off_t offset_of_header,
struct rbuf *rb, struct rbuf *rb,
......
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment