Skip to content

M
mariadb
Commit cb030cb4 authored by John Esmet's avatar John Esmet
Browse files
Options

fixes #194 Move x1764 to util/ and make the public API consistent with 

the rest of the code.
parent 79b67bed
Show whitespace changes
Inline Side-by-side
Showing with 84 additions and 94 deletions
ft/CMakeLists.txt
View file @ cb030cb4
......@@ -69,7 +69,6 @@ set(FT_SOURCES
txn_child_manager
txn_manager
ule
x1764
xids
ybt
"${CMAKE_CURRENT_BINARY_DIR}/log_code"
......
ft/block_table.cc
View file @ cb030cb4
......@@ -580,7 +580,7 @@ void toku_serialize_translation_to_wbuf(BLOCK_TABLE bt, int fd, struct wbuf *w,
wbuf_DISKOFF(w, t->block_translation[i].u.diskoff);
wbuf_DISKOFF(w, t->block_translation[i].size);
}
uint32_t checksum = x1764_finish(&w->checksum);
uint32_t checksum = toku_x1764_finish(&w->checksum);
wbuf_int(w, checksum);
*address = t->block_translation[b.b].u.diskoff;
*size = size_translation;
......@@ -915,7 +915,7 @@ translation_deserialize_from_buffer(struct translation *t, // destination int
t->type = TRANSLATION_CHECKPOINTED;
{
// check the checksum
uint32_t x1764 = x1764_memory(translation_buffer, size_on_disk - 4);
uint32_t x1764 = toku_x1764_memory(translation_buffer, size_on_disk - 4);
uint64_t offset = size_on_disk - 4;
//printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
uint32_t stored_x1764 = toku_dtoh32(*(int*)(translation_buffer + offset));
......
ft/ft-node-deserialize.cc
View file @ cb030cb4
......@@ -193,7 +193,7 @@ check_node_info_checksum(struct rbuf *rb)
{
int r = 0;
// Verify checksum of header stored.
uint32_t checksum = x1764_memory(rb->buf, rb->ndone);
uint32_t checksum = toku_x1764_memory(rb->buf, rb->ndone);
uint32_t stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
......@@ -229,7 +229,7 @@ check_legacy_end_checksum(struct rbuf *rb)
{
int r = 0;
uint32_t expected_xsum = rbuf_int(rb);
uint32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size - 4);
if (expected_xsum != actual_xsum) {
r = TOKUDB_BAD_CHECKSUM;
}
......
ft/ft-serialize.cc
View file @ cb030cb4
......@@ -132,7 +132,7 @@ toku_serialize_descriptor_contents_to_fd(int fd, const DESCRIPTOR desc, DISKOFF
toku_serialize_descriptor_contents_to_wbuf(&w, desc);
{
//Add checksum
uint32_t checksum = x1764_finish(&w.checksum);
uint32_t checksum = toku_x1764_finish(&w.checksum);
wbuf_int(&w, checksum);
}
lazy_assert(w.ndone==w.size);
......@@ -185,7 +185,7 @@ deserialize_descriptor_from(int fd, BLOCK_TABLE bt, DESCRIPTOR desc, int layout_
}
{
// check the checksum
uint32_t x1764 = x1764_memory(dbuf, size-4);
uint32_t x1764 = toku_x1764_memory(dbuf, size-4);
//printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
uint32_t stored_x1764 = toku_dtoh32(*(int*)(dbuf + size-4));
if (x1764 != stored_x1764) {
......@@ -624,7 +624,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd,
//Verify checksum (FT_LAYOUT_VERSION_13 or later, when checksum function changed)
uint32_t calculated_x1764;
calculated_x1764 = x1764_memory(rb->buf, rb->size-4);
calculated_x1764 = toku_x1764_memory(rb->buf, rb->size-4);
uint32_t stored_x1764;
stored_x1764 = toku_dtoh32(*(int*)(rb->buf+rb->size-4));
if (calculated_x1764 != stored_x1764) {
......@@ -804,7 +804,7 @@ void toku_serialize_ft_to_wbuf (
wbuf_char(wbuf, (unsigned char) h->compression_method);
wbuf_MSN(wbuf, h->highest_unused_msn_for_upgrade);
wbuf_MSN(wbuf, h->max_msn_in_ft);
uint32_t checksum = x1764_finish(&wbuf->checksum);
uint32_t checksum = toku_x1764_finish(&wbuf->checksum);
wbuf_int(wbuf, checksum);
lazy_assert(wbuf->ndone == wbuf->size);
}
......
ft/ft_node-serialize.cc
View file @ cb030cb4
......@@ -279,7 +279,7 @@ serialize_node_header(FTNODE node, FTNODE_DISK_DATA ndd, struct wbuf *wbuf) {
wbuf_nocrc_int(wbuf, BP_SIZE (ndd, i)); // and the size
}
// checksum the header
uint32_t end_to_end_checksum = x1764_memory(wbuf->buf, wbuf_get_woffset(wbuf));
uint32_t end_to_end_checksum = toku_x1764_memory(wbuf->buf, wbuf_get_woffset(wbuf));
wbuf_nocrc_int(wbuf, end_to_end_checksum);
invariant(wbuf->ndone == wbuf->size);
}
......@@ -357,7 +357,7 @@ serialize_ftnode_partition(FTNODE node, int i, struct sub_block *sb) {
bd->serialize_to_wbuf(&wb);
}
uint32_t end_to_end_checksum = x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
uint32_t end_to_end_checksum = toku_x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
wbuf_nocrc_int(&wb, end_to_end_checksum);
invariant(wb.ndone == wb.size);
invariant(sb->uncompressed_size==wb.ndone);
......@@ -401,7 +401,7 @@ compress_ftnode_sub_block(struct sub_block *sb, enum toku_compression_method met
extra[1] = toku_htod32(sb->uncompressed_size);
// now checksum the entire thing
sb->compressed_size += 8; // now add the eight bytes that we saved for the sizes
sb->xsum = x1764_memory(sb->compressed_ptr,sb->compressed_size);
sb->xsum = toku_x1764_memory(sb->compressed_ptr,sb->compressed_size);
//
// This is the end result for Dr. No and forward. For ftnodes, sb->compressed_ptr contains
......@@ -465,7 +465,7 @@ static void serialize_ftnode_info(FTNODE node,
}
}
uint32_t end_to_end_checksum = x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
uint32_t end_to_end_checksum = toku_x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
wbuf_nocrc_int(&wb, end_to_end_checksum);
invariant(wb.ndone == wb.size);
invariant(sb->uncompressed_size==wb.ndone);
......@@ -1248,7 +1248,7 @@ read_compressed_sub_block(struct rbuf *rb, struct sub_block *sb)
rbuf_literal_bytes(rb, cp, sb->compressed_size);
sb->xsum = rbuf_int(rb);
// let's check the checksum
uint32_t actual_xsum = x1764_memory((char *)sb->compressed_ptr-8, 8+sb->compressed_size);
uint32_t actual_xsum = toku_x1764_memory((char *)sb->compressed_ptr-8, 8+sb->compressed_size);
if (sb->xsum != actual_xsum) {
r = TOKUDB_BAD_CHECKSUM;
}
......@@ -1293,7 +1293,7 @@ verify_ftnode_sub_block (struct sub_block *sb)
// first verify the checksum
uint32_t data_size = sb->uncompressed_size - 4; // checksum is 4 bytes at end
uint32_t stored_xsum = toku_dtoh32(*((uint32_t *)((char *)sb->uncompressed_ptr + data_size)));
uint32_t actual_xsum = x1764_memory(sb->uncompressed_ptr, data_size);
uint32_t actual_xsum = toku_x1764_memory(sb->uncompressed_ptr, data_size);
if (stored_xsum != actual_xsum) {
dump_bad_block((Bytef *) sb->uncompressed_ptr, sb->uncompressed_size);
r = TOKUDB_BAD_CHECKSUM;
......@@ -1656,7 +1656,7 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
}
uint32_t checksum;
checksum = x1764_memory(rb->buf, rb->ndone);
checksum = toku_x1764_memory(rb->buf, rb->ndone);
uint32_t stored_checksum;
stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
......@@ -1682,7 +1682,7 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
sb_node_info.xsum = rbuf_int(rb);
// let's check the checksum
uint32_t actual_xsum;
actual_xsum = x1764_memory((char *)sb_node_info.compressed_ptr-8, 8+sb_node_info.compressed_size);
actual_xsum = toku_x1764_memory((char *)sb_node_info.compressed_ptr-8, 8+sb_node_info.compressed_size);
if (sb_node_info.xsum != actual_xsum) {
r = TOKUDB_BAD_CHECKSUM;
goto cleanup;
......@@ -1953,7 +1953,7 @@ deserialize_and_upgrade_internal_node(FTNODE node,
// still have the pointer to the buffer).
if (version >= FT_FIRST_LAYOUT_VERSION_WITH_END_TO_END_CHECKSUM) {
uint32_t expected_xsum = toku_dtoh32(*(uint32_t*)(rb->buf+rb->size-4)); // 27. checksum
uint32_t actual_xsum = x1764_memory(rb->buf, rb->size-4);
uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size-4);
if (expected_xsum != actual_xsum) {
fprintf(stderr, "%s:%d: Bad checksum: expected = %" PRIx32 ", actual= %" PRIx32 "\n",
__FUNCTION__,
......@@ -2093,7 +2093,7 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
// Checksum (end to end) is only on version 14
if (has_end_to_end_checksum) {
uint32_t expected_xsum = rbuf_int(rb); // 17. checksum
uint32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size - 4);
if (expected_xsum != actual_xsum) {
fprintf(stderr, "%s:%d: Bad checksum: expected = %" PRIx32 ", actual= %" PRIx32 "\n",
__FUNCTION__,
......@@ -2292,7 +2292,7 @@ deserialize_ftnode_from_rbuf(
}
// verify checksum of header stored
uint32_t checksum;
checksum = x1764_memory(rb->buf, rb->ndone);
checksum = toku_x1764_memory(rb->buf, rb->ndone);
uint32_t stored_checksum;
stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
......@@ -2684,7 +2684,7 @@ serialize_uncompressed_block_to_memory(char * uncompressed_buf,
// compute the header checksum and serialize it
uint32_t header_length = (char *)ptr - (char *)compressed_buf;
uint32_t xsum = x1764_memory(compressed_buf, header_length);
uint32_t xsum = toku_x1764_memory(compressed_buf, header_length);
*ptr = toku_htod32(xsum);
uint32_t padded_len = roundup_to_multiple(512, header_len + compressed_len);
......@@ -2856,7 +2856,7 @@ decompress_from_raw_block_into_rbuf(uint8_t *raw_block, size_t raw_block_size, s
{ // verify the header checksum
uint32_t header_length = node_header_overhead + sub_block_header_size(n_sub_blocks);
invariant(header_length <= raw_block_size);
uint32_t xsum = x1764_memory(raw_block, header_length);
uint32_t xsum = toku_x1764_memory(raw_block, header_length);
uint32_t stored_xsum = toku_dtoh32(*(uint32_t *)(raw_block + header_length));
if (xsum != stored_xsum) {
r = TOKUDB_BAD_CHECKSUM;
......
ft/ftloader.cc
View file @ cb030cb4
......@@ -101,7 +101,9 @@ PATENT RIGHTS GRANT:
#include <toku_assert.h>
#include <string.h>
#include <fcntl.h>
#include "x1764.h"
#include <util/x1764.h>
#include "ftloader-internal.h"
#include "ft-internal.h"
#include "sub_block.h"
......@@ -2621,7 +2623,7 @@ static int toku_loader_write_ft_from_q (FTLOADER bl,
char *XMALLOC_N(desc_size, buf);
wbuf_init(&wbuf, buf, desc_size);
toku_serialize_descriptor_contents_to_wbuf(&wbuf, descriptor);
uint32_t checksum = x1764_finish(&wbuf.checksum);
uint32_t checksum = toku_x1764_finish(&wbuf.checksum);
wbuf_int(&wbuf, checksum);
invariant(wbuf.ndone==desc_size);
r = toku_os_write(out.fd, wbuf.buf, wbuf.ndone);
......@@ -2992,7 +2994,7 @@ static int write_translation_table (struct dbout *out, long long *off_of_transla
putbuf_int64(&ttable, out->translation[i].off);
putbuf_int64(&ttable, out->translation[i].size);
}
unsigned int checksum = x1764_memory(ttable.buf, ttable.off);
unsigned int checksum = toku_x1764_memory(ttable.buf, ttable.off);
putbuf_int32(&ttable, checksum);
// pad it to 512 zeros
long long encoded_length = ttable.off;
......
ft/ftverify.cc
View file @ cb030cb4
......@@ -98,7 +98,6 @@ PATENT RIGHTS GRANT:
#include "ft-internal.h"
#include "ft_layout_version.h"
#include "block_table.h"
#include "x1764.h"
#include "rbuf.h"
#include "sub_block.h"
......
ft/leafentry.h
View file @ cb030cb4
......@@ -96,11 +96,10 @@ PATENT RIGHTS GRANT:
#include <toku_portability.h>
#include <util/mempool.h>
#include <util/omt.h>
#include "txn_manager.h"
#include "rbuf.h"
#include "x1764.h"
#include "omt.h"
/*
Memory format of packed leaf entry
......
ft/log.h
View file @ cb030cb4
......@@ -94,19 +94,17 @@ PATENT RIGHTS GRANT:
#include <toku_portability.h>
#include <errno.h>
#include <db.h>
#include "fttypes.h"
#include "memory.h"
#include "x1764.h"
struct roll_entry;
#include "logger.h"
#include "rollback.h"
#include "recover.h"
#include "txn.h"
struct roll_entry;
static inline int toku_copy_BYTESTRING(BYTESTRING *target, BYTESTRING val) {
target->len = val.len;
target->data = (char *) toku_memdup(val.data, (size_t)val.len);
......@@ -117,6 +115,7 @@ static inline int toku_copy_BYTESTRING(BYTESTRING *target, BYTESTRING val) {
}
static inline void toku_free_TXNID(TXNID txnid __attribute__((__unused__))) {}
static inline void toku_free_TXNID_PAIR(TXNID_PAIR txnid __attribute__((__unused__))) {}
static inline void toku_free_LSN(LSN lsn __attribute__((__unused__))) {}
static inline void toku_free_uint64_t(uint64_t u __attribute__((__unused__))) {}
static inline void toku_free_uint32_t(uint32_t u __attribute__((__unused__))) {}
......
ft/logformat.cc
View file @ cb030cb4
......@@ -536,7 +536,7 @@ generate_log_writer (void) {
if (strcmp(field_type->name, "timestamp") == 0)
fprintf(cf, " if (timestamp == 0) timestamp = toku_get_timestamp();\n");
fprintf(cf, " wbuf_nocrc_%s(&wbuf, %s);\n", field_type->type, field_type->name));
fprintf(cf, " wbuf_nocrc_int(&wbuf, x1764_memory(wbuf.buf, wbuf.ndone));\n");
fprintf(cf, " wbuf_nocrc_int(&wbuf, toku_x1764_memory(wbuf.buf, wbuf.ndone));\n");
fprintf(cf, " wbuf_nocrc_int(&wbuf, buflen);\n");
fprintf(cf, " assert(wbuf.ndone==buflen);\n");
fprintf(cf, " logger->inbuf.n_in_buf += buflen;\n");
......@@ -558,7 +558,7 @@ generate_log_reader (void) {
fprintf(cf, " uint32_t checksum_in_file, len_in_file;\n");
fprintf(cf, " r=toku_fread_uint32_t_nocrclen(infile, &checksum_in_file); actual_len+=4; if (r!=0) return r;\n");
fprintf(cf, " r=toku_fread_uint32_t_nocrclen(infile, &len_in_file); actual_len+=4; if (r!=0) return r;\n");
fprintf(cf, " if (checksum_in_file!=x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
fprintf(cf, " if (checksum_in_file!=toku_x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
fprintf(cf, " return 0;\n");
fprintf(cf, "}\n\n");
});
......@@ -568,12 +568,12 @@ generate_log_reader (void) {
fprintf(cf, " uint32_t len1; int r;\n");
fprintf(cf, " uint32_t ignorelen=0;\n");
fprintf(cf, " struct x1764 checksum;\n");
fprintf(cf, " x1764_init(&checksum);\n");
fprintf(cf, " toku_x1764_init(&checksum);\n");
fprintf(cf, " r = toku_fread_uint32_t(infile, &len1, &checksum, &ignorelen); if (r!=0) return r;\n");
fprintf(cf, " int cmd=fgetc(infile);\n");
fprintf(cf, " if (cmd==EOF) return EOF;\n");
fprintf(cf, " char cmdchar = (char)cmd;\n");
fprintf(cf, " x1764_add(&checksum, &cmdchar, 1);\n");
fprintf(cf, " toku_x1764_add(&checksum, &cmdchar, 1);\n");
fprintf(cf, " le->cmd=(enum lt_cmd)cmd;\n");
fprintf(cf, " switch ((enum lt_cmd)cmd) {\n");
DO_LOGTYPES(lt, {
......@@ -639,14 +639,14 @@ generate_logprint (void) {
fprintf(pf, " uint32_t len1, crc_in_file;\n");
fprintf(pf, " uint32_t ignorelen=0;\n");
fprintf(pf, " struct x1764 checksum;\n");
fprintf(pf, " x1764_init(&checksum);\n");
fprintf(pf, " toku_x1764_init(&checksum);\n");
fprintf(pf, " r=toku_fread_uint32_t(f, &len1, &checksum, &ignorelen);\n");
fprintf(pf, " if (r==EOF) return EOF;\n");
fprintf(pf, " cmd=fgetc(f);\n");
fprintf(pf, " if (cmd==EOF) return DB_BADFORMAT;\n");
fprintf(pf, " uint32_t len_in_file, len=1+4; // cmd + len1\n");
fprintf(pf, " char charcmd = (char)cmd;\n");
fprintf(pf, " x1764_add(&checksum, &charcmd, 1);\n");
fprintf(pf, " toku_x1764_add(&checksum, &charcmd, 1);\n");
fprintf(pf, " switch ((enum lt_cmd)cmd) {\n");
DO_LOGTYPES(lt, { if (strlen(lt->name)>maxnamelen) maxnamelen=strlen(lt->name); });
DO_LOGTYPES(lt, {
......@@ -664,7 +664,7 @@ generate_logprint (void) {
fprintf(pf, "); if (r!=0) return r;\n");
});
fprintf(pf, " {\n");
fprintf(pf, " uint32_t actual_murmur = x1764_finish(&checksum);\n");
fprintf(pf, " uint32_t actual_murmur = toku_x1764_finish(&checksum);\n");
fprintf(pf, " r = toku_fread_uint32_t_nocrclen (f, &crc_in_file); len+=4; if (r!=0) return r;\n");
fprintf(pf, " fprintf(outf, \" crc=%%08x\", crc_in_file);\n");
fprintf(pf, " if (crc_in_file!=actual_murmur) fprintf(outf, \" checksum=%%08x\", actual_murmur);\n");
......
ft/logger.cc
View file @ cb030cb4
......@@ -933,7 +933,7 @@ int toku_fread_uint8_t (FILE *f, uint8_t *v, struct x1764 *mm, uint32_t *len) {
int vi=fgetc(f);
if (vi==EOF) return -1;
uint8_t vc=(uint8_t)vi;
x1764_add(mm, &vc, 1);
toku_x1764_add(mm, &vc, 1);
(*len)++;
*v = vc;
return 0;
......
ft/recover.cc
View file @ cb030cb4
......@@ -97,6 +97,7 @@ PATENT RIGHTS GRANT:
#include "cachetable.h"
#include "checkpoint.h"
#include "txn_manager.h"
#include "omt.h"
int tokudb_recovery_trace = 0; // turn on recovery tracing, default off.
......
ft/recover.h
View file @ cb030cb4
......@@ -96,10 +96,10 @@ PATENT RIGHTS GRANT:
#include <errno.h>
#include <db.h>
#include <util/x1764.h>
#include "fttypes.h"
#include "memory.h"
#include "x1764.h"
typedef void (*prepared_txn_callback_t)(DB_ENV*, TOKUTXN);
typedef void (*keep_cachetable_callback_t)(DB_ENV*, CACHETABLE);
......
ft/sub_block.cc
View file @ cb030cb4
......@@ -89,16 +89,15 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "sub_block.h"
#include "compress.h"
#include "sub_block.h"
#include "quicklz.h"
#include "x1764.h"
#include <memory.h>
#include <toku_assert.h>
#include <toku_portability.h>
#include <util/threadpool.h>
#include <util/x1764.h>
#include <stdio.h>
#include <string.h>
......@@ -268,7 +267,7 @@ compress_sub_block(struct sub_block *sub_block, enum toku_compression_method met
method
);
// checksum it
sub_block->xsum = x1764_memory(sub_block->compressed_ptr, sub_block->compressed_size);
sub_block->xsum = toku_x1764_memory(sub_block->compressed_ptr, sub_block->compressed_size);
}
void *
......@@ -365,7 +364,7 @@ decompress_sub_block(void *compress_ptr, uint32_t compress_size, void *uncompres
int result = 0;
// verify checksum
uint32_t xsum = x1764_memory(compress_ptr, compress_size);
uint32_t xsum = toku_x1764_memory(compress_ptr, compress_size);
if (xsum != expected_xsum) {
if (verbose_decompress_sub_block) fprintf(stderr, "%s:%d xsum %u expected %u\n", __FUNCTION__, __LINE__, xsum, expected_xsum);
result = EINVAL;
......
ft/tests/ft-serialize-test.cc
View file @ cb030cb4
......@@ -147,13 +147,6 @@ class test_key_le_pair {
}
};
struct check_leafentries_struct {
int nelts;
LEAFENTRY *elts;
int i;
int (*cmp)(OMTVALUE, void *);
};
enum ftnode_verify_type {
read_all=1,
read_compressed,
......
ft/tokuftdump.cc
View file @ cb030cb4
......@@ -432,7 +432,7 @@ verify_block(unsigned char *cp, uint64_t file_offset, uint64_t size) {
printf("header length too big: %u\n", header_length);
return;
}
uint32_t header_xsum = x1764_memory(cp, header_length);
uint32_t header_xsum = toku_x1764_memory(cp, header_length);
uint32_t expected_xsum = toku_dtoh32(get_unaligned_uint32(&cp[header_length]));
if (header_xsum != expected_xsum) {
printf("header checksum failed: %u %u\n", header_xsum, expected_xsum);
......@@ -450,7 +450,7 @@ verify_block(unsigned char *cp, uint64_t file_offset, uint64_t size) {
// verify the sub block header
uint32_t offset = header_length + 4;
for (uint32_t i = 0 ; i < n_sub_blocks; i++) {
uint32_t xsum = x1764_memory(cp + offset, sub_block[i].compressed_size);
uint32_t xsum = toku_x1764_memory(cp + offset, sub_block[i].compressed_size);
printf("%u: %u %u %u", i, sub_block[i].compressed_size, sub_block[i].uncompressed_size, sub_block[i].xsum);
if (xsum != sub_block[i].xsum)
printf(" fail %u offset %" PRIu64, xsum, file_offset + offset);
......
ft/wbuf.h
View file @ cb030cb4
......@@ -95,9 +95,9 @@ PATENT RIGHTS GRANT:
#include <string.h>
#include <portability/toku_htonl.h>
#include <util/x1764.h>
#include "fttypes.h"
#include "x1764.h"
#define CRC_INCR
......@@ -121,7 +121,7 @@ static inline void wbuf_nocrc_init (struct wbuf *w, void *buf, DISKOFF size) {
static inline void wbuf_init (struct wbuf *w, void *buf, DISKOFF size) {
wbuf_nocrc_init(w, buf, size);
x1764_init(&w->checksum);
toku_x1764_init(&w->checksum);
}
static inline size_t wbuf_get_woffset(struct wbuf *w) {
......@@ -142,7 +142,7 @@ static inline void wbuf_nocrc_uint8_t (struct wbuf *w, uint8_t ch) {
static inline void wbuf_char (struct wbuf *w, unsigned char ch) {
wbuf_nocrc_char (w, ch);
x1764_add(&w->checksum, &w->buf[w->ndone-1], 1);
toku_x1764_add(&w->checksum, &w->buf[w->ndone-1], 1);
}
//Write an int that MUST be in network order regardless of disk order
......@@ -150,7 +150,7 @@ static void wbuf_network_int (struct wbuf *w, int32_t i) __attribute__((__unused
static void wbuf_network_int (struct wbuf *w, int32_t i) {
assert(w->ndone + 4 <= w->size);
*(uint32_t*)(&w->buf[w->ndone]) = toku_htonl(i);
x1764_add(&w->checksum, &w->buf[w->ndone], 4);
toku_x1764_add(&w->checksum, &w->buf[w->ndone], 4);
w->ndone += 4;
}
......@@ -176,7 +176,7 @@ static inline void wbuf_nocrc_int (struct wbuf *w, int32_t i) {
static inline void wbuf_int (struct wbuf *w, int32_t i) {
wbuf_nocrc_int(w, i);
x1764_add(&w->checksum, &w->buf[w->ndone-4], 4);
toku_x1764_add(&w->checksum, &w->buf[w->ndone-4], 4);
}
static inline void wbuf_nocrc_uint (struct wbuf *w, uint32_t i) {
......@@ -207,7 +207,7 @@ static inline void wbuf_nocrc_literal_bytes(struct wbuf *w, bytevec bytes_bv, ui
static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, uint32_t nbytes) {
wbuf_nocrc_literal_bytes(w, bytes_bv, nbytes);
x1764_add(&w->checksum, &w->buf[w->ndone-nbytes], nbytes);
toku_x1764_add(&w->checksum, &w->buf[w->ndone-nbytes], nbytes);
}
static void wbuf_nocrc_bytes (struct wbuf *w, bytevec bytes_bv, uint32_t nbytes) {
......
ft/xids.h
View file @ cb030cb4
......@@ -104,12 +104,10 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "x1764.h"
#include "rbuf.h"
#include "wbuf.h"
#include "tokuconst.h"
//Retrieve an XIDS representing the root transaction.
XIDS xids_get_root_xids(void);
......
util/CMakeLists.txt
View file @ cb030cb4
......@@ -6,6 +6,7 @@ set(util_srcs
partitioned_counter
threadpool
scoped_malloc
x1764
)
add_library(util SHARED ${util_srcs})
......
ft/tests/x1764-test.cc → util/tests/x1764-test.cc
View file @ cb030cb4
......@@ -88,17 +88,17 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#include "test.h"
#include <util/x1764.h>
static void
test0 (void) {
uint32_t c = x1764_memory("", 0);
uint32_t c = toku_x1764_memory("", 0);
assert(c==~(0U));
struct x1764 cs;
x1764_init(&cs);
x1764_add(&cs, "", 0);
c = x1764_finish(&cs);
toku_x1764_init(&cs);
toku_x1764_add(&cs, "", 0);
c = toku_x1764_finish(&cs);
assert(c==~(0U));
}
......@@ -110,7 +110,7 @@ test1 (void) {
for (i=0; i<=8; i++) {
uint64_t expect64 = (i==8) ? v : v&((1LL<<(8*i))-1);
uint32_t expect = expect64 ^ (expect64>>32);
c = x1764_memory(&v, i);
c = toku_x1764_memory(&v, i);
//printf("i=%d c=%08x expect=%08x\n", i, c, expect);
assert(c==~expect);
}
......@@ -127,33 +127,33 @@ test2 (void) {
int j;
for (j=i; j<=N; j++) {
// checksum from i (inclusive to j (exclusive)
uint32_t c = x1764_memory(&v[i], j-i);
uint32_t c = toku_x1764_memory(&v[i], j-i);
// Now compute the checksum incrementally with various strides.
int stride;
for (stride=1; stride<=j-i; stride++) {
int k;
struct x1764 s;
x1764_init(&s);
toku_x1764_init(&s);
for (k=i; k+stride<=j; k+=stride) {
x1764_add(&s, &v[k], stride);
toku_x1764_add(&s, &v[k], stride);
}
x1764_add(&s, &v[k], j-k);
uint32_t c2 = x1764_finish(&s);
toku_x1764_add(&s, &v[k], j-k);
uint32_t c2 = toku_x1764_finish(&s);
assert(c2==c);
}
// Now use some random strides.
{
int k=i;
struct x1764 s;
x1764_init(&s);
toku_x1764_init(&s);
while (1) {
stride=random()%16;
if (k+stride>j) break;
x1764_add(&s, &v[k], stride);
toku_x1764_add(&s, &v[k], stride);
k+=stride;
}
x1764_add(&s, &v[k], j-k);
uint32_t c2 = x1764_finish(&s);
toku_x1764_add(&s, &v[k], j-k);
uint32_t c2 = toku_x1764_finish(&s);
assert(c2==c);
}
}
......@@ -170,8 +170,8 @@ test3 (void)
for (int off=0; off<32; off++) {
if (verbose) {printf("."); fflush(stdout);}
for (int len=0; len+off<datalen; len++) {
uint32_t reference_sum = x1764_memory_simple(data+off, len);
uint32_t fast_sum = x1764_memory (data+off, len);
uint32_t reference_sum = toku_x1764_memory_simple(data+off, len);
uint32_t fast_sum = toku_x1764_memory (data+off, len);
assert(reference_sum==fast_sum);
}
}
......
ft/x1764.cc → util/x1764.cc
View file @ cb030cb4
......@@ -96,7 +96,7 @@ PATENT RIGHTS GRANT:
#define PRINT 0
uint32_t x1764_memory_simple (const void *buf, int len)
uint32_t toku_x1764_memory_simple (const void *buf, int len)
{
const uint64_t *CAST_FROM_VOIDP(lbuf, buf);
uint64_t c=0;
......@@ -118,7 +118,7 @@ uint32_t x1764_memory_simple (const void *buf, int len)
return ~((c&0xFFFFFFFF) ^ (c>>32));
}
uint32_t x1764_memory (const void *vbuf, int len)
uint32_t toku_x1764_memory (const void *vbuf, int len)
{
const uint8_t *CAST_FROM_VOIDP(buf, vbuf);
int len_4_words = 4*sizeof(uint64_t);
......@@ -149,13 +149,13 @@ uint32_t x1764_memory (const void *vbuf, int len)
}
void x1764_init(struct x1764 *l) {
void toku_x1764_init(struct x1764 *l) {
l->sum=0;
l->input=0;
l->n_input_bytes=0;
}
void x1764_add (struct x1764 *l, const void *vbuf, int len) {
void toku_x1764_add (struct x1764 *l, const void *vbuf, int len) {
if (PRINT) printf("%d: n_input_bytes=%d len=%d\n", __LINE__, l->n_input_bytes, len);
int n_input_bytes = l->n_input_bytes;
const unsigned char *CAST_FROM_VOIDP(cbuf, vbuf);
......@@ -287,7 +287,7 @@ void x1764_add (struct x1764 *l, const void *vbuf, int len) {
l->input = input;
if (PRINT) printf("%d: n_input_bytes=%d\n", __LINE__, l->n_input_bytes);
}
uint32_t x1764_finish (struct x1764 *l) {
uint32_t toku_x1764_finish (struct x1764 *l) {
if (PRINT) printf("%d: n_input_bytes=%d\n", __LINE__, l->n_input_bytes);
int len = l->n_input_bytes;
if (len>0) {
......
ft/x1764.h → util/x1764.h
View file @ cb030cb4
......@@ -99,11 +99,11 @@ PATENT RIGHTS GRANT:
// If any odd bytes numbers are left at the end, they are filled in at the low end.
uint32_t x1764_memory (const void *buf, int len);
uint32_t toku_x1764_memory (const void *buf, int len);
// Effect: Compute x1764 on the bytes of buf. Return the 32 bit answer.
uint32_t x1764_memory_simple (const void *buf, int len);
// Effect: Same as x1764_memory, but not highly optimized (more likely to be correct). Useful for testing the optimized version.
uint32_t toku_x1764_memory_simple (const void *buf, int len);
// Effect: Same as toku_x1764_memory, but not highly optimized (more likely to be correct). Useful for testing the optimized version.
// For incrementally computing an x1764, use the following interfaces.
......@@ -113,13 +113,13 @@ struct x1764 {
int n_input_bytes;
};
void x1764_init(struct x1764 *l);
void toku_x1764_init(struct x1764 *l);
// Effect: Initialize *l.
void x1764_add (struct x1764 *l, const void *vbuf, int len);
void toku_x1764_add (struct x1764 *l, const void *vbuf, int len);
// Effect: Add more bytes to *l.
uint32_t x1764_finish (struct x1764 *l);
uint32_t toku_x1764_finish (struct x1764 *l);
// Effect: Return the final 32-bit result.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7