Skip to content

M
mariadb
Commit aef38189 authored by Sergei Golubchik's avatar Sergei Golubchik
Browse files
Options

tokudb 7.1.8

parents c799d65b b585b4f7 b5dd1e7f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 18898 additions and 11462 deletions
.bzrignore
View file @ aef38189
......@@ -1415,7 +1415,7 @@ storage/tokudb/ft-index/portability/tests/try-uninit
storage/tokudb/ft-index/src/merge_archives_tokufractaltree_static.cmake
storage/tokudb/ft-index/src/tokufractaltree_static_depends.cc
storage/tokudb/ft-index/src/tests/recovery_fileops_unit_dir
storage/tokudb/ft-index/portability/toku_config.h
storage/tokudb/ft-index/toku_include/toku_config.h
storage/tokudb/ft-index/util/tests/marked-omt-test
storage/tokudb/ft-index/util/tests/omt-tmpl-test
storage/tokudb/ft-index/util/tests/sort-tmpl-test
......@@ -1429,8 +1429,8 @@ storage/tokudb/ft-index/util/tests/test_partitioned_counter
storage/tokudb/ft-index/util/tests/test_partitioned_counter_5833
storage/tokudb/ft-index/util/tests/threadpool-test
storage/tokudb/ft-index/util/tests/threadpool-testrunf
storage/tokudb/ft-index/tools/tokudb_dump
storage/tokudb/ft-index/tools/tokudb_gen
storage/tokudb/ft-index/tools/tokudb_load
storage/tokudb/ft-index/utils/tokudb_dump
storage/tokudb/ft-index/utils/tokudb_gen
storage/tokudb/ft-index/utils/tokudb_load
libmysql/libmysql_versions.ld
scripts/mysql_config.pl
storage/tokudb/CMakeLists.txt
View file @ aef38189
......@@ -18,12 +18,9 @@ IF(NOT LIBJEMALLOC)
ENDIF()
############################################
SET(TOKUDB_VERSION "7.1.7")
SET(TOKUDB_VERSION "7.1.8")
SET(TOKUDB_DEB_FILES "usr/lib/mysql/plugin/ha_tokudb.so\netc/mysql/conf.d/tokudb.cnf\nusr/bin/tokuftdump\nusr/share/doc/mariadb-server-5.5/README-TOKUDB\nusr/share/doc/mariadb-server-5.5/README.md" PARENT_SCOPE)
SET(USE_BDB OFF CACHE BOOL "")
SET(USE_VALGRIND OFF CACHE BOOL "")
SET(BUILD_TESTING OFF CACHE BOOL "")
SET(TOKU_DEBUG_PARANOID OFF CACHE BOOL "")
MARK_AS_ADVANCED(BUILDNAME)
MARK_AS_ADVANCED(BUILD_TESTING)
MARK_AS_ADVANCED(CMAKE_TOKUDB_REVISION)
......@@ -36,6 +33,10 @@ MARK_AS_ADVANCED(USE_VALGRIND)
MARK_AS_ADVANCED(XZ_SOURCE_DIR)
############################################
SET(BUILD_TESTING OFF CACHE BOOL "")
SET(USE_VALGRIND OFF CACHE BOOL "")
SET(TOKU_DEBUG_PARANOID OFF CACHE BOOL "")
IF(NOT DEFINED TOKUDB_VERSION)
IF(DEFINED ENV{TOKUDB_VERSION})
SET(TOKUDB_VERSION $ENV{TOKUDB_VERSION})
......@@ -53,14 +54,36 @@ IF(DEFINED TOKUDB_CHECK_JEMALLOC)
ADD_DEFINITIONS("-DTOKUDB_CHECK_JEMALLOC=${TOKUDB_CHECK_JEMALLOC}")
ENDIF()
## adds a compiler flag if the compiler supports it
include(CheckCCompilerFlag)
include(CheckCXXCompilerFlag)
macro(set_cflags_if_supported)
foreach(flag ${ARGN})
check_c_compiler_flag(${flag} HAVE_C_${flag})
if (HAVE_C_${flag})
set(CMAKE_C_FLAGS "${flag} ${CMAKE_C_FLAGS}")
endif ()
check_cxx_compiler_flag(${flag} HAVE_CXX_${flag})
if (HAVE_CXX_${flag})
set(CMAKE_CXX_FLAGS "${flag} ${CMAKE_CXX_FLAGS}")
endif ()
endforeach(flag)
endmacro(set_cflags_if_supported)
set_cflags_if_supported(-Wno-missing-field-initializers)
ADD_SUBDIRECTORY(ft-index)
# TODO: clean up includes in ft-index
INCLUDE_DIRECTORIES(ft-index)
INCLUDE_DIRECTORIES(ft-index/include)
INCLUDE_DIRECTORIES(ft-index/portability)
INCLUDE_DIRECTORIES(ft-index/toku_include)
INCLUDE_DIRECTORIES(ft-index/util)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/buildheader)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/portability)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/toku_include)
SET(TOKUDB_PLUGIN_DYNAMIC "ha_tokudb")
SET(TOKUDB_SOURCES ha_tokudb.cc)
......
storage/tokudb/README.md
View file @ aef38189
......@@ -24,14 +24,14 @@ working MySQL or MariaDB with Tokutek patches, and with the TokuDB storage
engine, called `make.mysql.bash`. This script will download copies of the
needed source code from github and build everything.
To build MySQL 5.5.37 with TokuDB 7.1.6:
To build MySQL 5.5.38 with TokuDB 7.1.7:
```sh
scripts/make.mysql.bash --mysqlbuild=mysql-5.5.37-tokudb-7.1.6-linux-x86_64
scripts/make.mysql.bash --mysqlbuild=mysql-5.5.38-tokudb-7.1.7-linux-x86_64
```
To build MariaDB 5.5.37 with TokuDB 7.1.6:
To build MariaDB 5.5.38 with TokuDB 7.1.7:
```sh
scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.37-tokudb-7.1.6-linux-x86_64
scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.38-tokudb-7.1.7-linux-x86_64
```
Before you start, make sure you have a C++11-compatible compiler (GCC >=
......
storage/tokudb/ft-index/CMakeLists.txt
View file @ aef38189
......@@ -6,31 +6,6 @@ project(TokuDB)
set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
## Versions of gcc >= 4.9.0 require special version of 'ar' and 'ranlib' for
## link-time optimizations to work properly.
##
## From https://gcc.gnu.org/gcc-4.9/changes.html:
##
## When using a linker plugin, compiling with the -flto option now
## generates slim objects files (.o) which only contain intermediate
## language representation for LTO. Use -ffat-lto-objects to create
## files which contain additionally the object code. To generate
## static libraries suitable for LTO processing, use gcc-ar and
## gcc-ranlib; to list symbols from a slim object file use
## gcc-nm. (Requires that ar, ranlib and nm have been compiled with
## plugin support.)
if ((CMAKE_CXX_COMPILER_ID STREQUAL GNU) AND
NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9.0"))
find_program(gcc_ar "gcc-ar")
if (gcc_ar)
set(CMAKE_AR "${gcc_ar}")
endif ()
find_program(gcc_ranlib "gcc-ranlib")
if (gcc_ranlib)
set(CMAKE_RANLIB "${gcc_ranlib}")
endif ()
endif()
include(TokuFeatureDetection)
include(TokuSetupCompiler)
include(TokuSetupCTest)
......@@ -51,12 +26,14 @@ if (USE_VALGRIND AND NOT VALGRIND_INCLUDE_DIR MATCHES NOTFOUND)
)
endif()
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/toku_include
${CMAKE_CURRENT_SOURCE_DIR}/portability
${CMAKE_CURRENT_SOURCE_DIR} ## so you can include <ft/ft-ops.h> from inside src/
${CMAKE_CURRENT_BINARY_DIR} ## for logging code
)
## include where config.h will be generated
include_directories(${CMAKE_CURRENT_BINARY_DIR}/portability)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/toku_include)
## build db.h and include where it will be generated
add_subdirectory(buildheader)
......@@ -74,9 +51,11 @@ add_subdirectory(portability)
add_subdirectory(ft)
add_subdirectory(locktree)
add_subdirectory(src)
add_subdirectory(tools)
add_subdirectory(utils)
## subdirectories that just install things
#add_subdirectory(include)
add_subdirectory(toku_include)
#add_subdirectory(examples)
INSTALL_DOCUMENTATION(README.md README-TOKUDB COMPONENT Server)
......
storage/tokudb/ft-index/CTestCustom.cmake.in deleted 100644 → 0
View file @ c799d65b
cmake_policy(SET CMP0012 NEW)
## these tests shouldn't run with valgrind
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE
ft/bnc-insert-benchmark
ft/brt-serialize-benchmark
ft/ft_loader-test-extractor-1
ft/ft_loader-test-extractor-2
ft/ft_loader-test-extractor-3
ft/upgrade_test_simple
portability/test-cache-line-boundary-fails
portability/try-leak-lost
portability/try-leak-reachable
portability/try-leak-uninit
util/helgrind_test_circular_buffer
util/helgrind_test_partitioned_counter
util/helgrind_test_partitioned_counter_5833
ydb/diskfull.tdb
ydb/drd_test_4015.tdb
ydb/drd_test_groupcommit_count.tdb
ydb/filesize.tdb
ydb/helgrind_helgrind1.tdb
ydb/helgrind_helgrind2.tdb
ydb/helgrind_helgrind3.tdb
ydb/helgrind_test_groupcommit_count.tdb
ydb/hot-optimize-table-tests.tdb
ydb/insert-dup-prelock.tdb
ydb/loader-cleanup-test2.tdb
ydb/loader-cleanup-test3.tdb
ydb/loader-stress-test4.tdb
ydb/maxsize-for-loader-B.tdb
ydb/openlimit17.tdb
ydb/openlimit17-locktree.tdb
ydb/preload-db-nested.tdb
ydb/stress-gc.tdb
ydb/stress-gc2.tdb
ydb/stress-test.bdb
ydb/stress-test.tdb
ydb/test-5138.tdb
ydb/test-prepare.tdb
ydb/test-prepare2.tdb
ydb/test-prepare3.tdb
ydb/test-recover1.tdb
ydb/test-recover2.tdb
ydb/test-recover3.tdb
ydb/test-xa-prepare.tdb
ydb/test4573-logtrim.tdb
ydb/test_3645.tdb
ydb/test_groupcommit_perf.bdb
ydb/test_groupcommit_perf.tdb
ydb/test_large_update_broadcast_small_cachetable.tdb
ydb/test_update_broadcast_stress.tdb
ydb/test_update_stress.tdb
ydb/upgrade-test-4.tdb
)
if (NOT @RUN_HELGRIND_TESTS@)
list(APPEND CTEST_CUSTOM_TESTS_IGNORE
util/helgrind_test_circular_buffer
util/helgrind_test_partitioned_counter
util/helgrind_test_partitioned_counter_5833
ydb/helgrind_helgrind1.tdb
ydb/helgrind_helgrind2.tdb
ydb/helgrind_helgrind3.tdb
ydb/helgrind_test_groupcommit_count.tdb
)
endif ()
if (NOT @RUN_DRD_TESTS@)
list(APPEND CTEST_CUSTOM_TESTS_IGNORE
ydb/drd_test_groupcommit_count.tdb
ydb/drd_test_4015.tdb
)
endif ()
## osx's pthreads prefer writers, so this test will deadlock
if (@CMAKE_SYSTEM_NAME@ STREQUAL Darwin)
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE portability/test-pthread-rwlock-rwr)
list(APPEND CTEST_CUSTOM_TESTS_IGNORE portability/test-pthread-rwlock-rwr)
endif ()
## tests that are supposed to crash will generate memcheck failures
set(tests_that_should_fail
ft/test-assertA
ft/test-assertB
portability/try-assert-zero
portability/try-assert0
ydb/recover-missing-dbfile-2.abortrecover
ydb/recover-missing-dbfile.abortrecover
ydb/test_db_no_env.tdb
ydb/test_truncate_txn_abort.tdb
)
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE ${tests_that_should_fail})
## don't run drd stress tests with valgrind either (because that would do valgrind twice)
set(stress_tests
test_stress0.tdb
test_stress1.tdb
test_stress2.tdb
test_stress3.tdb
test_stress4.tdb
test_stress5.tdb
test_stress6.tdb
test_stress7.tdb
test_stress_hot_indexing.tdb
test_stress_openclose.tdb
test_stress_with_verify.tdb
)
foreach(test ${stress_tests})
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE
ydb/drd_tiny_${test}
ydb/drd_mid_${test}
ydb/drd_large_${test}
)
if(NOT @RUN_LONG_TESTS@)
list(APPEND CTEST_CUSTOM_TESTS_IGNORE
ydb/drd_large_${test}
)
endif()
if (NOT @RUN_DRD_TESTS@)
list(APPEND CTEST_CUSTOM_TESTS_IGNORE
ydb/drd_tiny_${test}
ydb/drd_mid_${test}
ydb/drd_large_${test}
)
endif ()
endforeach(test)
## upgrade stress tests are 5 minutes long, don't need to run them always
if(NOT @RUN_LONG_TESTS@)
foreach(test ${stress_tests})
if (NOT ${test} MATCHES test_stress_openclose)
foreach(oldver 4.2.0 5.0.8 5.2.7 6.0.0 6.1.0 6.5.1 6.6.3)
foreach(p_or_s pristine stressed)
if (NOT (${test} MATCHES test_stress4 AND ${p_or_s} MATCHES stressed))
foreach(size 2000)
list(APPEND CTEST_CUSTOM_TESTS_IGNORE ydb/${test}/upgrade/${oldver}/${p_or_s}/${size})
endforeach(size)
endif ()
endforeach(p_or_s)
endforeach(oldver)
endif ()
endforeach(test)
endif()
set(tdb_tests_that_should_fail "ydb/${stress_tests}")
string(REGEX REPLACE ";" ";ydb/" stress_tests "${stress_tests}")
set(recover_stress_tests
ydb/recover-test_stress1.abortrecover
ydb/recover-test_stress2.abortrecover
ydb/recover-test_stress3.abortrecover
ydb/recover-test_stress_openclose.abortrecover
)
## we run stress tests separately, only run them if asked to
if(NOT @RUN_STRESS_TESTS@)
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE ${stress_tests} ${recover_stress_tests})
list(APPEND CTEST_CUSTOM_TESTS_IGNORE ${stress_tests} ${recover_stress_tests})
endif()
set(perf_tests
ydb/perf_checkpoint_var.tdb
ydb/perf_cursor_nop.tdb
ydb/perf_malloc_free.tdb
ydb/perf_nop.tdb
ydb/perf_ptquery.tdb
ydb/perf_ptquery2.tdb
ydb/perf_read_write.tdb
ydb/perf_xmalloc_free.tdb
)
## we also don't need to run perf tests every time
if(NOT @RUN_PERF_TESTS@)
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE ${perf_tests})
list(APPEND CTEST_CUSTOM_TESTS_IGNORE ${perf_tests})
endif()
## don't run perf tests with valgrind (that's slow)
file(GLOB perf_test_srcs RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/src/tests" perf_*.cc)
string(REGEX REPLACE "\\.cc(;|$)" ".tdb\\1" perf_tests "${perf_test_srcs}")
set(tdb_tests_that_should_fail "ydb/${perf_tests}")
string(REGEX REPLACE ";" ";ydb/" perf_tests "${perf_tests}")
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE ${perf_tests})
## these tests fail often and aren't helpful
set(known_failing_tests
ydb/diskfull.tdb
)
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE ${known_failing_tests})
list(APPEND CTEST_CUSTOM_TESTS_IGNORE ${known_failing_tests})
## these tests take a long time, only run them if asked to
set(long_running_tests
ft/is_empty
ft/upgrade_test_simple
ydb/checkpoint_1.tdb
ydb/checkpoint_stress.tdb
ydb/hotindexer-with-queries.tdb
ydb/hot-optimize-table-tests.tdb
ydb/loader-cleanup-test0.tdb
ydb/loader-cleanup-test0z.tdb
ydb/loader-cleanup-test2.tdb
ydb/loader-cleanup-test2z.tdb
ydb/loader-stress-test4.tdb
ydb/loader-stress-test4z.tdb
ydb/manyfiles.tdb
ydb/preload-db-nested.tdb
ydb/recover_stress.tdb
ydb/root_fifo_1.tdb
ydb/root_fifo_2.tdb
ydb/root_fifo_31.tdb
ydb/root_fifo_32.tdb
ydb/stress-gc.tdb
ydb/stress-test.tdb
ydb/test3529.tdb
ydb/test_logmax.tdb
ydb/test_txn_nested2.tdb
ydb/test_update_broadcast_stress.tdb
ydb/test_update_stress.tdb
)
if(NOT @RUN_LONG_TESTS@)
list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE ${long_running_tests})
list(APPEND CTEST_CUSTOM_TESTS_IGNORE ${long_running_tests})
endif()
## ignore log_print.cc in coverage report
list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE "log_print.cc")
list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION
# don't complain about warnings in xz source
"xz-4.999.9beta/src/liblzma"
# don't complain about clang missing warnings from xz code
"clang: warning: unknown warning option"
# don't complain about warnings in jemalloc source
"jemalloc/src"
"jemalloc/internal"
# don't complain about valgrind headers leaving things unused
"valgrind/valgrind.h"
"valgrind/memcheck.h"
# don't complain about ranlib or libtool on empty archive
"has no symbols"
"the table of contents is empty"
)
storage/tokudb/ft-index/buildheader/CMakeLists.txt
View file @ aef38189
......@@ -26,4 +26,4 @@ if (NOT DEFINED MYSQL_PROJECT_NAME_DOCSTRING)
DESTINATION include
COMPONENT tokukv_headers
)
endif ()
endif ()
\ No newline at end of file
storage/tokudb/ft-index/buildheader/make_tdb.cc
View file @ aef38189
......@@ -291,6 +291,7 @@ static void print_defines (void) {
printf("#define DB_IS_HOT_INDEX 0x00100000\n"); // private tokudb
printf("#define DBC_DISABLE_PREFETCHING 0x20000000\n"); // private tokudb
printf("#define DB_UPDATE_CMP_DESCRIPTOR 0x40000000\n"); // private tokudb
printf("#define TOKUFT_DIRTY_SHUTDOWN %x\n", 1<<31);
{
//dbt flags
......@@ -464,7 +465,6 @@ static void print_db_env_struct (void) {
"void (*set_loader_memory_size)(DB_ENV *env, uint64_t (*get_loader_memory_size_callback)(void))",
"uint64_t (*get_loader_memory_size)(DB_ENV *env)",
"void (*set_killed_callback)(DB_ENV *env, uint64_t default_killed_time_msec, uint64_t (*get_killed_time_callback)(uint64_t default_killed_time_msec), int (*killed_callback)(void))",
"void (*do_backtrace) (DB_ENV *env)",
NULL};
sort_and_dump_fields("db_env", true, extra);
......@@ -571,11 +571,13 @@ static void print_db_txn_struct (void) {
STRUCT_SETUP(DB_TXN, api_internal,"void *%s");
STRUCT_SETUP(DB_TXN, commit, "int (*%s) (DB_TXN*, uint32_t)");
STRUCT_SETUP(DB_TXN, prepare, "int (*%s) (DB_TXN*, uint8_t gid[DB_GID_SIZE])");
STRUCT_SETUP(DB_TXN, discard, "int (*%s) (DB_TXN*, uint32_t)");
STRUCT_SETUP(DB_TXN, id, "uint32_t (*%s) (DB_TXN *)");
STRUCT_SETUP(DB_TXN, mgrp, "DB_ENV *%s /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/");
STRUCT_SETUP(DB_TXN, parent, "DB_TXN *%s");
const char *extra[] = {
"int (*txn_stat)(DB_TXN *, struct txn_stat **)",
"struct toku_list open_txns",
"int (*commit_with_progress)(DB_TXN*, uint32_t, TXN_PROGRESS_POLL_FUNCTION, void*)",
"int (*abort_with_progress)(DB_TXN*, TXN_PROGRESS_POLL_FUNCTION, void*)",
"int (*xa_prepare) (DB_TXN*, TOKU_XA_XID *)",
......@@ -633,6 +635,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
//printf("#include <inttypes.h>\n");
printf("#if defined(__cplusplus) || defined(__cilkplusplus)\nextern \"C\" {\n#endif\n");
printf("#define TOKUDB 1\n");
printf("#define DB_VERSION_MAJOR %d\n", DB_VERSION_MAJOR);
printf("#define DB_VERSION_MINOR %d\n", DB_VERSION_MINOR);
printf("/* As of r40364 (post TokuDB 5.2.7), the patch version number is 100+ the BDB header patch version number.*/\n");
......@@ -651,6 +654,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
" char data[DB_GID_SIZE];\n"
"} TOKU_XA_XID;\n");
//Typedef toku_off_t
printf("#ifndef TOKU_OFF_T_DEFINED\n"
"#define TOKU_OFF_T_DEFINED\n"
"typedef int64_t toku_off_t;\n"
......@@ -669,10 +673,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
printf("typedef uint32_t db_recno_t;\n");
printf("typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);\n");
printf("struct simple_dbt {\n");
printf(" uint32_t len;\n");
printf(" void *data;\n");
printf("};\n");
printf("#include <tdb-internal.h>\n");
//stat64
printf("typedef struct __toku_db_btree_stat64 {\n");
......
storage/tokudb/ft-index/cmake/merge_archives_unix.cmake.in
View file @ aef38189
......@@ -26,52 +26,15 @@ SET(CMAKE_RANLIB "@CMAKE_RANLIB@")
SET(TEMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/merge_archives_${TARGET})
MAKE_DIRECTORY(${TEMP_DIR})
# Extract each archive to its own subdirectory(avoid object filename
# clashes) Since the lib may contain objects with the same name, we first
# list the archive contents, then uniquify the object names as we extract
# them.
# Extract each archive to its own subdirectory(avoid object filename clashes)
FOREACH(LIB ${STATIC_LIBS})
GET_FILENAME_COMPONENT(NAME_NO_EXT ${LIB} NAME_WE)
SET(TEMP_SUBDIR ${TEMP_DIR}/${NAME_NO_EXT})
MAKE_DIRECTORY(${TEMP_SUBDIR})
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} -t ${LIB}
OUTPUT_VARIABLE LIB_OBJS
COMMAND ${CMAKE_AR} -x ${LIB}
WORKING_DIRECTORY ${TEMP_SUBDIR}
)
STRING(REGEX REPLACE "\n" ";" LIB_OBJ_LIST "${LIB_OBJS}")
STRING(REGEX REPLACE ";$" "" LIB_OBJ_LIST "${LIB_OBJ_LIST}")
LIST(LENGTH LIB_OBJ_LIST LENGTH_WITH_DUPS)
SET(LIB_OBJ_LIST_NO_DUPS ${LIB_OBJ_LIST})
LIST(REMOVE_DUPLICATES LIB_OBJ_LIST_NO_DUPS)
LIST(LENGTH LIB_OBJ_LIST_NO_DUPS LENGTH_WITHOUT_DUPS)
IF(LENGTH_WITH_DUPS EQUAL LENGTH_WITHOUT_DUPS)
# Optimization for when lib doesn't actually have duplicate object
# names, we can just extract everything.
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} -x ${LIB}
WORKING_DIRECTORY ${TEMP_SUBDIR}
)
ELSE()
LIST(SORT LIB_OBJ_LIST)
SET(SAME_OBJ_COUNT 1)
SET(LAST_OBJ_NAME)
FOREACH(OBJ ${LIB_OBJ_LIST})
IF(OBJ STREQUAL LAST_OBJ_NAME)
GET_FILENAME_COMPONENT(OBJ_NO_EXT ${OBJ} NAME_WE)
FILE(RENAME "${TEMP_SUBDIR}/${OBJ}" "${TEMP_SUBDIR}/${OBJ_NO_EXT}.${SAME_OBJ_COUNT}.o")
MATH(EXPR SAME_OBJ_COUNT "${SAME_OBJ_COUNT}+1")
ELSE()
SET(SAME_OBJ_COUNT 1)
ENDIF()
SET(LAST_OBJ_NAME "${OBJ}")
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} -xN ${SAME_OBJ_COUNT} ${LIB} ${OBJ}
WORKING_DIRECTORY ${TEMP_SUBDIR}
)
ENDFOREACH()
ENDIF()
FILE(GLOB_RECURSE LIB_OBJECTS "${TEMP_SUBDIR}/*.o")
SET(OBJECTS ${OBJECTS} ${LIB_OBJECTS})
......@@ -88,7 +51,11 @@ ENDFOREACH()
FILE(TO_NATIVE_PATH ${TARGET_LOCATION} ${TARGET_LOCATION})
# Now pack the objects into library with ar.
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} rcs ${TARGET_LOCATION} ${ALL_OBJECTS}
COMMAND ${CMAKE_AR} -r ${TARGET_LOCATION} ${ALL_OBJECTS}
WORKING_DIRECTORY ${TEMP_DIR}
)
EXECUTE_PROCESS(
COMMAND ${CMAKE_RANLIB} ${TARGET_LOCATION}
WORKING_DIRECTORY ${TEMP_DIR}
)
......
storage/tokudb/ft-index/cmake_modules/TokuBuildTagDatabases.cmake
View file @ aef38189
......@@ -7,6 +7,7 @@ file(GLOB_RECURSE all_srcs
locktree/*.cc
portability/*.cc
src/*.cc
toku_include/*.cc
utils/*.cc
util/*.cc
db-benchmark-test/*.cc
......@@ -23,12 +24,13 @@ file(GLOB_RECURSE all_hdrs
locktree/*.h
portability/*.h
src/*.h
toku_include/*.h
utils/*.h
util/*.h
db-benchmark-test/*.h
)
list(APPEND all_hdrs
${CMAKE_CURRENT_BINARY_DIR}/portability/toku_config.h
${CMAKE_CURRENT_BINARY_DIR}/toku_include/toku_config.h
${CMAKE_CURRENT_BINARY_DIR}/buildheader/db.h
${CMAKE_CURRENT_BINARY_DIR}/ft/log_header.h
)
......@@ -77,7 +79,7 @@ if (USE_CSCOPE)
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.out"
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.in.out"
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.po.out"
COMMAND ${CSCOPE} -b -q -R -i"${CMAKE_CURRENT_BINARY_DIR}/cscope.files" -I"${CMAKE_CURRENT_SOURCE_DIR}" -I"${CMAKE_CURRENT_SOURCE_DIR}/include" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/ft" -I"${CMAKE_CURRENT_SOURCE_DIR}/src" -I"${CMAKE_CURRENT_SOURCE_DIR}/locktree" -I"${CMAKE_CURRENT_SOURCE_DIR}/utils" -I"${CMAKE_CURRENT_SOURCE_DIR}/db-benchmark-test" -I"${CMAKE_CURRENT_BINARY_DIR}" -I"${CMAKE_CURRENT_BINARY_DIR}/portability" -I"${CMAKE_CURRENT_BINARY_DIR}/buildheader"
COMMAND ${CSCOPE} -b -q -R -i"${CMAKE_CURRENT_BINARY_DIR}/cscope.files" -I"${CMAKE_CURRENT_SOURCE_DIR}" -I"${CMAKE_CURRENT_SOURCE_DIR}/include" -I"${CMAKE_CURRENT_SOURCE_DIR}/toku_include" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/ft" -I"${CMAKE_CURRENT_SOURCE_DIR}/src" -I"${CMAKE_CURRENT_SOURCE_DIR}/locktree" -I"${CMAKE_CURRENT_SOURCE_DIR}/utils" -I"${CMAKE_CURRENT_SOURCE_DIR}/db-benchmark-test" -I"${CMAKE_CURRENT_BINARY_DIR}" -I"${CMAKE_CURRENT_BINARY_DIR}/toku_include" -I"${CMAKE_CURRENT_BINARY_DIR}/buildheader"
DEPENDS ${all_srcs} ${all_hdrs} install_tdb_h generate_config_h generate_log_code
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
add_custom_target(build_cscope.out ALL DEPENDS
......
storage/tokudb/ft-index/cmake_modules/TokuSetupCTest.cmake
View file @ aef38189
......@@ -94,6 +94,8 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
## set up full valgrind suppressions file (concatenate the suppressions files)
file(READ ft/valgrind.suppressions valgrind_suppressions)
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${valgrind_suppressions}")
file(READ src/tests/bdb.suppressions bdb_suppressions)
file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bdb_suppressions}")
file(READ bash.suppressions bash_suppressions)
file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bash_suppressions}")
......
storage/tokudb/ft-index/ft/CMakeLists.txt
View file @ aef38189
......@@ -52,7 +52,9 @@ set(FT_SOURCES
logfilemgr
logger
log_upgrade
memarena
minicron
omt
pqueue
queue
quicklz
......@@ -67,6 +69,7 @@ set(FT_SOURCES
txn_child_manager
txn_manager
ule
x1764
xids
ybt
"${CMAKE_CURRENT_BINARY_DIR}/log_code"
......
storage/tokudb/ft-index/ft/background_job_manager.cc
View file @ aef38189
......@@ -89,12 +89,11 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2011-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <portability/toku_config.h>
#include "background_job_manager.h"
#include "toku_config.h"
#include <memory.h>
#include <toku_pthread.h>
#include "background_job_manager.h"
struct background_job_manager_struct {
bool accepting_jobs;
uint32_t num_jobs;
......
storage/tokudb/ft-index/ft/block_table.cc
View file @ aef38189
......@@ -427,7 +427,7 @@ toku_ft_unlock (FT ft) {
unlock_for_blocktable(bt);
}
// Also used only in ft-serialize-test.
// Also used only in brt-serialize-test.
void
toku_block_free(BLOCK_TABLE bt, uint64_t offset) {
lock_for_blocktable(bt);
......@@ -580,7 +580,7 @@ void toku_serialize_translation_to_wbuf(BLOCK_TABLE bt, int fd, struct wbuf *w,
wbuf_DISKOFF(w, t->block_translation[i].u.diskoff);
wbuf_DISKOFF(w, t->block_translation[i].size);
}
uint32_t checksum = toku_x1764_finish(&w->checksum);
uint32_t checksum = x1764_finish(&w->checksum);
wbuf_int(w, checksum);
*address = t->block_translation[b.b].u.diskoff;
*size = size_translation;
......@@ -915,7 +915,7 @@ translation_deserialize_from_buffer(struct translation *t, // destination int
t->type = TRANSLATION_CHECKPOINTED;
{
// check the checksum
uint32_t x1764 = toku_x1764_memory(translation_buffer, size_on_disk - 4);
uint32_t x1764 = x1764_memory(translation_buffer, size_on_disk - 4);
uint64_t offset = size_on_disk - 4;
//printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
uint32_t stored_x1764 = toku_dtoh32(*(int*)(translation_buffer + offset));
......
storage/tokudb/ft-index/ft/bndata.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/bndata.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/cachetable.cc
View file @ aef38189
......@@ -370,7 +370,7 @@ toku_cachetable_set_env_dir(CACHETABLE ct, const char *env_dir) {
// What cachefile goes with particular iname (iname relative to env)?
// The transaction that is adding the reference might not have a reference
// to the ft, therefore the cachefile might be closing.
// to the brt, therefore the cachefile might be closing.
// If closing, we want to return that it is not there, but must wait till after
// the close has finished.
// Once the close has finished, there must not be a cachefile with that name
......@@ -380,7 +380,7 @@ int toku_cachefile_of_iname_in_env (CACHETABLE ct, const char *iname_in_env, CAC
}
// What cachefile goes with particular fd?
// This function can only be called if the ft is still open, so file must
// This function can only be called if the brt is still open, so file must
// still be open
int toku_cachefile_of_filenum (CACHETABLE ct, FILENUM filenum, CACHEFILE *cf) {
return ct->cf_list.cachefile_of_filenum(filenum, cf);
......@@ -642,7 +642,7 @@ static void cachetable_free_pair(PAIR p) {
cachetable_evictions++;
PAIR_ATTR new_attr = p->attr;
// Note that flush_callback is called with write_me false, so the only purpose of this
// call is to tell the ft layer to evict the node (keep_me is false).
// call is to tell the brt layer to evict the node (keep_me is false).
// Also, because we have already removed the PAIR from the cachetable in
// cachetable_remove_pair, we cannot pass in p->cachefile and p->cachefile->fd
// for the first two parameters, as these may be invalid (#5171), so, we
......@@ -1302,6 +1302,8 @@ void toku_cachetable_pf_pinned_pair(
pair_unlock(p);
}
// NOW A TEST ONLY FUNCTION!!!
int toku_cachetable_get_and_pin (
CACHEFILE cachefile,
CACHEKEY key,
......@@ -1571,7 +1573,7 @@ exit:
return try_again;
}
int toku_cachetable_get_and_pin_with_dep_pairs (
int toku_cachetable_get_and_pin_with_dep_pairs_batched (
CACHEFILE cachefile,
CACHEKEY key,
uint32_t fullhash,
......@@ -1764,6 +1766,43 @@ got_value:
return 0;
}
int toku_cachetable_get_and_pin_with_dep_pairs (
CACHEFILE cachefile,
CACHEKEY key,
uint32_t fullhash,
void**value,
long *sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
pair_lock_type lock_type,
void* read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
uint32_t num_dependent_pairs, // number of dependent pairs that we may need to checkpoint
PAIR* dependent_pairs,
enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
)
// See cachetable.h
{
int r = toku_cachetable_get_and_pin_with_dep_pairs_batched(
cachefile,
key,
fullhash,
value,
sizep,
write_callback,
fetch_callback,
pf_req_callback,
pf_callback,
lock_type,
read_extraargs,
num_dependent_pairs,
dependent_pairs,
dependent_dirty
);
return r;
}
// Lookup a key in the cachetable. If it is found and it is not being written, then
// acquire a read lock on the pair, update the LRU list, and return sucess.
//
......@@ -2009,7 +2048,7 @@ maybe_pin_pair(
return retval;
}
int toku_cachetable_get_and_pin_nonblocking(
int toku_cachetable_get_and_pin_nonblocking_batched(
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
......@@ -2161,6 +2200,40 @@ try_again:
abort();
}
int toku_cachetable_get_and_pin_nonblocking (
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
void**value,
long* sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
pair_lock_type lock_type,
void *read_extraargs,
UNLOCKERS unlockers
)
// See cachetable.h.
{
int r = 0;
r = toku_cachetable_get_and_pin_nonblocking_batched(
cf,
key,
fullhash,
value,
sizep,
write_callback,
fetch_callback,
pf_req_callback,
pf_callback,
lock_type,
read_extraargs,
unlockers
);
return r;
}
struct cachefile_prefetch_args {
PAIR p;
CACHETABLE_FETCH_CALLBACK fetch_callback;
......
storage/tokudb/ft-index/ft/cachetable.h
View file @ aef38189
......@@ -344,6 +344,28 @@ void toku_cachetable_put(CACHEFILE cf, CACHEKEY key, uint32_t fullhash,
// then the required PAIRs are written to disk for checkpoint.
// KEY PROPERTY OF DEPENDENT PAIRS: They are already locked by the client
// Returns: 0 if the memory object is in memory, otherwise an error number.
// Rationale:
// begin_batched_pin and end_batched_pin take and release a read lock on the pair list.
// Normally, that would be done within this get_and_pin, but we want to pin multiple nodes with a single acquisition of the read lock.
int toku_cachetable_get_and_pin_with_dep_pairs_batched (
CACHEFILE cachefile,
CACHEKEY key,
uint32_t fullhash,
void**value,
long *sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
pair_lock_type lock_type,
void* read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
uint32_t num_dependent_pairs, // number of dependent pairs that we may need to checkpoint
PAIR* dependent_pairs,
enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
);
// Effect: call toku_cachetable_get_and_pin_with_dep_pairs_batched once,
// wrapped in begin_batched_pin and end_batched_pin calls.
int toku_cachetable_get_and_pin_with_dep_pairs (
CACHEFILE cachefile,
CACHEKEY key,
......@@ -361,6 +383,7 @@ int toku_cachetable_get_and_pin_with_dep_pairs (
enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
);
// Get and pin a memory object.
// Effects: If the memory object is in the cachetable acquire the PAIR lock on it.
// Otherwise, fetch it from storage by calling the fetch callback. If the fetch
......@@ -399,7 +422,10 @@ struct unlockers {
// Effect: If the block is in the cachetable, then return it.
// Otherwise call the functions in unlockers, fetch the data (but don't pin it, since we'll just end up pinning it again later), and return TOKUDB_TRY_AGAIN.
int toku_cachetable_get_and_pin_nonblocking (
// Rationale:
// begin_batched_pin and end_batched_pin take and release a read lock on the pair list.
// Normally, that would be done within this get_and_pin, but we want to pin multiple nodes with a single acquisition of the read lock.
int toku_cachetable_get_and_pin_nonblocking_batched (
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
......@@ -414,6 +440,23 @@ int toku_cachetable_get_and_pin_nonblocking (
UNLOCKERS unlockers
);
// Effect: call toku_cachetable_get_and_pin_nonblocking_batched once,
// wrapped in begin_batched_pin and end_batched_pin calls.
int toku_cachetable_get_and_pin_nonblocking (
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
void**value,
long *sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback __attribute__((unused)),
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback __attribute__((unused)),
pair_lock_type lock_type,
void *read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
UNLOCKERS unlockers
);
int toku_cachetable_maybe_get_and_pin (CACHEFILE, CACHEKEY, uint32_t /*fullhash*/, pair_lock_type, void**);
// Effect: Maybe get and pin a memory object.
// This function is similar to the get_and_pin function except that it
......
storage/tokudb/ft-index/ft/checksum-benchmarks/adler32.cc 0 → 100644
View file @ aef38189
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <toku_assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <zlib.h>
#include <openssl/md2.h>
#include <openssl/md4.h>
#include <openssl/md5.h>
const unsigned int prime = 2000000011;
unsigned int karprabin (unsigned char *datac, int N) {
assert(N%4==0);
unsigned int *data=(unsigned int*)datac;
N=N/4;
int i;
unsigned int result=0;
for (i=0; i<N; i++) {
result=(result*prime)+data[i];
}
return result;
}
// According to
// P. L'Ecuyer, "Tables of Linear Congruential Generators of
// Different Sizes and Good Lattice Structure", Mathematics of
// Computation 68:225, 249--260 (1999).
// m=2^{32}-5 a=1588635695 is good.
const unsigned int mkr = 4294967291U;
const unsigned int akr = 1588635695U;
// But this is slower
unsigned int karprabinP (unsigned char *datac, int N) {
assert(N%4==0);
unsigned int *data=(unsigned int*)datac;
N=N/4;
int i;
unsigned long long result=0;
for (i=0; i<N; i++) {
result=((result*akr)+data[i])%mkr;
}
return result;
}
float tdiff (struct timeval *start, struct timeval *end) {
return (end->tv_sec-start->tv_sec) +1e-6*(end->tv_usec - start->tv_usec);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
struct timeval start, end;
const int N=2<<20;
unsigned char *data=malloc(N);
int i;
assert(data);
for (i=0; i<N; i++) data[i]=random();
// adler32
{
uLong a32 = adler32(0L, Z_NULL, 0);
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
a32 = adler32(a32, data, N);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("adler32=%lu, time=%9.6fs %9.6fns/b\n", a32, tm, 1e9*tm/N);
}
}
// crc32
{
uLong c32 = crc32(0L, Z_NULL, 0);
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
c32 = crc32(c32, data, N);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("crc32=%lu, time=%9.6fs %9.6fns/b\n", c32, tm, 1e9*tm/N);
}
}
// MD2
{
unsigned char buf[MD2_DIGEST_LENGTH];
int j;
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
MD2(data, N, buf);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("md2=");
for (j=0; j<MD2_DIGEST_LENGTH; j++) {
printf("%02x", buf[j]);
}
printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
}
}
// MD4
{
unsigned char buf[MD4_DIGEST_LENGTH];
int j;
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
MD4(data, N, buf);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("md4=");
for (j=0; j<MD4_DIGEST_LENGTH; j++) {
printf("%02x", buf[j]);
}
printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
}
}
// MD5
{
unsigned char buf[MD5_DIGEST_LENGTH];
int j;
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
MD5(data, N, buf);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("md5=");
for (j=0; j<MD5_DIGEST_LENGTH; j++) {
printf("%02x", buf[j]);
}
printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
}
}
// karp rabin
{
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
unsigned int kr = karprabin(data, N);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("kr=%ud time=%9.6fs %9.6fns/b\n", kr, tm, 1e9*tm/N);
}
}
free(data);
return 0;
}
storage/tokudb/ft-index/ft/compression-ratio/cratio.cc 0 → 100644
View file @ aef38189
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Measure the extent to which we can compress a file.
* Works on version 8. */
#define _XOPEN_SOURCE 500
#include <arpa/inet.h>
#include <toku_assert.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <zlib.h>
#include <toku_portability.h>
toku_off_t fd_size (int fd) {
int64_t file_size;
int r = toku_os_get_file_size(fd, &file_size);
assert(r==0);
return file_size;
}
#define NSIZE (1<<20)
unsigned char fbuf[NSIZE];
unsigned char cbuf[NSIZE+500];
void
measure_header (int fd, toku_off_t off, // read header from this offset
toku_off_t *usize, // size uncompressed (but not including any padding)
toku_off_t *csize) // compressed size
{
int r;
r=pread(fd, fbuf, 12, off);
assert(r==12);
assert(memcmp(fbuf,"tokudata",8)==0);
int bsize = toku_dtoh32(*(uint32_t*)(fbuf+8));
//printf("Bsize=%d\n", bsize);
(*usize)+=bsize;
assert(bsize<=NSIZE);
r=pread(fd, fbuf, bsize, off);
assert(r==bsize);
uLongf destLen=sizeof(cbuf);
r=compress2(cbuf, &destLen,
fbuf+20, bsize-=20, // skip magic nodesize and version
1);
assert(r==Z_OK);
destLen+=16; // account for the size and magic and version
//printf("Csize=%ld\n", destLen);
(*csize)+=destLen;
}
void
measure_node (int fd, toku_off_t off, // read header from this offset
toku_off_t *usize, // size uncompressed (but not including any padding)
toku_off_t *csize) // compressed size
{
int r;
r=pread(fd, fbuf, 24, off);
assert(r==24);
//printf("fbuf[0..7]=%c%c%c%c%c%c%c%c\n", fbuf[0], fbuf[1], fbuf[2], fbuf[3], fbuf[4], fbuf[5], fbuf[6], fbuf[7]);
assert(memcmp(fbuf,"tokuleaf",8)==0 || memcmp(fbuf, "tokunode", 8)==0);
assert(8==toku_dtoh32(*(uint32_t*)(fbuf+8))); // check file version
int bsize = toku_dtoh32(*(uint32_t*)(fbuf+20));
//printf("Bsize=%d\n", bsize);
(*usize)+=bsize;
assert(bsize<=NSIZE);
r=pread(fd, fbuf, bsize, off);
assert(r==bsize);
uLongf destLen=sizeof(cbuf);
r=compress2(cbuf, &destLen,
fbuf+28, bsize-=28, // skip constant header stuff
1);
destLen += 24; // add in magic (8), version(4), lsn (8), and size (4). Actually lsn will be compressed, but ignore that for now.
assert(r==Z_OK);
//printf("Csize=%ld\n", destLen);
(*csize)+=destLen;
}
/* The single argument is the filename to measure. */
int main (int argc, const char *argv[]) {
assert(argc==2);
const char *fname=argv[1];
int fd = open(fname, O_RDONLY);
assert(fd>=0);
toku_off_t fsize = fd_size(fd);
printf("fsize (uncompressed with padding)=%lld\n", (long long)fsize);
toku_off_t usize=0, csize=0;
measure_header(fd, 0, &usize, &csize);
toku_off_t i;
for (i=NSIZE; i+24<fsize; i+=NSIZE) {
measure_node(fd, i, &usize, &csize);
}
printf("usize (uncompressed with no padding)=%10lld (ratio=%5.2f)\n", (long long)usize, (double)fsize/(double)usize);
printf("csize (compressed) =%10lld (ratio=%5.2f)\n", (long long)csize, (double)fsize/(double)csize);
close(fd);
return 0;
}
storage/tokudb/ft-index/ft/fifo.cc
View file @ aef38189
......@@ -102,6 +102,7 @@ struct fifo {
int memory_used; // How many bytes are in use?
};
const int fifo_initial_size = 4096;
static void fifo_init(struct fifo *fifo) {
fifo->n_items_in_fifo = 0;
fifo->memory = 0;
......@@ -117,12 +118,12 @@ static int fifo_entry_size(struct fifo_entry *entry) {
}
__attribute__((const,nonnull))
size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg) {
size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd) {
// This must stay in sync with fifo_entry_size because that's what we
// really trust. But sometimes we only have an in-memory FT_MSG, not
// a serialized fifo_entry so we have to fake it.
return sizeof (struct fifo_entry) + msg->u.id.key->size + msg->u.id.val->size
+ xids_get_size(msg->xids)
return sizeof (struct fifo_entry) + cmd->u.id.key->size + cmd->u.id.val->size
+ xids_get_size(cmd->xids)
- sizeof(XIDS_S);
}
......
storage/tokudb/ft-index/ft/fifo.h
View file @ aef38189
......@@ -97,6 +97,10 @@ PATENT RIGHTS GRANT:
// If the fifo_entry is unpacked, the compiler aligns the xids array and we waste a lot of space
#if TOKU_WINDOWS
#pragma pack(push, 1)
#endif
struct __attribute__((__packed__)) fifo_entry {
unsigned int keylen;
unsigned int vallen;
......@@ -106,7 +110,7 @@ struct __attribute__((__packed__)) fifo_entry {
XIDS_S xids_s;
};
// get and set the ft message type for a fifo entry.
// get and set the brt message type for a fifo entry.
// it is internally stored as a single unsigned char.
static inline enum ft_msg_type
fifo_entry_get_msg_type(const struct fifo_entry * entry)
......@@ -124,6 +128,10 @@ fifo_entry_set_msg_type(struct fifo_entry * entry,
entry->type = type;
}
#if TOKU_WINDOWS
#pragma pack(pop)
#endif
typedef struct fifo *FIFO;
int toku_fifo_create(FIFO *);
......@@ -141,6 +149,9 @@ unsigned long toku_fifo_memory_size_in_use(FIFO fifo); // return how much memor
unsigned long toku_fifo_memory_footprint(FIFO fifo); // return how much memory the fifo occupies
//These two are problematic, since I don't want to malloc() the bytevecs, but dequeueing the fifo frees the memory.
//int toku_fifo_peek_deq (FIFO, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, uint32_t *type, TXNID *xid);
//int toku_fifo_peek_deq_cmdstruct (FIFO, FT_MSG, DBT*, DBT*); // fill in the FT_MSG, using the two DBTs for the DBT part.
void toku_fifo_iterate(FIFO, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, void*), void*);
#define FIFO_ITERATE(fifo,keyvar,keylenvar,datavar,datalenvar,typevar,msnvar,xidsvar,is_freshvar,body) ({ \
......@@ -167,7 +178,7 @@ int toku_fifo_iterate_internal_has_more(FIFO fifo, int off);
int toku_fifo_iterate_internal_next(FIFO fifo, int off);
struct fifo_entry * toku_fifo_iterate_internal_get_entry(FIFO fifo, int off);
size_t toku_fifo_internal_entry_memsize(struct fifo_entry *e) __attribute__((const,nonnull));
size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg) __attribute__((const,nonnull));
size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd) __attribute__((const,nonnull));
DBT *fill_dbt_for_fifo_entry(DBT *dbt, const struct fifo_entry *entry);
struct fifo_entry *toku_fifo_get_entry(FIFO fifo, int off);
......
storage/tokudb/ft-index/ft/ft-cachetable-wrappers.cc
View file @ aef38189
......@@ -201,8 +201,8 @@ toku_create_new_ftnode (
// then a PL_WRITE_CHEAP lock is grabbed
//
int
toku_pin_ftnode_for_query(
FT_HANDLE ft_handle,
toku_pin_ftnode_batched(
FT_HANDLE brt,
BLOCKNUM blocknum,
uint32_t fullhash,
UNLOCKERS unlockers,
......@@ -225,13 +225,13 @@ toku_pin_ftnode_for_query(
paranoid_invariant(bfe->type == ftnode_fetch_subset);
}
int r = toku_cachetable_get_and_pin_nonblocking(
ft_handle->ft->cf,
int r = toku_cachetable_get_and_pin_nonblocking_batched(
brt->ft->cf,
blocknum,
fullhash,
&node_v,
NULL,
get_write_callbacks_for_node(ft_handle->ft),
get_write_callbacks_for_node(brt->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
......@@ -245,7 +245,7 @@ toku_pin_ftnode_for_query(
node = static_cast<FTNODE>(node_v);
if (apply_ancestor_messages && node->height == 0) {
needs_ancestors_messages = toku_ft_leaf_needs_ancestors_messages(
ft_handle->ft,
brt->ft,
node,
ancestors,
bounds,
......@@ -255,20 +255,20 @@ toku_pin_ftnode_for_query(
if (needs_ancestors_messages) {
toku::context apply_messages_ctx(CTX_MESSAGE_APPLICATION);
toku_unpin_ftnode_read_only(ft_handle->ft, node);
int rr = toku_cachetable_get_and_pin_nonblocking(
ft_handle->ft->cf,
blocknum,
fullhash,
&node_v,
NULL,
get_write_callbacks_for_node(ft_handle->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
PL_WRITE_CHEAP,
bfe, //read_extraargs
unlockers);
toku_unpin_ftnode_read_only(brt->ft, node);
int rr = toku_cachetable_get_and_pin_nonblocking_batched(
brt->ft->cf,
blocknum,
fullhash,
&node_v,
NULL,
get_write_callbacks_for_node(brt->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
PL_WRITE_CHEAP,
bfe, //read_extraargs
unlockers);
if (rr != 0) {
assert(rr == TOKUDB_TRY_AGAIN); // Any other error and we should bomb out ASAP.
r = TOKUDB_TRY_AGAIN;
......@@ -276,7 +276,7 @@ toku_pin_ftnode_for_query(
}
node = static_cast<FTNODE>(node_v);
toku_apply_ancestors_messages_to_node(
ft_handle,
brt,
node,
ancestors,
bounds,
......@@ -317,14 +317,54 @@ exit:
}
void
toku_pin_ftnode_with_dep_nodes(
toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE *dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p,
bool move_messages)
{
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
h,
blocknum,
fullhash,
bfe,
lock_type,
num_dependent_nodes,
dependent_nodes,
node_p,
move_messages
);
}
void
toku_pin_ftnode_off_client_thread(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p)
{
toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
h, blocknum, fullhash, bfe, lock_type, num_dependent_nodes, dependent_nodes, node_p, true);
}
void
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p,
bool move_messages)
{
......@@ -336,7 +376,7 @@ toku_pin_ftnode_with_dep_nodes(
dependent_dirty_bits[i] = (enum cachetable_dirty) dependent_nodes[i]->dirty;
}
int r = toku_cachetable_get_and_pin_with_dep_pairs(
int r = toku_cachetable_get_and_pin_with_dep_pairs_batched(
h->cf,
blocknum,
fullhash,
......@@ -352,22 +392,27 @@ toku_pin_ftnode_with_dep_nodes(
dependent_pairs,
dependent_dirty_bits
);
invariant_zero(r);
assert(r==0);
FTNODE node = (FTNODE) node_v;
if (lock_type != PL_READ && node->height > 0 && move_messages) {
if ((lock_type != PL_READ) && node->height > 0 && move_messages) {
toku_move_ftnode_messages_to_stale(h, node);
}
*node_p = node;
}
void toku_pin_ftnode(FT ft,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
FTNODE *node_p,
bool move_messages) {
toku_pin_ftnode_with_dep_nodes(ft, blocknum, fullhash, bfe, lock_type, 0, nullptr, node_p, move_messages);
void
toku_pin_ftnode_off_client_thread_batched(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p)
{
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
h, blocknum, fullhash, bfe, lock_type, num_dependent_nodes, dependent_nodes, node_p, true);
}
int toku_maybe_pin_ftnode_clean(FT ft, BLOCKNUM blocknum, uint32_t fullhash, pair_lock_type lock_type, FTNODE *nodep) {
......@@ -384,12 +429,24 @@ cleanup:
return r;
}
void toku_unpin_ftnode(FT ft, FTNODE node) {
int r = toku_cachetable_unpin(ft->cf,
node->ct_pair,
static_cast<enum cachetable_dirty>(node->dirty),
make_ftnode_pair_attr(node));
invariant_zero(r);
void
toku_unpin_ftnode_off_client_thread(FT ft, FTNODE node)
{
int r = toku_cachetable_unpin(
ft->cf,
node->ct_pair,
(enum cachetable_dirty) node->dirty,
make_ftnode_pair_attr(node)
);
assert(r==0);
}
void
toku_unpin_ftnode(FT ft, FTNODE node)
{
// printf("%*sUnpin %ld\n", 8-node->height, "", node->thisnodename.b);
//VERIFY_NODE(brt,node);
toku_unpin_ftnode_off_client_thread(ft, node);
}
void
......@@ -403,25 +460,3 @@ toku_unpin_ftnode_read_only(FT ft, FTNODE node)
);
assert(r==0);
}
void toku_ftnode_swap_pair_values(FTNODE a, FTNODE b)
// Effect: Swap the blocknum, fullhash, and PAIR for for a and b
// Requires: Both nodes are pinned
{
BLOCKNUM tmp_blocknum = a->thisnodename;
uint32_t tmp_fullhash = a->fullhash;
PAIR tmp_pair = a->ct_pair;
a->thisnodename = b->thisnodename;
a->fullhash = b->fullhash;
a->ct_pair = b->ct_pair;
b->thisnodename = tmp_blocknum;
b->fullhash = tmp_fullhash;
b->ct_pair = tmp_pair;
// A and B swapped pair pointers, but we still have to swap
// the actual pair values (ie: the FTNODEs they represent)
// in the cachetable.
toku_cachetable_swap_pair_values(a->ct_pair, b->ct_pair);
}
storage/tokudb/ft-index/ft/ft-cachetable-wrappers.h
View file @ aef38189
......@@ -138,10 +138,13 @@ toku_create_new_ftnode (
int n_children
);
// This function returns a pinned ftnode to the caller.
/**
* Batched version of toku_pin_ftnode, see cachetable batched API for more
* details.
*/
int
toku_pin_ftnode_for_query(
FT_HANDLE ft_h,
toku_pin_ftnode_batched(
FT_HANDLE brt,
BLOCKNUM blocknum,
uint32_t fullhash,
UNLOCKERS unlockers,
......@@ -153,27 +156,34 @@ toku_pin_ftnode_for_query(
bool* msgs_applied
);
// Pins an ftnode without dependent pairs
void toku_pin_ftnode(
/**
* Unfortunately, this function is poorly named
* as over time, client threads have also started
* calling this function.
* This function returns a pinned ftnode to the caller.
* Unlike toku_pin_ftnode, this function blocks until the node is pinned.
*/
void
toku_pin_ftnode_off_client_thread(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
FTNODE *node_p,
bool move_messages
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p
);
// Pins an ftnode with dependent pairs
// Unlike toku_pin_ftnode_for_query, this function blocks until the node is pinned.
void toku_pin_ftnode_with_dep_nodes(
void
toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE *dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p,
bool move_messages
);
......@@ -185,12 +195,53 @@ void toku_pin_ftnode_with_dep_nodes(
int toku_maybe_pin_ftnode_clean(FT ft, BLOCKNUM blocknum, uint32_t fullhash, pair_lock_type lock_type, FTNODE *nodep);
/**
* Effect: Unpin an ftnode.
* Batched version of toku_pin_ftnode_off_client_thread, see cachetable
* batched API for more details.
*/
void toku_unpin_ftnode(FT h, FTNODE node);
void toku_unpin_ftnode_read_only(FT ft, FTNODE node);
void
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p,
bool move_messages
);
// Effect: Swaps pair values of two pinned nodes
void toku_ftnode_swap_pair_values(FTNODE nodea, FTNODE nodeb);
/**
* Batched version of toku_pin_ftnode_off_client_thread, see cachetable
* batched API for more details.
*/
void
toku_pin_ftnode_off_client_thread_batched(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p
);
/**
* Effect: Unpin a brt node. Used for
* nodes that were pinned off client thread.
*/
void
toku_unpin_ftnode_off_client_thread(FT h, FTNODE node);
/**
* Effect: Unpin a brt node.
* Used for nodes pinned on a client thread
*/
void
toku_unpin_ftnode(FT h, FTNODE node);
void
toku_unpin_ftnode_read_only(FT ft, FTNODE node);
#endif
storage/tokudb/ft-index/ft/ft-flusher.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ft-flusher.h
View file @ aef38189
......@@ -124,7 +124,7 @@ typedef enum {
FT_FLUSHER_SPLIT_NONLEAF, // number of nonleaf nodes split
FT_FLUSHER_MERGE_LEAF, // number of times leaf nodes are merged
FT_FLUSHER_MERGE_NONLEAF, // number of times nonleaf nodes are merged
FT_FLUSHER_BALANCE_LEAF, // number of times a leaf node is balanced
FT_FLUSHER_BALANCE_LEAF, // number of times a leaf node is balanced inside brt
FT_FLUSHER_STATUS_NUM_ROWS
} ft_flusher_status_entry;
......@@ -230,7 +230,7 @@ void toku_ft_hot_get_status(FT_HOT_STATUS);
* we go until the end of the FT.
*/
int
toku_ft_hot_optimize(FT_HANDLE ft_h, DBT* left, DBT* right,
toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
int (*progress_callback)(void *extra, float progress),
void *progress_extra, uint64_t* loops_run);
......
storage/tokudb/ft-index/ft/ft-hot-flusher.cc
View file @ aef38189
......@@ -298,9 +298,9 @@ hot_flusher_destroy(struct hot_flusher_extra *flusher)
// Entry point for Hot Optimize Table (HOT). Note, this function is
// not recursive. It iterates over root-to-leaf paths.
int
toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
int (*progress_callback)(void *extra, float progress),
void *progress_extra, uint64_t* loops_run)
toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
int (*progress_callback)(void *extra, float progress),
void *progress_extra, uint64_t* loops_run)
{
toku::context flush_ctx(CTX_FLUSH);
......@@ -316,7 +316,7 @@ toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
// start of HOT operation
(void) toku_sync_fetch_and_add(&STATUS_VALUE(FT_HOT_NUM_STARTED), 1);
toku_ft_note_hot_begin(ft_handle);
toku_ft_note_hot_begin(brt);
// Higher level logic prevents a dictionary from being deleted or
// truncated during a hot optimize operation. Doing so would violate
......@@ -329,16 +329,17 @@ toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
{
// Get root node (the first parent of each successive HOT
// call.)
toku_calculate_root_offset_pointer(ft_handle->ft, &root_key, &fullhash);
toku_calculate_root_offset_pointer(brt->ft, &root_key, &fullhash);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, ft_handle->ft);
toku_pin_ftnode(ft_handle->ft,
(BLOCKNUM) root_key,
fullhash,
&bfe,
PL_WRITE_EXPENSIVE,
&root,
true);
fill_bfe_for_full_read(&bfe, brt->ft);
toku_pin_ftnode_off_client_thread(brt->ft,
(BLOCKNUM) root_key,
fullhash,
&bfe,
PL_WRITE_EXPENSIVE,
0,
NULL,
&root);
toku_assert_entire_node_in_memory(root);
}
......@@ -364,12 +365,12 @@ toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
// This should recurse to the bottom of the tree and then
// return.
if (root->height > 0) {
toku_ft_flush_some_child(ft_handle->ft, root, &advice);
toku_ft_flush_some_child(brt->ft, root, &advice);
} else {
// Since there are no children to flush, we should abort
// the HOT call.
flusher.rightmost_leaf_seen = 1;
toku_unpin_ftnode(ft_handle->ft, root);
toku_unpin_ftnode_off_client_thread(brt->ft, root);
}
// Set the highest pivot key seen here, since the parent may
......@@ -385,8 +386,8 @@ toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
else if (right) {
// if we have flushed past the bounds set for us,
// set rightmost_leaf_seen so we exit
FAKE_DB(db, &ft_handle->ft->cmp_descriptor);
int cmp = ft_handle->ft->compare_fun(&db, &flusher.max_current_key, right);
FAKE_DB(db, &brt->ft->cmp_descriptor);
int cmp = brt->ft->compare_fun(&db, &flusher.max_current_key, right);
if (cmp > 0) {
flusher.rightmost_leaf_seen = 1;
}
......@@ -416,7 +417,7 @@ toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
if (r == 0) { success = true; }
{
toku_ft_note_hot_complete(ft_handle, success, msn_at_start_of_hot);
toku_ft_note_hot_complete(brt, success, msn_at_start_of_hot);
}
if (success) {
......
storage/tokudb/ft-index/ft/ft-internal.h
View file @ aef38189
......@@ -92,7 +92,7 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <portability/toku_config.h>
#include "toku_config.h"
#include <toku_race_tools.h>
// Symbol TOKUDB_REVISION is not defined by fractal-tree makefiles, so
......@@ -118,15 +118,11 @@ PATENT RIGHTS GRANT:
#include "bndata.h"
enum { KEY_VALUE_OVERHEAD = 8 }; /* Must store the two lengths. */
enum { FT_MSG_OVERHEAD = (2 + sizeof(MSN)) }; // the type plus freshness plus MSN
enum { FT_CMD_OVERHEAD = (2 + sizeof(MSN)) }; // the type plus freshness plus MSN
enum { FT_DEFAULT_FANOUT = 16 };
enum { FT_DEFAULT_NODE_SIZE = 4 * 1024 * 1024 };
enum { FT_DEFAULT_BASEMENT_NODE_SIZE = 128 * 1024 };
// We optimize for a sequential insert pattern if 100 consecutive injections
// happen into the rightmost leaf node due to promotion.
enum { FT_SEQINSERT_SCORE_THRESHOLD = 100 };
//
// Field in ftnode_fetch_extra that tells the
// partial fetch callback what piece of the node
......@@ -458,7 +454,7 @@ enum {
uint32_t compute_child_fullhash (CACHEFILE cf, FTNODE node, int childnum);
// The ft_header is not managed by the cachetable. Instead, it hangs off the cachefile as userdata.
// The brt_header is not managed by the cachetable. Instead, it hangs off the cachefile as userdata.
enum ft_type {FT_CURRENT=1, FT_CHECKPOINT_INPROGRESS};
......@@ -474,7 +470,7 @@ struct ft_header {
// LSN of creation of "checkpoint-begin" record in log.
LSN checkpoint_lsn;
// see ft_layout_version.h. maybe don't need this if we assume
// see brt_layout_version.h. maybe don't need this if we assume
// it's always the current version after deserializing
const int layout_version;
// different (<) from layout_version if upgraded from a previous
......@@ -508,7 +504,7 @@ struct ft_header {
enum toku_compression_method compression_method;
unsigned int fanout;
// Current Minimum MSN to be used when upgrading pre-MSN FT's.
// Current Minimum MSN to be used when upgrading pre-MSN BRT's.
// This is decremented from our currnt MIN_MSN so as not to clash
// with any existing 'normal' MSN's.
MSN highest_unused_msn_for_upgrade;
......@@ -530,7 +526,7 @@ struct ft_header {
STAT64INFO_S on_disk_stats;
};
// ft_header is always the current version.
// brt_header is always the current version.
struct ft {
FT_HEADER h;
FT_HEADER checkpoint_header;
......@@ -576,22 +572,6 @@ struct ft {
// is this ft a blackhole? if so, all messages are dropped.
bool blackhole;
// The blocknum of the rightmost leaf node in the tree. Stays constant through splits
// and merges using pair-swapping (like the root node, see toku_ftnode_swap_pair_values())
//
// This field only transitions from RESERVED_BLOCKNUM_NULL to non-null, never back.
// We initialize it when promotion inserts into a non-root leaf node on the right extreme.
// We use the blocktable lock to protect the initialize transition, though it's not really
// necessary since all threads should be setting it to the same value. We maintain that invariant
// on first initialization, see ft_set_or_verify_rightmost_blocknum()
BLOCKNUM rightmost_blocknum;
// sequential access pattern heuristic
// - when promotion pushes a message directly into the rightmost leaf, the score goes up.
// - if the score is high enough, we optimistically attempt to insert directly into the rightmost leaf
// - if our attempt fails because the key was not in range of the rightmost leaf, we reset the score back to 0
uint32_t seqinsert_score;
};
// Allocate a DB struct off the stack and only set its comparison
......@@ -728,7 +708,7 @@ void toku_assert_entire_node_in_memory(FTNODE node);
// append a child node to a parent node
void toku_ft_nonleaf_append_child(FTNODE node, FTNODE child, const DBT *pivotkey);
// append a message to a nonleaf node child buffer
// append a cmd to a nonleaf node child buffer
void toku_ft_append_to_child_buffer(ft_compare_func compare_fun, DESCRIPTOR desc, FTNODE node, int childnum, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const DBT *key, const DBT *val);
STAT64INFO_S toku_get_and_clear_basement_stats(FTNODE leafnode);
......@@ -788,7 +768,7 @@ static inline CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT h) {
static const FTNODE null_ftnode=0;
/* an ft cursor is represented as a kv pair in a tree */
/* a brt cursor is represented as a kv pair in a tree */
struct ft_cursor {
struct toku_list cursors_link;
FT_HANDLE ft_handle;
......@@ -989,7 +969,7 @@ __attribute__((nonnull))
void toku_ft_bn_update_max_msn(FTNODE node, MSN max_msn_applied, int child_to_read);
__attribute__((const,nonnull))
size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg);
size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd);
int
toku_ft_search_which_child(
......@@ -1038,25 +1018,26 @@ int toku_ftnode_hot_next_child(FTNODE node,
/* Stuff for testing */
// toku_testsetup_initialize() must be called before any other test_setup_xxx() functions are called.
void toku_testsetup_initialize(void);
int toku_testsetup_leaf(FT_HANDLE ft_h, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
int toku_testsetup_nonleaf (FT_HANDLE ft_h, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
int toku_testsetup_root(FT_HANDLE ft_h, BLOCKNUM);
int toku_testsetup_get_sersize(FT_HANDLE ft_h, BLOCKNUM); // Return the size on disk.
int toku_testsetup_insert_to_leaf (FT_HANDLE ft_h, BLOCKNUM, const char *key, int keylen, const char *val, int vallen);
int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_h, BLOCKNUM, enum ft_msg_type, const char *key, int keylen, const char *val, int vallen);
int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
int toku_testsetup_root(FT_HANDLE brt, BLOCKNUM);
int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM); // Return the size on disk.
int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM, const char *key, int keylen, const char *val, int vallen);
int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM, enum ft_msg_type, const char *key, int keylen, const char *val, int vallen);
void toku_pin_node_with_min_bfe(FTNODE* node, BLOCKNUM b, FT_HANDLE t);
void toku_ft_root_put_msg(FT h, FT_MSG msg, txn_gc_info *gc_info);
// toku_ft_root_put_cmd() accepts non-constant cmd because this is where we set the msn
void toku_ft_root_put_cmd(FT h, FT_MSG_S * cmd, txn_gc_info *gc_info);
void
toku_get_node_for_verify(
BLOCKNUM blocknum,
FT_HANDLE ft_h,
FT_HANDLE brt,
FTNODE* nodep
);
int
toku_verify_ftnode (FT_HANDLE ft_h,
toku_verify_ftnode (FT_HANDLE brt,
MSN rootmsn, MSN parentmsn_with_messages, bool messages_exist_above,
FTNODE node, int height,
const DBT *lesser_pivot, // Everything in the subtree should be > lesser_pivot. (lesser_pivot==NULL if there is no lesser pivot.)
......@@ -1204,11 +1185,6 @@ typedef enum {
FT_PRO_NUM_STOP_LOCK_CHILD,
FT_PRO_NUM_STOP_CHILD_INMEM,
FT_PRO_NUM_DIDNT_WANT_PROMOTE,
FT_BASEMENT_DESERIALIZE_FIXED_KEYSIZE, // how many basement nodes were deserialized with a fixed keysize
FT_BASEMENT_DESERIALIZE_VARIABLE_KEYSIZE, // how many basement nodes were deserialized with a variable keysize
FT_PRO_RIGHTMOST_LEAF_SHORTCUT_SUCCESS,
FT_PRO_RIGHTMOST_LEAF_SHORTCUT_FAIL_POS,
FT_PRO_RIGHTMOST_LEAF_SHORTCUT_FAIL_REACTIVE,
FT_STATUS_NUM_ROWS
} ft_status_entry;
......@@ -1220,9 +1196,9 @@ typedef struct {
void toku_ft_get_status(FT_STATUS);
void
toku_ft_bn_apply_msg_once(
toku_ft_bn_apply_cmd_once (
BASEMENTNODE bn,
const FT_MSG msg,
const FT_MSG cmd,
uint32_t idx,
LEAFENTRY le,
txn_gc_info *gc_info,
......@@ -1231,38 +1207,38 @@ toku_ft_bn_apply_msg_once(
);
void
toku_ft_bn_apply_msg(
toku_ft_bn_apply_cmd (
ft_compare_func compare_fun,
ft_update_func update_fun,
DESCRIPTOR desc,
BASEMENTNODE bn,
FT_MSG msg,
FT_MSG cmd,
txn_gc_info *gc_info,
uint64_t *workdone,
STAT64INFO stats_to_update
);
void
toku_ft_leaf_apply_msg(
toku_ft_leaf_apply_cmd (
ft_compare_func compare_fun,
ft_update_func update_fun,
DESCRIPTOR desc,
FTNODE node,
int target_childnum,
FT_MSG msg,
FT_MSG cmd,
txn_gc_info *gc_info,
uint64_t *workdone,
STAT64INFO stats_to_update
);
void
toku_ft_node_put_msg(
toku_ft_node_put_cmd (
ft_compare_func compare_fun,
ft_update_func update_fun,
DESCRIPTOR desc,
FTNODE node,
int target_childnum,
FT_MSG msg,
FT_MSG cmd,
bool is_fresh,
txn_gc_info *gc_info,
size_t flow_deltas[],
......
storage/tokudb/ft-index/ft/ft-node-deserialize.cc
View file @ aef38189
......@@ -193,7 +193,7 @@ check_node_info_checksum(struct rbuf *rb)
{
int r = 0;
// Verify checksum of header stored.
uint32_t checksum = toku_x1764_memory(rb->buf, rb->ndone);
uint32_t checksum = x1764_memory(rb->buf, rb->ndone);
uint32_t stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
......@@ -229,7 +229,7 @@ check_legacy_end_checksum(struct rbuf *rb)
{
int r = 0;
uint32_t expected_xsum = rbuf_int(rb);
uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size - 4);
uint32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
if (expected_xsum != actual_xsum) {
r = TOKUDB_BAD_CHECKSUM;
}
......
storage/tokudb/ft-index/ft/ft-ops.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ft-ops.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ft-serialize.cc
View file @ aef38189
......@@ -132,7 +132,7 @@ toku_serialize_descriptor_contents_to_fd(int fd, const DESCRIPTOR desc, DISKOFF
toku_serialize_descriptor_contents_to_wbuf(&w, desc);
{
//Add checksum
uint32_t checksum = toku_x1764_finish(&w.checksum);
uint32_t checksum = x1764_finish(&w.checksum);
wbuf_int(&w, checksum);
}
lazy_assert(w.ndone==w.size);
......@@ -185,7 +185,7 @@ deserialize_descriptor_from(int fd, BLOCK_TABLE bt, DESCRIPTOR desc, int layout_
}
{
// check the checksum
uint32_t x1764 = toku_x1764_memory(dbuf, size-4);
uint32_t x1764 = x1764_memory(dbuf, size-4);
//printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
uint32_t stored_x1764 = toku_dtoh32(*(int*)(dbuf + size-4));
if (x1764 != stored_x1764) {
......@@ -210,7 +210,7 @@ exit:
int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
// Effect: Deserialize the ft header.
// We deserialize ft_header only once and then share everything with all the FTs.
// We deserialize brt header only once and then share everything with all the brts.
{
int r;
FT ft = NULL;
......@@ -462,8 +462,6 @@ serialize_ft_min_size (uint32_t version) {
size_t size = 0;
switch(version) {
case FT_LAYOUT_VERSION_27:
case FT_LAYOUT_VERSION_26:
case FT_LAYOUT_VERSION_25:
case FT_LAYOUT_VERSION_24:
case FT_LAYOUT_VERSION_23:
......@@ -625,7 +623,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd,
//Verify checksum (FT_LAYOUT_VERSION_13 or later, when checksum function changed)
uint32_t calculated_x1764;
calculated_x1764 = toku_x1764_memory(rb->buf, rb->size-4);
calculated_x1764 = x1764_memory(rb->buf, rb->size-4);
uint32_t stored_x1764;
stored_x1764 = toku_dtoh32(*(int*)(rb->buf+rb->size-4));
if (calculated_x1764 != stored_x1764) {
......@@ -805,7 +803,7 @@ void toku_serialize_ft_to_wbuf (
wbuf_char(wbuf, (unsigned char) h->compression_method);
wbuf_MSN(wbuf, h->highest_unused_msn_for_upgrade);
wbuf_MSN(wbuf, h->max_msn_in_ft);
uint32_t checksum = toku_x1764_finish(&wbuf->checksum);
uint32_t checksum = x1764_finish(&wbuf->checksum);
wbuf_int(wbuf, checksum);
lazy_assert(wbuf->ndone == wbuf->size);
}
......
storage/tokudb/ft-index/ft/ft-test-helpers.cc
View file @ aef38189
......@@ -96,7 +96,7 @@ PATENT RIGHTS GRANT:
#include "fttypes.h"
#include "ule.h"
// dummymsn needed to simulate msn because messages are injected at a lower level than toku_ft_root_put_msg()
// dummymsn needed to simulate msn because messages are injected at a lower level than toku_ft_root_put_cmd()
#define MIN_DUMMYMSN ((MSN) {(uint64_t)1 << 62})
static MSN dummymsn;
static int testsetup_initialized = 0;
......@@ -119,10 +119,10 @@ next_dummymsn(void) {
bool ignore_if_was_already_open;
int toku_testsetup_leaf(FT_HANDLE ft_handle, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens) {
int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens) {
FTNODE node;
assert(testsetup_initialized);
toku_create_new_ftnode(ft_handle, &node, 0, n_children);
toku_create_new_ftnode(brt, &node, 0, n_children);
int i;
for (i=0; i<n_children; i++) {
BP_STATE(node,i) = PT_AVAIL;
......@@ -134,15 +134,15 @@ int toku_testsetup_leaf(FT_HANDLE ft_handle, BLOCKNUM *blocknum, int n_children,
}
*blocknum = node->thisnodename;
toku_unpin_ftnode(ft_handle->ft, node);
toku_unpin_ftnode(brt->ft, node);
return 0;
}
// Don't bother to clean up carefully if something goes wrong. (E.g., it's OK to have malloced stuff that hasn't been freed.)
int toku_testsetup_nonleaf (FT_HANDLE ft_handle, int height, BLOCKNUM *blocknum, int n_children, BLOCKNUM *children, char **keys, int *keylens) {
int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *blocknum, int n_children, BLOCKNUM *children, char **keys, int *keylens) {
FTNODE node;
assert(testsetup_initialized);
toku_create_new_ftnode(ft_handle, &node, height, n_children);
toku_create_new_ftnode(brt, &node, height, n_children);
int i;
for (i=0; i<n_children; i++) {
BP_BLOCKNUM(node, i) = children[i];
......@@ -153,28 +153,28 @@ int toku_testsetup_nonleaf (FT_HANDLE ft_handle, int height, BLOCKNUM *blocknum,
node->totalchildkeylens += keylens[i];
}
*blocknum = node->thisnodename;
toku_unpin_ftnode(ft_handle->ft, node);
toku_unpin_ftnode(brt->ft, node);
return 0;
}
int toku_testsetup_root(FT_HANDLE ft_handle, BLOCKNUM blocknum) {
int toku_testsetup_root(FT_HANDLE brt, BLOCKNUM blocknum) {
assert(testsetup_initialized);
ft_handle->ft->h->root_blocknum = blocknum;
brt->ft->h->root_blocknum = blocknum;
return 0;
}
int toku_testsetup_get_sersize(FT_HANDLE ft_handle, BLOCKNUM diskoff) // Return the size on disk
int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM diskoff) // Return the size on disk
{
assert(testsetup_initialized);
void *node_v;
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, ft_handle->ft);
fill_bfe_for_full_read(&bfe, brt->ft);
int r = toku_cachetable_get_and_pin(
ft_handle->ft->cf, diskoff,
toku_cachetable_hash(ft_handle->ft->cf, diskoff),
brt->ft->cf, diskoff,
toku_cachetable_hash(brt->ft->cf, diskoff),
&node_v,
NULL,
get_write_callbacks_for_node(ft_handle->ft),
get_write_callbacks_for_node(brt->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
......@@ -184,25 +184,25 @@ int toku_testsetup_get_sersize(FT_HANDLE ft_handle, BLOCKNUM diskoff) // Return
assert(r==0);
FTNODE CAST_FROM_VOIDP(node, node_v);
int size = toku_serialize_ftnode_size(node);
toku_unpin_ftnode(ft_handle->ft, node);
toku_unpin_ftnode(brt->ft, node);
return size;
}
int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const char *key, int keylen, const char *val, int vallen) {
int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM blocknum, const char *key, int keylen, const char *val, int vallen) {
void *node_v;
int r;
assert(testsetup_initialized);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, ft_handle->ft);
fill_bfe_for_full_read(&bfe, brt->ft);
r = toku_cachetable_get_and_pin(
ft_handle->ft->cf,
brt->ft->cf,
blocknum,
toku_cachetable_hash(ft_handle->ft->cf, blocknum),
toku_cachetable_hash(brt->ft->cf, blocknum),
&node_v,
NULL,
get_write_callbacks_for_node(ft_handle->ft),
get_write_callbacks_for_node(brt->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
......@@ -216,19 +216,19 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const
DBT keydbt,valdbt;
MSN msn = next_dummymsn();
FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(),
FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(),
.u = { .id = { toku_fill_dbt(&keydbt, key, keylen),
toku_fill_dbt(&valdbt, val, vallen) } } };
static size_t zero_flow_deltas[] = { 0, 0 };
txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
toku_ft_node_put_msg (
ft_handle->ft->compare_fun,
ft_handle->ft->update_fun,
&ft_handle->ft->cmp_descriptor,
toku_ft_node_put_cmd (
brt->ft->compare_fun,
brt->ft->update_fun,
&brt->ft->cmp_descriptor,
node,
-1,
&msg,
&cmd,
true,
&gc_info,
zero_flow_deltas,
......@@ -237,7 +237,7 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const
toku_verify_or_set_counts(node);
toku_unpin_ftnode(ft_handle->ft, node);
toku_unpin_ftnode(brt->ft, node);
return 0;
}
......@@ -254,32 +254,33 @@ toku_pin_node_with_min_bfe(FTNODE* node, BLOCKNUM b, FT_HANDLE t)
{
struct ftnode_fetch_extra bfe;
fill_bfe_for_min_read(&bfe, t->ft);
toku_pin_ftnode(
toku_pin_ftnode_off_client_thread(
t->ft,
b,
toku_cachetable_hash(t->ft->cf, b),
&bfe,
PL_WRITE_EXPENSIVE,
node,
true
0,
NULL,
node
);
}
int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, enum ft_msg_type msgtype, const char *key, int keylen, const char *val, int vallen) {
int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM blocknum, enum ft_msg_type cmdtype, const char *key, int keylen, const char *val, int vallen) {
void *node_v;
int r;
assert(testsetup_initialized);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, ft_handle->ft);
fill_bfe_for_full_read(&bfe, brt->ft);
r = toku_cachetable_get_and_pin(
ft_handle->ft->cf,
brt->ft->cf,
blocknum,
toku_cachetable_hash(ft_handle->ft->cf, blocknum),
toku_cachetable_hash(brt->ft->cf, blocknum),
&node_v,
NULL,
get_write_callbacks_for_node(ft_handle->ft),
get_write_callbacks_for_node(brt->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
......@@ -293,19 +294,19 @@ int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, en
DBT k;
int childnum = toku_ftnode_which_child(node,
toku_fill_dbt(&k, key, keylen),
&ft_handle->ft->cmp_descriptor, ft_handle->ft->compare_fun);
&brt->ft->cmp_descriptor, brt->ft->compare_fun);
XIDS xids_0 = xids_get_root_xids();
MSN msn = next_dummymsn();
toku_bnc_insert_msg(BNC(node, childnum), key, keylen, val, vallen, msgtype, msn, xids_0, true, NULL, testhelper_string_key_cmp);
toku_bnc_insert_msg(BNC(node, childnum), key, keylen, val, vallen, cmdtype, msn, xids_0, true, NULL, testhelper_string_key_cmp);
// Hack to get the test working. The problem is that this test
// is directly queueing something in a FIFO instead of
// using ft APIs.
// using brt APIs.
node->max_msn_applied_to_node_on_disk = msn;
node->dirty = 1;
// Also hack max_msn_in_ft
ft_handle->ft->h->max_msn_in_ft = msn;
brt->ft->h->max_msn_in_ft = msn;
toku_unpin_ftnode(ft_handle->ft, node);
toku_unpin_ftnode(brt->ft, node);
return 0;
}
storage/tokudb/ft-index/ft/ft-verify.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ft.cc
View file @ aef38189
......@@ -309,14 +309,16 @@ static void ft_close(CACHEFILE cachefile, int fd, void *header_v, bool oplsn_val
}
}
if (ft->h->dirty) { // this is the only place this bit is tested (in currentheader)
if (logger) { //Rollback cachefile MUST NOT BE CLOSED DIRTY
//It can be checkpointed only via 'checkpoint'
assert(logger->rollback_cachefile != cachefile);
bool do_checkpoint = true;
if (logger && logger->rollback_cachefile == cachefile) {
do_checkpoint = false;
}
if (do_checkpoint) {
ft_begin_checkpoint(lsn, header_v);
ft_checkpoint(cachefile, fd, ft);
ft_end_checkpoint(cachefile, fd, header_v);
assert(!ft->h->dirty); // dirty bit should be cleared by begin_checkpoint and never set again (because we're closing the dictionary)
}
ft_begin_checkpoint(lsn, header_v);
ft_checkpoint(cachefile, fd, ft);
ft_end_checkpoint(cachefile, fd, header_v);
assert(!ft->h->dirty); // dirty bit should be cleared by begin_checkpoint and never set again (because we're closing the dictionary)
}
}
......@@ -465,8 +467,8 @@ void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn) {
*ftp = ft;
}
// TODO: (Zardosht) get rid of ft parameter
int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_handle, CACHEFILE cf, LSN max_acceptable_lsn, FT *header)
// TODO: (Zardosht) get rid of brt parameter
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header)
// If the cachefile already has the header, then just get it.
// If the cachefile has not been initialized, then don't modify anything.
// max_acceptable_lsn is the latest acceptable checkpointed version of the file.
......@@ -475,8 +477,8 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_handle, CACHEFILE cf, LSN
FT h;
if ((h = (FT) toku_cachefile_get_userdata(cf))!=0) {
*header = h;
assert(ft_handle->options.update_fun == h->update_fun);
assert(ft_handle->options.compare_fun == h->compare_fun);
assert(brt->options.update_fun == h->update_fun);
assert(brt->options.compare_fun == h->compare_fun);
return 0;
}
}
......@@ -494,8 +496,8 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_handle, CACHEFILE cf, LSN
// GCC 4.8 seems to get confused by the gotos in the deserialize code and think h is maybe uninitialized.
invariant_notnull(h);
h->cf = cf;
h->compare_fun = ft_handle->options.compare_fun;
h->update_fun = ft_handle->options.update_fun;
h->compare_fun = brt->options.compare_fun;
h->update_fun = brt->options.update_fun;
toku_cachefile_set_userdata(cf,
(void*)h,
ft_log_fassociate_during_checkpoint,
......@@ -557,13 +559,13 @@ FT_HANDLE toku_ft_get_only_existing_ft_handle(FT h) {
return ft_handle_ret;
}
// Purpose: set fields in ft_header to capture accountability info for start of HOT optimize.
// Purpose: set fields in brt_header to capture accountability info for start of HOT optimize.
// Note: HOT accountability variables in header are modified only while holding header lock.
// (Header lock is really needed for touching the dirty bit, but it's useful and
// convenient here for keeping the HOT variables threadsafe.)
void
toku_ft_note_hot_begin(FT_HANDLE ft_handle) {
FT ft = ft_handle->ft;
toku_ft_note_hot_begin(FT_HANDLE brt) {
FT ft = brt->ft;
time_t now = time(NULL);
// hold lock around setting and clearing of dirty bit
......@@ -576,11 +578,11 @@ toku_ft_note_hot_begin(FT_HANDLE ft_handle) {
}
// Purpose: set fields in ft_header to capture accountability info for end of HOT optimize.
// Purpose: set fields in brt_header to capture accountability info for end of HOT optimize.
// Note: See note for toku_ft_note_hot_begin().
void
toku_ft_note_hot_complete(FT_HANDLE ft_handle, bool success, MSN msn_at_start_of_hot) {
FT ft = ft_handle->ft;
toku_ft_note_hot_complete(FT_HANDLE brt, bool success, MSN msn_at_start_of_hot) {
FT ft = brt->ft;
time_t now = time(NULL);
toku_ft_lock(ft);
......@@ -626,7 +628,7 @@ toku_ft_init(FT ft,
ft->h->checkpoint_lsn = checkpoint_lsn;
}
// Open an ft for use by redirect. The new ft must have the same dict_id as the old_ft passed in. (FILENUM is assigned by the ft_handle_open() function.)
// Open a brt for use by redirect. The new brt must have the same dict_id as the old_ft passed in. (FILENUM is assigned by the ft_handle_open() function.)
static int
ft_handle_open_for_redirect(FT_HANDLE *new_ftp, const char *fname_in_env, TOKUTXN txn, FT old_h) {
FT_HANDLE t;
......@@ -664,9 +666,9 @@ dictionary_redirect_internal(const char *dst_fname_in_env, FT src_h, TOKUTXN txn
FT dst_h = NULL;
struct toku_list *list;
// open a dummy ft based off of
// open a dummy brt based off of
// dst_fname_in_env to get the header
// then we will change all the ft's to have
// then we will change all the brt's to have
// their headers point to dst_h instead of src_h
FT_HANDLE tmp_dst_ft = NULL;
r = ft_handle_open_for_redirect(&tmp_dst_ft, dst_fname_in_env, txn, src_h);
......@@ -680,7 +682,7 @@ dictionary_redirect_internal(const char *dst_fname_in_env, FT src_h, TOKUTXN txn
assert(dst_filenum.fileid!=FILENUM_NONE.fileid);
assert(dst_filenum.fileid!=src_filenum.fileid); //Cannot be same file.
// for each live ft_handle, ft_handle->ft is currently src_h
// for each live brt, brt->ft is currently src_h
// we want to change it to dummy_dst
toku_ft_grab_reflock(src_h);
while (!toku_list_empty(&src_h->live_ft_handles)) {
......@@ -720,7 +722,7 @@ toku_dictionary_redirect_abort(FT old_h, FT new_h, TOKUTXN txn) {
FILENUM new_filenum = toku_cachefile_filenum(new_h->cf);
assert(old_filenum.fileid!=new_filenum.fileid); //Cannot be same file.
//No living fts in old header.
//No living brts in old header.
toku_ft_grab_reflock(old_h);
assert(toku_list_empty(&old_h->live_ft_handles));
toku_ft_release_reflock(old_h);
......@@ -738,13 +740,13 @@ toku_dictionary_redirect_abort(FT old_h, FT new_h, TOKUTXN txn) {
/****
* on redirect or abort:
* if redirect txn_note_doing_work(txn)
* if redirect connect src ft to txn (txn modified this ft)
* for each src ft
* open ft to dst file (create new ft struct)
* if redirect connect dst ft to txn
* redirect db to new ft
* redirect cursors to new ft
* close all src fts
* if redirect connect src brt to txn (txn modified this brt)
* for each src brt
* open brt to dst file (create new brt struct)
* if redirect connect dst brt to txn
* redirect db to new brt
* redirect cursors to new brt
* close all src brts
* if redirect make rollback log entry
*
* on commit:
......@@ -756,21 +758,21 @@ int
toku_dictionary_redirect (const char *dst_fname_in_env, FT_HANDLE old_ft_h, TOKUTXN txn) {
// Input args:
// new file name for dictionary (relative to env)
// old_ft_h is a live ft of open handle ({DB, FT_HANDLE} pair) that currently refers to old dictionary file.
// old_ft_h is a live brt of open handle ({DB, BRT} pair) that currently refers to old dictionary file.
// (old_ft_h may be one of many handles to the dictionary.)
// txn that created the loader
// Requires:
// multi operation lock is held.
// The ft is open. (which implies there can be no zombies.)
// The brt is open. (which implies there can be no zombies.)
// The new file must be a valid dictionary.
// The block size and flags in the new file must match the existing FT.
// The block size and flags in the new file must match the existing BRT.
// The new file must already have its descriptor in it (and it must match the existing descriptor).
// Effect:
// Open new FTs (and related header and cachefile) to the new dictionary file with a new FILENUM.
// Redirect all DBs that point to fts that point to the old file to point to fts that point to the new file.
// Redirect all DBs that point to brts that point to the old file to point to brts that point to the new file.
// Copy the dictionary id (dict_id) from the header of the original file to the header of the new file.
// Create a rollback log entry.
// The original FT, header, cachefile and file remain unchanged. They will be cleaned up on commmit.
// The original BRT, header, cachefile and file remain unchanged. They will be cleaned up on commmit.
// If the txn aborts, then this operation will be undone
int r;
......@@ -1077,8 +1079,8 @@ garbage_helper(BLOCKNUM blocknum, int64_t UU(size), int64_t UU(address), void *e
goto exit;
}
for (int i = 0; i < node->n_children; ++i) {
bn_data* bd = BLB_DATA(node, i);
r = bd->iterate<struct garbage_helper_extra, garbage_leafentry_helper>(info);
BN_DATA bd = BLB_DATA(node, i);
r = bd->omt_iterate<struct garbage_helper_extra, garbage_leafentry_helper>(info);
if (r != 0) {
goto exit;
}
......
storage/tokudb/ft-index/ft/ft.h
View file @ aef38189
......@@ -113,7 +113,7 @@ void toku_ft_release_reflock(FT ft);
void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn);
void toku_ft_free (FT h);
int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_h, CACHEFILE cf, LSN max_acceptable_lsn, FT *header);
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header);
void toku_ft_note_ft_handle_open(FT ft, FT_HANDLE live);
bool toku_ft_needed_unlocked(FT ft);
......@@ -125,8 +125,8 @@ void toku_ft_evict_from_memory(FT ft, bool oplsn_valid, LSN oplsn);
FT_HANDLE toku_ft_get_only_existing_ft_handle(FT h);
void toku_ft_note_hot_begin(FT_HANDLE ft_h);
void toku_ft_note_hot_complete(FT_HANDLE ft_h, bool success, MSN msn_at_start_of_hot);
void toku_ft_note_hot_begin(FT_HANDLE brt);
void toku_ft_note_hot_complete(FT_HANDLE brt, bool success, MSN msn_at_start_of_hot);
void
toku_ft_init(
......
storage/tokudb/ft-index/ft/ft_layout_version.h
View file @ aef38189
......@@ -99,7 +99,7 @@ enum ft_layout_version_e {
FT_LAYOUT_VERSION_7 = 7, // Diff from 6 to 7: Add exact-bit to leafentry_estimate #818, add magic to header #22, add per-subdatase flags #333
FT_LAYOUT_VERSION_8 = 8, // Diff from 7 to 8: Use murmur instead of crc32. We are going to make a simplification and stop supporting version 7 and before. Current As of Beta 1.0.6
FT_LAYOUT_VERSION_9 = 9, // Diff from 8 to 9: Variable-sized blocks and compression.
FT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from ft layer
FT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from brt layer
FT_LAYOUT_VERSION_11 = 11, // Diff from 10 to 11: Nested transaction leafentries (completely redesigned). FT_CMDs on disk now support XIDS (multiple txnids) instead of exactly one.
FT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added FT_CMD 'FT_INSERT_NO_OVERWRITE', compressed block format, num old blocks
FT_LAYOUT_VERSION_13 = 13, // Diff from 12 to 13: Fixed loader pivot bug, added build_id to every node, timestamps to ft
......@@ -107,8 +107,8 @@ enum ft_layout_version_e {
FT_LAYOUT_VERSION_15 = 15, // Diff from 14 to 15: basement nodes, last verification time
FT_LAYOUT_VERSION_16 = 16, // Dr. No: No subtree estimates, partition layout information represented more transparently.
// ALERT ALERT ALERT: version 16 never released to customers, internal and beta use only
FT_LAYOUT_VERSION_17 = 17, // Dr. No: Add STAT64INFO_S to ft header
FT_LAYOUT_VERSION_18 = 18, // Dr. No: Add HOT info to ft header
FT_LAYOUT_VERSION_17 = 17, // Dr. No: Add STAT64INFO_S to brt_header
FT_LAYOUT_VERSION_18 = 18, // Dr. No: Add HOT info to brt_header
FT_LAYOUT_VERSION_19 = 19, // Doofenshmirtz: Add compression method, highest_unused_msn_for_upgrade
FT_LAYOUT_VERSION_20 = 20, // Deadshot: Add compression method to log_fcreate,
// mgr_last_xid after begin checkpoint,
......@@ -119,8 +119,6 @@ enum ft_layout_version_e {
FT_LAYOUT_VERSION_23 = 23, // Ming: Fix upgrade path #5902
FT_LAYOUT_VERSION_24 = 24, // Riddler: change logentries that log transactions to store TXNID_PAIRs instead of TXNIDs
FT_LAYOUT_VERSION_25 = 25, // SecretSquirrel: ROLLBACK_LOG_NODES (on disk and in memory) now just use blocknum (instead of blocknum + hash) to point to other log nodes. same for xstillopen log entry
FT_LAYOUT_VERSION_26 = 26, // Hojo: basements store key/vals separately on disk for fixed klpair length BNs
FT_LAYOUT_VERSION_27 = 27, // serialize message trees with nonleaf buffers to avoid key, msn sort on deserialize
FT_NEXT_VERSION, // the version after the current version
FT_LAYOUT_VERSION = FT_NEXT_VERSION-1, // A hack so I don't have to change this line.
FT_LAYOUT_MIN_SUPPORTED_VERSION = FT_LAYOUT_VERSION_13, // Minimum version supported
......
storage/tokudb/ft-index/ft/ftloader-internal.h
View file @ aef38189
......@@ -257,7 +257,7 @@ struct ft_loader_s {
int progress_callback_result; // initially zero, if any call to the poll function callback returns nonzero, we save the result here (and don't call the poll callback function again).
LSN load_lsn; //LSN of the fsynced 'load' log entry. Write this LSN (as checkpoint_lsn) in ft headers made by this loader.
LSN load_lsn; //LSN of the fsynced 'load' log entry. Write this LSN (as checkpoint_lsn) in brt headers made by this loader.
TXNID load_root_xid; //(Root) transaction that performed the load.
QUEUE *fractal_queues; // an array of work queues, one for each secondary index.
......@@ -280,7 +280,7 @@ uint64_t toku_ft_loader_get_n_rows(FTLOADER bl);
struct fractal_thread_args {
FTLOADER bl;
const DESCRIPTOR descriptor;
int fd; // write the ft into fd.
int fd; // write the brt into tfd.
int progress_allocation;
QUEUE q;
uint64_t total_disksize_estimate;
......@@ -312,17 +312,17 @@ int toku_merge_some_files_using_dbufio (const bool to_q, FIDX dest_data, QUEUE q
int ft_loader_sort_and_write_rows (struct rowset *rows, struct merge_fileset *fs, FTLOADER bl, int which_db, DB *dest_db, ft_compare_func);
// This is probably only for testing.
int toku_loader_write_ft_from_q_in_C (FTLOADER bl,
const DESCRIPTOR descriptor,
int fd, // write to here
int progress_allocation,
QUEUE q,
uint64_t total_disksize_estimate,
int which_db,
uint32_t target_nodesize,
uint32_t target_basementnodesize,
enum toku_compression_method target_compression_method,
uint32_t fanout);
int toku_loader_write_brt_from_q_in_C (FTLOADER bl,
const DESCRIPTOR descriptor,
int fd, // write to here
int progress_allocation,
QUEUE q,
uint64_t total_disksize_estimate,
int which_db,
uint32_t target_nodesize,
uint32_t target_basementnodesize,
enum toku_compression_method target_compression_method,
uint32_t fanout);
int ft_loader_mergesort_row_array (struct row rows[/*n*/], int n, int which_db, DB *dest_db, ft_compare_func, FTLOADER, struct rowset *);
......@@ -339,7 +339,7 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp,
CACHETABLE cachetable,
generate_row_for_put_func g,
DB *src_db,
int N, FT_HANDLE ft_hs[/*N*/], DB* dbs[/*N*/],
int N, FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
const char *new_fnames_in_env[/*N*/],
ft_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,
......
storage/tokudb/ft-index/ft/ftloader.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ftloader.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/fttypes.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ftverify.cc
View file @ aef38189
......@@ -98,6 +98,7 @@ PATENT RIGHTS GRANT:
#include "ft-internal.h"
#include "ft_layout_version.h"
#include "block_table.h"
#include "x1764.h"
#include "rbuf.h"
#include "sub_block.h"
......
storage/tokudb/ft-index/ft/hash-benchmarks/hash-benchmark.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/le-cursor.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/leafentry.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/locking-benchmarks/process_locks.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/locking-benchmarks/pthread-locks.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/locking-benchmarks/trylock.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/log-internal.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/log.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/log_upgrade.cc
View file @ aef38189
......@@ -321,8 +321,8 @@ toku_maybe_upgrade_log(const char *env_dir, const char *log_dir, LSN * lsn_of_cl
r = 0; //Logs are up to date
else {
FOOTPRINT(4);
LSN last_lsn = ZERO_LSN;
TXNID last_xid = TXNID_NONE;
LSN last_lsn;
TXNID last_xid;
r = verify_clean_shutdown_of_log_version(log_dir, version_of_logs_on_disk, &last_lsn, &last_xid);
if (r != 0) {
goto cleanup;
......
storage/tokudb/ft-index/ft/logcursor.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/logformat.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/logger.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/logger.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/omt.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/omt.h 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/quicklz.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/rbuf.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/recover.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/recover.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/roll.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/rollback-apply.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/rollback.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/sub_block.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/tests/dmt-test2.cc deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/ft-index/ft/tests/ft-test.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/tests/test.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/tokuftdump.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/txn.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/txn.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/txn_manager.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ule-internal.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/ule.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/wbuf.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/x1764-speedup/x1764-speedup-test.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/xids-internal.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/ft/xids.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/include/CMakeLists.txt 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/include/tdb-internal.h 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/locktree/locktree.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/locktree/manager.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/portability/huge_page_detection.h 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/loader.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/bdb.suppressions 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/cilktests/bar2.cilk 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/cilktests/foo.cilk 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/cilktests/foo2.cpp 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/cilktests/get.cilk 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/simple.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/test.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/test1426.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/test_txn_abort9.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/test_txn_commit8.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/xa-dirty-commit.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/xa-dirty-rollback.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/xa-txn-discard-abort.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/tests/xa-txn-discard-commit.cc 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb-internal.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_cursor.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_db.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_db.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_lib.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_load.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_row_lock.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_txn.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/src/ydb_write.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/toku_include/CMakeLists.txt 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/toku_include/old-db.h 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/util/CMakeLists.txt
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/util/context.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/util/dmt.cc deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/ft-index/util/dmt.h deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/ft-index/util/fmutex.h deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/ft-index/util/kibbutz.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/util/mempool.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/util/mempool.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/util/omt.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ft-index/util/tests/omt-test.cc deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/ha_tokudb.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/ha_tokudb.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/ha_tokudb_alter_56.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/hatoku_hton.cc
View file @ aef38189
This diff is collapsed.
storage/tokudb/hatoku_hton.h
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb/r/bf_delete_nonpart.result 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb/r/bf_delete_trigger.result 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb/r/bf_select_part.result 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb/t/bf_delete_nonpart.test 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb/t/bf_delete_trigger.test 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb/t/bf_select_part.test 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb_bugs/r/part_index_scan.result 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/mysql-test/tokudb_bugs/t/part_index_scan.test 0 → 100644
View file @ aef38189
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/nodistinct.q8.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q0.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q0.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q1.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q1.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q2.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q2.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q3.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q3.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q4.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q4.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q5.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q5.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q6.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q6.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q7.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q7.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.10y.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.10y.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.1y.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.1y.year5.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.2y.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.3y.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.4y.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q8.year5.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q9.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/q9.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/qcount.main.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/qcount.result deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.ontime/qcount.sql deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/atc.readme deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/run.atc.ontime.bash deleted 100755 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/run.iibench.bash deleted 100755 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/run.mysql.tests.bash deleted 100755 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/run.sql.bench.bash deleted 100755 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/run.tpch.bash deleted 100755 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/setup.mysql.bash deleted 100755 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/test.mysql.bash deleted 100755 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/testbuildfromsrc.bash deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/testsandbox.bash deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
storage/tokudb/scripts/tpch.readme deleted 100644 → 0
View file @ c799d65b
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7