Commit 7bf391c2 authored by Jan Lindström's avatar Jan Lindström

MDEV-7108: Make long semaphore wait timeout configurable

Merge Facebook commit https://github.com/facebook/mysql-5.6/commit/cd063ab930f05efdba39d504543998512d1bd71f
authored by Peng Tian from https://github.com/facebook/mysql-5.6

Introduced a new configuration variable innodb_fatal_semaphore_wait_threshold,
it makes the fatal semaphore timeout configurable. Modified original commit
so that no MariaDB server files are changed, instead introduced a new
InnoDB/XtraDB configuration variable.

Its default/min/max vlaues are 600/1/2^32-1 in seconds (it was hardcoded
as 600, now its default value is 600, so the default behavior of this diff
should be no change).
parent ea832268
# Establish connection con1 (user=root)
# Establish connection con2 (user=root)
drop table if exists t1;
# Switch to connection con1
create table t1 (id integer, x integer) engine = InnoDB;
insert into t1 values(0, 0);
set @@debug_dbug="d,fatal-semaphore-timeout";
set autocommit=0;
# Sending query on con1,
# the session will hold lock table mutex and sleep
SELECT * from t1 where id = 0 FOR UPDATE;
# Switch to connection con2
set autocommit=0;
# Sending query on con2,
# the session will be blocked on the lock table mutex and
# thus be put into sync arry
SELECT * from t1 where id = 0 FOR UPDATE;
# Switched to the default connection
# Waitting for mysqld to crash
# Mysqld crash was detected
# Waitting for reconnect after mysqld restarts
# Reconnected after mysqld was successfully restarted
# Cleaning up before exit
drop table if exists t1;
# Clean exit
select @@global.innodb_fatal_semaphore_wait_threshold;
@@global.innodb_fatal_semaphore_wait_threshold
600
select @@session.innodb_fatal_semaphore_wait_threshold;
ERROR HY000: Variable 'innodb_fatal_semaphore_wait_threshold' is a GLOBAL variable
show global variables like 'innodb_fatal_semaphore_wait_threshold';
Variable_name Value
innodb_fatal_semaphore_wait_threshold 600
show session variables like 'innodb_fatal_semaphore_wait_threshold';
Variable_name Value
innodb_fatal_semaphore_wait_threshold 600
select * from information_schema.global_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
VARIABLE_NAME VARIABLE_VALUE
INNODB_FATAL_SEMAPHORE_WAIT_THRESHOLD 600
select * from information_schema.session_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
VARIABLE_NAME VARIABLE_VALUE
INNODB_FATAL_SEMAPHORE_WAIT_THRESHOLD 600
set global innodb_fatal_semaphore_wait_threshold=1;
ERROR HY000: Variable 'innodb_fatal_semaphore_wait_threshold' is a read only variable
set session innodb_fatal_semaphore_wait_threshold=1;
ERROR HY000: Variable 'innodb_fatal_semaphore_wait_threshold' is a read only variable
......@@ -663,6 +663,20 @@ NUMERIC_BLOCK_SIZE 0
ENUM_VALUE_LIST NULL
READ_ONLY NO
COMMAND_LINE_ARGUMENT OPTIONAL
VARIABLE_NAME INNODB_FATAL_SEMAPHORE_WAIT_THRESHOLD
SESSION_VALUE NULL
GLOBAL_VALUE 600
GLOBAL_VALUE_ORIGIN COMPILE-TIME
DEFAULT_VALUE 600
VARIABLE_SCOPE GLOBAL
VARIABLE_TYPE BIGINT UNSIGNED
VARIABLE_COMMENT Maximum number of seconds that semaphore times out in InnoDB.
NUMERIC_MIN_VALUE 1
NUMERIC_MAX_VALUE 4294967295
NUMERIC_BLOCK_SIZE 0
ENUM_VALUE_LIST NULL
READ_ONLY YES
COMMAND_LINE_ARGUMENT REQUIRED
VARIABLE_NAME INNODB_FILE_FORMAT
SESSION_VALUE NULL
GLOBAL_VALUE Antelope
......
# Only test in debug mode since DBUG_EXECUTE_IF is used
--source include/have_debug.inc
# Can't test this with embedded server
--source include/not_embedded.inc
# Don't test this with valgrind
--source include/not_valgrind.inc
# Only test for innodb
--source include/have_innodb.inc
--echo # Establish connection con1 (user=root)
connect (con1,localhost,root,,);
--echo # Establish connection con2 (user=root)
connect (con2,localhost,root,,);
--disable_warnings
drop table if exists t1;
--enable_warnings
--echo # Switch to connection con1
connection con1;
eval create table t1 (id integer, x integer) engine = InnoDB;
insert into t1 values(0, 0);
# Enable the debug injection.
set @@debug_dbug="d,fatal-semaphore-timeout";
set autocommit=0;
# The following query will hang for an hour since the debug injection
# code will sleep an hour after holding the lock table mutex
--echo # Sending query on con1,
--echo # the session will hold lock table mutex and sleep
--send
SELECT * from t1 where id = 0 FOR UPDATE;
# To make sure con1 holding the lock table mutex and sleeping
--sleep 2
--echo # Switch to connection con2
connection con2;
set autocommit=0;
# The following query will be blocked on the lock table mutex held by
# con1 so it will be put into sync array.
--echo # Sending query on con2,
--echo # the session will be blocked on the lock table mutex and
--echo # thus be put into sync arry
--send
SELECT * from t1 where id = 0 FOR UPDATE;
# Waitting for mysqld to abort due to fatal semaphore timeout.
# Please note that, in the master.opt file, the fatal timeout
# was set to 1 second, but in mysqld debug mode, this timeout
# value will be timed 10 because UNIV_DEBUG_VALGRIND is set
# (see sync_array_print_long_waits_low() in storage/innobase/sync/sync0arr.cc)
# so the actual timeout will be 1 * 10 = 10 seconds. Besides,
# mysqld will abort after detecting this fatal timeout 10 times in
# a loop with interval of 1 second (see srv_error_monitor_thread
# thread in torage/innobase/srv/srv0srv.cc), so mysqld will abort
# in 1 * 10 + 1 * 10 = 20 seconds after con2 being blocked on
# the lock table mutex.
#
# P.S. the default fatal sempahore timeout is 600 seconds,
# so mysqld will abort after 600 * 10 + 1 * 10 = 6010 seconds
# in debug mode and 600 + 1 * 10 = 610 seconds in release mode.
--echo # Switched to the default connection
connection default;
--disable_result_log
--disable_query_log
# Since this test generates lot of errors in log, suppress checking errors
call mtr.add_suppression(".*");
# The crash is expected
exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
--echo # Waitting for mysqld to crash
# It will take 20 seconds to detect the long semaphore and mysqld to abort.
# This test will be treated as pass as long as mysqld crash/restart is dectected
# in 60 seconds.
let $counter= 60;
let $mysql_errno= 0;
while (!$mysql_errno)
{
--error 0,1040,1053,2002,2003,2006,2013
show status;
dec $counter;
if (!$counter)
{
# This will fail this test.
--die Server failed to dissapear
}
--sleep 1
}
--echo # Mysqld crash was detected
--echo # Waitting for reconnect after mysqld restarts
enable_reconnect;
connection default;
--exec echo "restart:--log-error=$error_log" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
# Call script that will poll the server waiting for it to be back online again
source include/wait_until_connected_again.inc;
--echo # Reconnected after mysqld was successfully restarted
--echo # Cleaning up before exit
--disable_warnings
drop table if exists t1;
--enable_warnings
--echo # Clean exit
--source include/have_innodb.inc
# bool readonly
#
# show values;
#
select @@global.innodb_fatal_semaphore_wait_threshold;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
select @@session.innodb_fatal_semaphore_wait_threshold;
show global variables like 'innodb_fatal_semaphore_wait_threshold';
show session variables like 'innodb_fatal_semaphore_wait_threshold';
select * from information_schema.global_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
select * from information_schema.session_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
#
# show that it's read-only
#
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set global innodb_fatal_semaphore_wait_threshold=1;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set session innodb_fatal_semaphore_wait_threshold=1;
......@@ -13219,7 +13219,7 @@ ha_innobase::check(
CHECK TABLE. */
os_increment_counter_by_amount(
server_mutex,
srv_fatal_semaphore_wait_threshold,
srv_fatal_semaphore_wait_threshold,
SRV_SEMAPHORE_WAIT_EXTENSION);
bool valid = btr_validate_index(index, prebuilt->trx);
......@@ -13227,7 +13227,7 @@ ha_innobase::check(
CHECK TABLE. */
os_decrement_counter_by_amount(
server_mutex,
srv_fatal_semaphore_wait_threshold,
srv_fatal_semaphore_wait_threshold,
SRV_SEMAPHORE_WAIT_EXTENSION);
if (!valid) {
......@@ -18836,6 +18836,15 @@ static MYSQL_SYSVAR_BOOL(use_mtflush, srv_use_mtflush,
"Use multi-threaded flush. Default FALSE.",
NULL, NULL, FALSE);
static MYSQL_SYSVAR_ULONG(fatal_semaphore_wait_threshold, srv_fatal_semaphore_wait_threshold,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"Maximum number of seconds that semaphore times out in InnoDB.",
NULL, NULL,
DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT, /* Default setting */
1, /* Minimum setting */
UINT_MAX32, /* Maximum setting */
0);
static struct st_mysql_sys_var* innobase_system_variables[]= {
MYSQL_SYSVAR(additional_mem_pool_size),
MYSQL_SYSVAR(api_trx_level),
......@@ -19010,6 +19019,8 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
MYSQL_SYSVAR(compression_algorithm),
MYSQL_SYSVAR(mtflush_threads),
MYSQL_SYSVAR(use_mtflush),
MYSQL_SYSVAR(fatal_semaphore_wait_threshold),
NULL
};
......
......@@ -492,7 +492,6 @@ extern my_bool srv_ibuf_disable_background_merge;
extern my_bool srv_purge_view_update_only_debug;
#endif /* UNIV_DEBUG */
extern ulint srv_fatal_semaphore_wait_threshold;
#define SRV_SEMAPHORE_WAIT_EXTENSION 7200
extern ulint srv_dml_needed_delay;
......@@ -531,6 +530,11 @@ extern srv_stats_t srv_stats;
/** Simulate compression failures. */
extern uint srv_simulate_comp_failures;
/** Fatal semaphore wait threshold = maximum number of seconds
that semaphore times out in InnoDB */
#define DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT 600
extern ulong srv_fatal_semaphore_wait_threshold;
# ifdef UNIV_PFS_THREAD
/* Keys to register InnoDB threads with performance schema */
extern mysql_pfs_key_t buf_page_cleaner_thread_key;
......
......@@ -4987,6 +4987,9 @@ lock_table(
lock_mutex_enter();
DBUG_EXECUTE_IF("fatal-semaphore-timeout",
{ os_thread_sleep(3600000000); });
/* We have to check if the new lock is compatible with any locks
other transactions have in the table lock queue. */
......
......@@ -80,7 +80,7 @@ extern int wsrep_debug;
extern int wsrep_trx_is_aborting(void *thd_ptr);
#endif
/* The following is the maximum allowed duration of a lock wait. */
UNIV_INTERN ulint srv_fatal_semaphore_wait_threshold = 600;
UNIV_INTERN ulong srv_fatal_semaphore_wait_threshold = DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT;
/* How much data manipulation language (DML) statements need to be delayed,
in microseconds, in order to reduce the lagging of the purge thread. */
......
......@@ -13897,7 +13897,7 @@ ha_innobase::check(
CHECK TABLE. */
os_increment_counter_by_amount(
server_mutex,
srv_fatal_semaphore_wait_threshold,
srv_fatal_semaphore_wait_threshold,
SRV_SEMAPHORE_WAIT_EXTENSION);
bool valid = btr_validate_index(index, prebuilt->trx);
......@@ -13905,7 +13905,7 @@ ha_innobase::check(
CHECK TABLE. */
os_decrement_counter_by_amount(
server_mutex,
srv_fatal_semaphore_wait_threshold,
srv_fatal_semaphore_wait_threshold,
SRV_SEMAPHORE_WAIT_EXTENSION);
if (!valid) {
......@@ -20128,6 +20128,15 @@ static MYSQL_SYSVAR_BOOL(use_mtflush, srv_use_mtflush,
"Use multi-threaded flush. Default FALSE.",
NULL, NULL, FALSE);
static MYSQL_SYSVAR_ULONG(fatal_semaphore_wait_threshold, srv_fatal_semaphore_wait_threshold,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"Maximum number of seconds that semaphore times out in InnoDB.",
NULL, NULL,
DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT, /* Default setting */
1, /* Minimum setting */
UINT_MAX32, /* Maximum setting */
0);
static struct st_mysql_sys_var* innobase_system_variables[]= {
MYSQL_SYSVAR(log_block_size),
MYSQL_SYSVAR(additional_mem_pool_size),
......@@ -20340,6 +20349,8 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
MYSQL_SYSVAR(compression_algorithm),
MYSQL_SYSVAR(mtflush_threads),
MYSQL_SYSVAR(use_mtflush),
MYSQL_SYSVAR(fatal_semaphore_wait_threshold),
NULL
};
......
......@@ -585,7 +585,6 @@ extern my_bool srv_ibuf_disable_background_merge;
extern my_bool srv_purge_view_update_only_debug;
#endif /* UNIV_DEBUG */
extern ulint srv_fatal_semaphore_wait_threshold;
#define SRV_SEMAPHORE_WAIT_EXTENSION 7200
extern ulint srv_dml_needed_delay;
extern long long srv_kill_idle_transaction;
......@@ -670,6 +669,11 @@ extern my_bool srv_fake_changes_locks;
/** Simulate compression failures. */
extern uint srv_simulate_comp_failures;
/** Fatal semaphore wait threshold = maximum number of seconds
that semaphore times out in InnoDB */
#define DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT 600
extern ulong srv_fatal_semaphore_wait_threshold;
# ifdef UNIV_PFS_THREAD
/* Keys to register InnoDB threads with performance schema */
extern mysql_pfs_key_t buf_page_cleaner_thread_key;
......
......@@ -5028,6 +5028,9 @@ lock_table(
lock_mutex_enter();
DBUG_EXECUTE_IF("fatal-semaphore-timeout",
{ os_thread_sleep(3600000000); });
/* We have to check if the new lock is compatible with any locks
other transactions have in the table lock queue. */
......
......@@ -95,7 +95,7 @@ in the server */
UNIV_INTERN ulint srv_activity_count = 0;
/* The following is the maximum allowed duration of a lock wait. */
UNIV_INTERN ulint srv_fatal_semaphore_wait_threshold = 600;
UNIV_INTERN ulong srv_fatal_semaphore_wait_threshold = DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT;
/**/
UNIV_INTERN long long srv_kill_idle_transaction = 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment