Commit 8bc5eabe authored by Jan Lindström's avatar Jan Lindström

MDEV-7084: innodb index stats inadequate using constant

innodb_stats_sample_pages

Analysis: If you set the number of analyzed pages 
to very low number compared to actual pages on 
that table/index it randomly pics those pages 
(default 8 pages), this leads to fact that query 
after analyze table returns different results. If 
the index tree is small, smaller than 10 * 
n_sample_pages + total_external_size, then the 
estimate is ok. For bigger index trees it is 
common that we do not see any borders between 
key values in the few pages we pick. But still 
there may be n_sample_pages different key values, 
or even more. And it just tries to 
approximate to n_sample_pages (8).

Fix: (1) Introduced new dynamic configuration variable
innodb_stats_sample_traditional  that retains
the current design. Default false.

(2) If traditional sample is not used we use
n_sample_pages = max(min(srv_stats_sample_pages,
                         index->stat_index_size),
                     log2(index->stat_index_size)*
                          srv_stats_sample_pages);

(3) Introduced new dynamic configuration variable
stat_modified_counter (default = 0) if set
sets lower bound for row updates when statistics is re-estimated.

If user has provided upper bound for how many rows needs to be updated
before we calculate new statistics we use minimum of provided value
and 1/16 of table every 16th round. If no upper bound is provided
(srv_stats_modified_counter = 0, default) then calculate new statistics
if 1 / 16 of table has been modified
since the last time a statistics batch was run.
We calculate statistics at most every 16th round, since we may have
a counter table which is very small and updated very often.
@param t table
@return true if the table has changed too much and stats need to be
recalculated
*/
#define DICT_TABLE_CHANGED_TOO_MUCH(t) \
	((ib_int64_t) (t)->stat_modified_counter > (srv_stats_modified_counter ? \
	ut_min(srv_stats_modified_counter, (16 + (t)->stat_n_rows / 16)) : \
		16 + (t)->stat_n_rows / 16))
parent 6ea41f1e
SET @start_global_value = @@global.innodb_stats_modified_counter;
SELECT @start_global_value;
@start_global_value
0
Valid values are one or above
select @@global.innodb_stats_modified_counter >=1;
@@global.innodb_stats_modified_counter >=1
0
select @@global.innodb_stats_modified_counter;
@@global.innodb_stats_modified_counter
0
select @@session.innodb_stats_modified_counter;
ERROR HY000: Variable 'innodb_stats_modified_counter' is a GLOBAL variable
show global variables like 'innodb_stats_modified_counter';
Variable_name Value
innodb_stats_modified_counter 0
show session variables like 'innodb_stats_modified_counter';
Variable_name Value
innodb_stats_modified_counter 0
select * from information_schema.global_variables where variable_name='innodb_stats_modified_counter';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_MODIFIED_COUNTER 0
select * from information_schema.session_variables where variable_name='innodb_stats_modified_counter';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_MODIFIED_COUNTER 0
set global innodb_stats_modified_counter=10;
select @@global.innodb_stats_modified_counter;
@@global.innodb_stats_modified_counter
10
select * from information_schema.global_variables where variable_name='innodb_stats_modified_counter';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_MODIFIED_COUNTER 10
select * from information_schema.session_variables where variable_name='innodb_stats_modified_counter';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_MODIFIED_COUNTER 10
set session innodb_stats_modified_counter=1;
ERROR HY000: Variable 'innodb_stats_modified_counter' is a GLOBAL variable and should be set with SET GLOBAL
set global innodb_stats_modified_counter=1.1;
ERROR 42000: Incorrect argument type to variable 'innodb_stats_modified_counter'
set global innodb_stats_modified_counter=1e1;
ERROR 42000: Incorrect argument type to variable 'innodb_stats_modified_counter'
set global innodb_stats_modified_counter="foo";
ERROR 42000: Incorrect argument type to variable 'innodb_stats_modified_counter'
set global innodb_stats_modified_counter=-7;
Warnings:
Warning 1292 Truncated incorrect innodb_stats_modified_counter value: '-7'
select @@global.innodb_stats_modified_counter;
@@global.innodb_stats_modified_counter
0
select * from information_schema.global_variables where variable_name='innodb_stats_modified_counter';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_MODIFIED_COUNTER 0
SET @@global.innodb_stats_modified_counter = @start_global_value;
SELECT @@global.innodb_stats_modified_counter;
@@global.innodb_stats_modified_counter
0
SET @start_global_value = @@global.innodb_stats_traditional;
SELECT @start_global_value;
@start_global_value
0
Valid values are 'ON' and 'OFF'
select @@global.innodb_stats_traditional in (0, 1);
@@global.innodb_stats_traditional in (0, 1)
1
select @@global.innodb_stats_traditional;
@@global.innodb_stats_traditional
0
select @@session.innodb_stats_traditional;
ERROR HY000: Variable 'innodb_stats_traditional' is a GLOBAL variable
show global variables like 'innodb_stats_traditional';
Variable_name Value
innodb_stats_traditional OFF
show session variables like 'innodb_stats_traditional';
Variable_name Value
innodb_stats_traditional OFF
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL OFF
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL OFF
set global innodb_stats_traditional='OFF';
select @@global.innodb_stats_traditional;
@@global.innodb_stats_traditional
0
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL OFF
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL OFF
set @@global.innodb_stats_traditional=1;
select @@global.innodb_stats_traditional;
@@global.innodb_stats_traditional
1
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL ON
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL ON
set global innodb_stats_traditional=0;
select @@global.innodb_stats_traditional;
@@global.innodb_stats_traditional
0
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL OFF
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL OFF
set @@global.innodb_stats_traditional='ON';
select @@global.innodb_stats_traditional;
@@global.innodb_stats_traditional
1
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL ON
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL ON
set session innodb_stats_traditional='OFF';
ERROR HY000: Variable 'innodb_stats_traditional' is a GLOBAL variable and should be set with SET GLOBAL
set @@session.innodb_stats_traditional='ON';
ERROR HY000: Variable 'innodb_stats_traditional' is a GLOBAL variable and should be set with SET GLOBAL
set global innodb_stats_traditional=1.1;
ERROR 42000: Incorrect argument type to variable 'innodb_stats_traditional'
set global innodb_stats_traditional=1e1;
ERROR 42000: Incorrect argument type to variable 'innodb_stats_traditional'
set global innodb_stats_traditional=2;
ERROR 42000: Variable 'innodb_stats_traditional' can't be set to the value of '2'
set global innodb_stats_traditional=-3;
ERROR 42000: Variable 'innodb_stats_traditional' can't be set to the value of '-3'
select @@global.innodb_stats_traditional;
@@global.innodb_stats_traditional
1
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL ON
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
VARIABLE_NAME VARIABLE_VALUE
INNODB_STATS_TRADITIONAL ON
set global innodb_stats_traditional='AUTO';
ERROR 42000: Variable 'innodb_stats_traditional' can't be set to the value of 'AUTO'
SET @@global.innodb_stats_traditional = @start_global_value;
SELECT @@global.innodb_stats_traditional;
@@global.innodb_stats_traditional
0
--source include/have_innodb.inc
SET @start_global_value = @@global.innodb_stats_modified_counter;
SELECT @start_global_value;
#
# exists as global only
#
--echo Valid values are one or above
select @@global.innodb_stats_modified_counter >=1;
select @@global.innodb_stats_modified_counter;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
select @@session.innodb_stats_modified_counter;
show global variables like 'innodb_stats_modified_counter';
show session variables like 'innodb_stats_modified_counter';
select * from information_schema.global_variables where variable_name='innodb_stats_modified_counter';
select * from information_schema.session_variables where variable_name='innodb_stats_modified_counter';
#
# show that it's writable
#
set global innodb_stats_modified_counter=10;
select @@global.innodb_stats_modified_counter;
select * from information_schema.global_variables where variable_name='innodb_stats_modified_counter';
select * from information_schema.session_variables where variable_name='innodb_stats_modified_counter';
--error ER_GLOBAL_VARIABLE
set session innodb_stats_modified_counter=1;
#
# incorrect types
#
--error ER_WRONG_TYPE_FOR_VAR
set global innodb_stats_modified_counter=1.1;
--error ER_WRONG_TYPE_FOR_VAR
set global innodb_stats_modified_counter=1e1;
--error ER_WRONG_TYPE_FOR_VAR
set global innodb_stats_modified_counter="foo";
set global innodb_stats_modified_counter=-7;
select @@global.innodb_stats_modified_counter;
select * from information_schema.global_variables where variable_name='innodb_stats_modified_counter';
#
# cleanup
#
SET @@global.innodb_stats_modified_counter = @start_global_value;
SELECT @@global.innodb_stats_modified_counter;
--source include/have_innodb.inc
SET @start_global_value = @@global.innodb_stats_traditional;
SELECT @start_global_value;
#
# exists as global only
#
--echo Valid values are 'ON' and 'OFF'
select @@global.innodb_stats_traditional in (0, 1);
select @@global.innodb_stats_traditional;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
select @@session.innodb_stats_traditional;
show global variables like 'innodb_stats_traditional';
show session variables like 'innodb_stats_traditional';
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
#
# show that it's writable
#
set global innodb_stats_traditional='OFF';
select @@global.innodb_stats_traditional;
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
set @@global.innodb_stats_traditional=1;
select @@global.innodb_stats_traditional;
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
set global innodb_stats_traditional=0;
select @@global.innodb_stats_traditional;
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
set @@global.innodb_stats_traditional='ON';
select @@global.innodb_stats_traditional;
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
--error ER_GLOBAL_VARIABLE
set session innodb_stats_traditional='OFF';
--error ER_GLOBAL_VARIABLE
set @@session.innodb_stats_traditional='ON';
#
# incorrect types
#
--error ER_WRONG_TYPE_FOR_VAR
set global innodb_stats_traditional=1.1;
--error ER_WRONG_TYPE_FOR_VAR
set global innodb_stats_traditional=1e1;
--error ER_WRONG_VALUE_FOR_VAR
set global innodb_stats_traditional=2;
--error ER_WRONG_VALUE_FOR_VAR
set global innodb_stats_traditional=-3;
select @@global.innodb_stats_traditional;
select * from information_schema.global_variables where variable_name='innodb_stats_traditional';
select * from information_schema.session_variables where variable_name='innodb_stats_traditional';
--error ER_WRONG_VALUE_FOR_VAR
set global innodb_stats_traditional='AUTO';
#
# Cleanup
#
SET @@global.innodb_stats_traditional = @start_global_value;
SELECT @@global.innodb_stats_traditional;
......@@ -3764,16 +3764,29 @@ btr_estimate_number_of_different_key_vals(
ut_error;
}
/* It makes no sense to test more pages than are contained
in the index, thus we lower the number if it is too high */
if (srv_stats_sample_pages > index->stat_index_size) {
if (srv_stats_sample_traditional) {
/* It makes no sense to test more pages than are contained
in the index, thus we lower the number if it is too high */
if (srv_stats_sample_pages > index->stat_index_size) {
if (index->stat_index_size > 0) {
n_sample_pages = index->stat_index_size;
} else {
n_sample_pages = 1;
}
} else {
n_sample_pages = srv_stats_sample_pages;
}
} else {
/* New logaritmic number of pages that are estimated. We
first pick minimun from srv_stats_sample_pages and number of
pages on index. Then we pick maximum from previous number of
pages and log2(number of index pages) * srv_stats_sample_pages. */
if (index->stat_index_size > 0) {
n_sample_pages = index->stat_index_size;
n_sample_pages = ut_max(ut_min(srv_stats_sample_pages, index->stat_index_size),
log2(index->stat_index_size)*srv_stats_sample_pages);
} else {
n_sample_pages = 1;
}
} else {
n_sample_pages = srv_stats_sample_pages;
}
/* We sample some pages in the index to get an estimate */
......
......@@ -11739,6 +11739,16 @@ static MYSQL_SYSVAR_ULONGLONG(stats_sample_pages, srv_stats_sample_pages,
"The number of index pages to sample when calculating statistics (default 8)",
NULL, NULL, 8, 1, ~0ULL, 0);
static MYSQL_SYSVAR_ULONGLONG(stats_modified_counter, srv_stats_modified_counter,
PLUGIN_VAR_RQCMDARG,
"The number of rows modified before we calculate new statistics (default 0 = current limits)",
NULL, NULL, 0, 0, ~0ULL, 0);
static MYSQL_SYSVAR_BOOL(stats_traditional, srv_stats_sample_traditional,
PLUGIN_VAR_RQCMDARG,
"Enable traditional statistic calculation based on number of configured pages (default false)",
NULL, NULL, FALSE);
static MYSQL_SYSVAR_BOOL(adaptive_hash_index, btr_search_enabled,
PLUGIN_VAR_OPCMDARG,
"Enable InnoDB adaptive hash index (enabled by default). "
......@@ -12001,6 +12011,8 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
MYSQL_SYSVAR(rollback_on_timeout),
MYSQL_SYSVAR(stats_on_metadata),
MYSQL_SYSVAR(stats_sample_pages),
MYSQL_SYSVAR(stats_modified_counter),
MYSQL_SYSVAR(stats_traditional),
MYSQL_SYSVAR(adaptive_hash_index),
MYSQL_SYSVAR(stats_method),
MYSQL_SYSVAR(replication_delay),
......
......@@ -1123,7 +1123,13 @@ dict_index_calc_min_rec_len(
/*========================*/
const dict_index_t* index); /*!< in: index */
/** Calculate new statistics if 1 / 16 of table has been modified
/**
If user has provided upper bound for how many rows needs to be updated
before we calculate new statistics we use minimum of provided value
and 1/16 of table every 16th round. If no upper bound is provided
(srv_stats_modified_counter = 0, default) then calculate new statistics
if 1 / 16 of table has been modified
since the last time a statistics batch was run.
We calculate statistics at most every 16th round, since we may have
a counter table which is very small and updated very often.
......@@ -1132,7 +1138,9 @@ a counter table which is very small and updated very often.
recalculated
*/
#define DICT_TABLE_CHANGED_TOO_MUCH(t) \
((ib_int64_t) (t)->stat_modified_counter > 16 + (t)->stat_n_rows / 16)
((ib_int64_t) (t)->stat_modified_counter > (srv_stats_modified_counter ? \
ut_min(srv_stats_modified_counter, (16 + (t)->stat_n_rows / 16)) : \
16 + (t)->stat_n_rows / 16))
/*********************************************************************//**
Calculates new estimates for table and index statistics. The statistics
......
......@@ -211,6 +211,8 @@ extern ulint srv_fast_shutdown; /* If this is 1, do not do a
extern ibool srv_innodb_status;
extern unsigned long long srv_stats_sample_pages;
extern unsigned long long srv_stats_modified_counter;
extern my_bool srv_stats_sample_traditional;
extern ibool srv_use_doublewrite_buf;
extern ibool srv_use_atomic_writes;
......
......@@ -749,6 +749,13 @@ static ulint srv_meter_high_water[SRV_MASTER + 1];
static ulint srv_meter_high_water2[SRV_MASTER + 1];
static ulint srv_meter_foreground[SRV_MASTER + 1];
#endif
/* The number of rows modified before we calculate new statistics (default 0
= current limits) */
UNIV_INTERN unsigned long long srv_stats_modified_counter = 0;
/* Enable traditional statistic calculation based on number of configured
pages default false. */
UNIV_INTERN my_bool srv_stats_sample_traditional = FALSE;
/* The following values give info about the activity going on in
the database. They are protected by the server mutex. The arrays
......
......@@ -3944,16 +3944,29 @@ btr_estimate_number_of_different_key_vals(
ut_error;
}
/* It makes no sense to test more pages than are contained
in the index, thus we lower the number if it is too high */
if (srv_stats_sample_pages > index->stat_index_size) {
if (srv_stats_sample_traditional) {
/* It makes no sense to test more pages than are contained
in the index, thus we lower the number if it is too high */
if (srv_stats_sample_pages > index->stat_index_size) {
if (index->stat_index_size > 0) {
n_sample_pages = index->stat_index_size;
} else {
n_sample_pages = 1;
}
} else {
n_sample_pages = srv_stats_sample_pages;
}
} else {
/* New logaritmic number of pages that are estimated. We
first pick minimun from srv_stats_sample_pages and number of
pages on index. Then we pick maximum from previous number of
pages and log2(number of index pages) * srv_stats_sample_pages. */
if (index->stat_index_size > 0) {
n_sample_pages = index->stat_index_size;
n_sample_pages = ut_max(ut_min(srv_stats_sample_pages, index->stat_index_size),
log2(index->stat_index_size)*srv_stats_sample_pages);
} else {
n_sample_pages = 1;
}
} else {
n_sample_pages = srv_stats_sample_pages;
}
/* We sample some pages in the index to get an estimate */
......
......@@ -13107,6 +13107,16 @@ static MYSQL_SYSVAR_ULONGLONG(stats_sample_pages, srv_stats_sample_pages,
"The number of index pages to sample when calculating statistics (default 8)",
NULL, NULL, 8, 1, ~0ULL, 0);
static MYSQL_SYSVAR_ULONGLONG(stats_modified_counter, srv_stats_modified_counter,
PLUGIN_VAR_RQCMDARG,
"The number of rows modified before we calculate new statistics (default 0 = current limits)",
NULL, NULL, 0, 0, ~0ULL, 0);
static MYSQL_SYSVAR_BOOL(stats_traditional, srv_stats_sample_traditional,
PLUGIN_VAR_RQCMDARG,
"Enable traditional statistic calculation based on number of configured pages (default false)",
NULL, NULL, FALSE);
static MYSQL_SYSVAR_ULINT(stats_auto_update, srv_stats_auto_update,
PLUGIN_VAR_RQCMDARG,
"Enable/Disable InnoDB's auto update statistics of indexes. "
......@@ -13680,6 +13690,8 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
MYSQL_SYSVAR(persistent_stats_root_page),
#endif
MYSQL_SYSVAR(stats_sample_pages),
MYSQL_SYSVAR(stats_modified_counter),
MYSQL_SYSVAR(stats_traditional),
MYSQL_SYSVAR(adaptive_hash_index),
MYSQL_SYSVAR(adaptive_hash_index_partitions),
MYSQL_SYSVAR(stats_method),
......
......@@ -1123,7 +1123,13 @@ dict_index_calc_min_rec_len(
/*========================*/
const dict_index_t* index); /*!< in: index */
/** Calculate new statistics if 1 / 16 of table has been modified
/**
If user has provided upper bound for how many rows needs to be updated
before we calculate new statistics we use minimum of provided value
and 1/16 of table every 16th round. If no upper bound is provided
(srv_stats_modified_counter = 0, default) then calculate new statistics
if 1 / 16 of table has been modified
since the last time a statistics batch was run.
We calculate statistics at most every 16th round, since we may have
a counter table which is very small and updated very often.
......@@ -1132,7 +1138,9 @@ a counter table which is very small and updated very often.
recalculated
*/
#define DICT_TABLE_CHANGED_TOO_MUCH(t) \
((ib_int64_t) (t)->stat_modified_counter > 16 + (t)->stat_n_rows / 16)
((ib_int64_t) (t)->stat_modified_counter > (srv_stats_modified_counter ? \
ut_min(srv_stats_modified_counter, (16 + (t)->stat_n_rows / 16)) : \
16 + (t)->stat_n_rows / 16))
/*********************************************************************//**
Calculates new estimates for table and index statistics. The statistics
......
......@@ -242,6 +242,8 @@ extern ulint srv_fast_shutdown; /* If this is 1, do not do a
extern ibool srv_innodb_status;
extern unsigned long long srv_stats_sample_pages;
extern unsigned long long srv_stats_modified_counter;
extern my_bool srv_stats_sample_traditional;
extern ulint srv_stats_auto_update;
extern ulint srv_stats_update_need_lock;
extern ibool srv_use_sys_stats_table;
......
......@@ -411,6 +411,13 @@ UNIV_INTERN ibool srv_use_sys_stats_table = FALSE;
#ifdef UNIV_DEBUG
UNIV_INTERN ulong srv_sys_stats_root_page = 0;
#endif
/* The number of rows modified before we calculate new statistics (default 0
= current limits) */
UNIV_INTERN unsigned long long srv_stats_modified_counter = 0;
/* Enable traditional statistic calculation based on number of configured
pages default false. */
UNIV_INTERN my_bool srv_stats_sample_traditional = FALSE;
UNIV_INTERN ibool srv_use_doublewrite_buf = TRUE;
UNIV_INTERN ibool srv_use_atomic_writes = FALSE;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment