Skip to content

M
mariadb
Commit a6b70da9 authored by Mattias Jonsson's avatar Mattias Jonsson
Browse files
Options

Bug#11766249 bug#59316: PARTITIONING AND INDEX_MERGE MEMORY LEAK 

When executing row-ordered-retrieval index merge,
the handler was cloned, but it used the wrong
memory root, so instead of allocating memory
on the thread/query's mem_root, it used the table's
mem_root, resulting in non released memory in the
table object, and was not freed until the table was
closed.

Solution was to ensure that memory used during cloning
of a handler was allocated from the correct memory root.

This was implemented by fixing handler::clone() to also
take a name argument, so it can be used with partitioning.
And in ha_partition only allocate the ha_partition's ref, and
call the original ha_partition partitions clone() and set at cloned
partitions.

Fix of .bzrignore on Windows with VS 2010
parent 2ab3b055
Show whitespace changes
Inline Side-by-side
Showing with 214 additions and 82 deletions
.bzrignore
View file @ a6b70da9
......@@ -37,7 +37,13 @@
*.user
*.vcproj
*.vcproj.cmake
*.vcxproj
*.vcxproj.filters
*/*.dir/*
*.dir
Debug
MySql.sdf
Win32
*/*_pure_*warnings
*/.deps
*/.libs/*
......@@ -46,6 +52,7 @@
*/minsizerel/*
*/release/*
*/relwithdebinfo/*
RelWithDebInfo
*~
.*.swp
./CMakeCache.txt
......@@ -607,6 +614,7 @@ include/mysql_h.ic
include/mysql_version.h
include/mysqld_ername.h
include/mysqld_error.h
include/mysqld_error.h.rule
include/openssl
include/readline
include/readline/*.h
......@@ -1879,7 +1887,9 @@ scripts/mysql_find_rows
scripts/mysql_fix_extensions
scripts/mysql_fix_privilege_tables
scripts/mysql_fix_privilege_tables.sql
scripts/mysql_fix_privilege_tables.sql.rule
scripts/mysql_fix_privilege_tables_sql.c
scripts/mysql_fix_privilege_tables_sql.c.rule
scripts/mysql_install_db
scripts/mysql_secure_installation
scripts/mysql_setpermission
......@@ -2116,6 +2126,7 @@ sql/handlerton.cc
sql/html
sql/latex
sql/lex_hash.h
sql/lex_hash.h.rule
sql/link_sources
sql/max/*
sql/message.h
......@@ -2147,6 +2158,7 @@ sql/sql_builtin.cc
sql/sql_select.cc.orig
sql/sql_yacc.cc
sql/sql_yacc.h
sql/sql_yacc.h.rule
sql/sql_yacc.output
sql/sql_yacc.yy.orig
sql/test_time
......
sql/ha_partition.cc
View file @ a6b70da9
......@@ -163,10 +163,14 @@ const uint ha_partition::NO_CURRENT_PART_ID= 0xFFFFFFFF;
*/
ha_partition::ha_partition(handlerton *hton, TABLE_SHARE *share)
:handler(hton, share), m_part_info(NULL), m_create_handler(FALSE),
m_is_sub_partitioned(0)
:handler(hton, share)
{
DBUG_ENTER("ha_partition::ha_partition(table)");
m_part_info= NULL;
m_create_handler= FALSE;
m_is_sub_partitioned= 0;
m_is_clone_of= NULL;
m_clone_mem_root= NULL;
init_handler_variables();
DBUG_VOID_RETURN;
}
......@@ -184,15 +188,46 @@ ha_partition::ha_partition(handlerton *hton, TABLE_SHARE *share)
*/
ha_partition::ha_partition(handlerton *hton, partition_info *part_info)
:handler(hton, NULL), m_part_info(part_info), m_create_handler(TRUE),
m_is_sub_partitioned(m_part_info->is_sub_partitioned())
:handler(hton, NULL)
{
DBUG_ENTER("ha_partition::ha_partition(part_info)");
DBUG_ASSERT(part_info);
m_part_info= part_info;
m_create_handler= TRUE;
m_is_sub_partitioned= m_part_info->is_sub_partitioned();
init_handler_variables();
DBUG_ASSERT(m_part_info);
DBUG_VOID_RETURN;
}
/**
ha_partition constructor method used by ha_partition::clone()
@param hton Handlerton (partition_hton)
@param share Table share object
@param part_info_arg partition_info to use
@param clone_arg ha_partition to clone
@param clme_mem_root_arg MEM_ROOT to use
@return New partition handler
*/
ha_partition::ha_partition(handlerton *hton, TABLE_SHARE *share,
partition_info *part_info_arg,
ha_partition *clone_arg,
MEM_ROOT *clone_mem_root_arg)
:handler(hton, share)
{
DBUG_ENTER("ha_partition::ha_partition(clone)");
m_part_info= part_info_arg;
m_create_handler= TRUE;
m_is_sub_partitioned= m_part_info->is_sub_partitioned();
m_is_clone_of= clone_arg;
m_clone_mem_root= clone_mem_root_arg;
init_handler_variables();
m_tot_parts= clone_arg->m_tot_parts;
DBUG_ASSERT(m_tot_parts);
DBUG_VOID_RETURN;
}
/*
Initialize handler object
......@@ -244,7 +279,6 @@ void ha_partition::init_handler_variables()
m_rec0= 0;
m_curr_key_info[0]= NULL;
m_curr_key_info[1]= NULL;
is_clone= FALSE,
m_part_func_monotonicity_info= NON_MONOTONIC;
auto_increment_lock= FALSE;
auto_increment_safe_stmt_log_lock= FALSE;
......@@ -359,7 +393,8 @@ bool ha_partition::initialize_partition(MEM_ROOT *mem_root)
*/
DBUG_RETURN(0);
}
else if (get_from_handler_file(table_share->normalized_path.str, mem_root))
else if (get_from_handler_file(table_share->normalized_path.str,
mem_root, false))
{
my_message(ER_UNKNOWN_ERROR, "Failed to read from the .par file", MYF(0));
DBUG_RETURN(1);
......@@ -1848,7 +1883,7 @@ uint ha_partition::del_ren_cre_table(const char *from,
DBUG_RETURN(TRUE);
}
if (get_from_handler_file(from, ha_thd()->mem_root))
if (get_from_handler_file(from, ha_thd()->mem_root, false))
DBUG_RETURN(TRUE);
DBUG_ASSERT(m_file_buffer);
DBUG_PRINT("enter", ("from: (%s) to: (%s)", from, to));
......@@ -2368,7 +2403,8 @@ error_end:
partitions.
*/
bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root)
bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root,
bool clone)
{
char buff[FN_REFLEN], *address_tot_name_len;
File file;
......@@ -2403,6 +2439,8 @@ bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root)
m_tot_parts= uint4korr((file_buffer) + 8);
DBUG_PRINT("info", ("No of parts = %u", m_tot_parts));
tot_partition_words= (m_tot_parts + 3) / 4;
if (!clone)
{
engine_array= (handlerton **) my_alloca(m_tot_parts * sizeof(handlerton*));
for (i= 0; i < m_tot_parts; i++)
{
......@@ -2413,6 +2451,7 @@ bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root)
if (!engine_array[i])
goto err3;
}
}
address_tot_name_len= file_buffer + 12 + 4 * tot_partition_words;
tot_name_words= (uint4korr(address_tot_name_len) + 3) / 4;
if (len_words != (tot_partition_words + tot_name_words + 4))
......@@ -2422,6 +2461,8 @@ bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root)
m_file_buffer= file_buffer; // Will be freed in clear_handler_file()
m_name_buffer_ptr= name_buffer_ptr;
if (!clone)
{
if (!(m_engine_array= (plugin_ref*)
my_malloc(m_tot_parts * sizeof(plugin_ref), MYF(MY_WME))))
goto err3;
......@@ -2430,8 +2471,9 @@ bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root)
m_engine_array[i]= ha_lock_engine(NULL, engine_array[i]);
my_afree((gptr) engine_array);
}
if (!m_file && create_handlers(mem_root))
if (!clone && !m_file && create_handlers(mem_root))
{
clear_handler_file();
DBUG_RETURN(TRUE);
......@@ -2439,6 +2481,7 @@ bool ha_partition::get_from_handler_file(const char *name, MEM_ROOT *mem_root)
DBUG_RETURN(FALSE);
err3:
if (!clone)
my_afree((gptr) engine_array);
err2:
my_free(file_buffer, MYF(0));
......@@ -2491,13 +2534,13 @@ void ha_data_partition_destroy(void *ha_data)
int ha_partition::open(const char *name, int mode, uint test_if_locked)
{
char *name_buffer_ptr= m_name_buffer_ptr;
char *name_buffer_ptr;
int error;
uint alloc_len;
handler **file;
char name_buff[FN_REFLEN];
bool is_not_tmp_table= (table_share->tmp_table == NO_TMP_TABLE);
ulonglong check_table_flags= 0;
ulonglong check_table_flags;
DBUG_ENTER("ha_partition::open");
DBUG_ASSERT(table->s == table_share);
......@@ -2505,8 +2548,9 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked)
m_mode= mode;
m_open_test_lock= test_if_locked;
m_part_field_array= m_part_info->full_part_field_array;
if (get_from_handler_file(name, &table->mem_root))
if (get_from_handler_file(name, &table->mem_root, test(m_is_clone_of)))
DBUG_RETURN(1);
name_buffer_ptr= m_name_buffer_ptr;
m_start_key.length= 0;
m_rec0= table->record[0];
m_rec_length= table_share->reclength;
......@@ -2542,8 +2586,9 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked)
DBUG_RETURN(1);
bitmap_clear_all(&m_bulk_insert_started);
/* Initialize the bitmap we use to determine what partitions are used */
if (!is_clone)
if (!m_is_clone_of)
{
DBUG_ASSERT(!m_clone_mem_root);
if (bitmap_init(&(m_part_info->used_partitions), NULL, m_tot_parts, TRUE))
{
bitmap_free(&m_bulk_insert_started);
......@@ -2552,6 +2597,36 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked)
bitmap_set_all(&(m_part_info->used_partitions));
}
if (m_is_clone_of)
{
uint i;
DBUG_ASSERT(m_clone_mem_root);
/* Allocate an array of handler pointers for the partitions handlers. */
alloc_len= (m_tot_parts + 1) * sizeof(handler*);
if (!(m_file= (handler **) alloc_root(m_clone_mem_root, alloc_len)))
goto err_alloc;
memset(m_file, 0, alloc_len);
/*
Populate them by cloning the original partitions. This also opens them.
Note that file->ref is allocated too.
*/
file= m_is_clone_of->m_file;
for (i= 0; i < m_tot_parts; i++)
{
create_partition_name(name_buff, name, name_buffer_ptr, NORMAL_PART_NAME,
FALSE);
if (!(m_file[i]= file[i]->clone((const char*) name_buff,
m_clone_mem_root)))
{
error= HA_ERR_INITIALIZATION;
file= &m_file[i];
goto err_handler;
}
name_buffer_ptr+= strlen(name_buffer_ptr) + 1;
}
}
else
{
file= m_file;
do
{
......@@ -2562,22 +2637,30 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked)
goto err_handler;
m_no_locks+= (*file)->lock_count();
name_buffer_ptr+= strlen(name_buffer_ptr) + 1;
} while (*(++file));
}
file= m_file;
ref_length= (*file)->ref_length;
check_table_flags= (((*file)->ha_table_flags() &
~(PARTITION_DISABLED_TABLE_FLAGS)) |
(PARTITION_ENABLED_TABLE_FLAGS));
file++;
do
{
DBUG_ASSERT(ref_length >= (*file)->ref_length);
set_if_bigger(ref_length, ((*file)->ref_length));
/*
Verify that all partitions have the same set of table flags.
Mask all flags that partitioning enables/disables.
*/
if (!check_table_flags)
{
check_table_flags= (((*file)->ha_table_flags() &
~(PARTITION_DISABLED_TABLE_FLAGS)) |
(PARTITION_ENABLED_TABLE_FLAGS));
}
else if (check_table_flags != (((*file)->ha_table_flags() &
if (check_table_flags != (((*file)->ha_table_flags() &
~(PARTITION_DISABLED_TABLE_FLAGS)) |
(PARTITION_ENABLED_TABLE_FLAGS)))
{
error= HA_ERR_INITIALIZATION;
/* set file to last handler, so all of them is closed */
file = &m_file[m_tot_parts - 1];
goto err_handler;
}
} while (*(++file));
......@@ -2589,6 +2672,7 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked)
*/
ref_length+= PARTITION_BYTES_IN_POS;
m_ref_length= ref_length;
/*
Release buffer read from .par file. It will not be reused again after
being opened once.
......@@ -2646,25 +2730,55 @@ err_handler:
DEBUG_SYNC(ha_thd(), "partition_open_error");
while (file-- != m_file)
(*file)->close();
err_alloc:
bitmap_free(&m_bulk_insert_started);
if (!is_clone)
if (!m_is_clone_of)
bitmap_free(&(m_part_info->used_partitions));
DBUG_RETURN(error);
}
handler *ha_partition::clone(MEM_ROOT *mem_root)
/**
Clone the open and locked partitioning handler.
@param mem_root MEM_ROOT to use.
@return Pointer to the successfully created clone or NULL
@details
This function creates a new ha_partition handler as a clone/copy. The
original (this) must already be opened and locked. The clone will use
the originals m_part_info.
It also allocates memory to ref + ref_dup.
In ha_partition::open() it will clone its original handlers partitions
which will allocate then om the correct MEM_ROOT and also open them.
*/
handler *ha_partition::clone(const char *name, MEM_ROOT *mem_root)
{
handler *new_handler= get_new_handler(table->s, mem_root,
table->s->db_type());
((ha_partition*)new_handler)->m_part_info= m_part_info;
((ha_partition*)new_handler)->is_clone= TRUE;
if (new_handler && !new_handler->ha_open(table,
table->s->normalized_path.str,
table->db_stat,
HA_OPEN_IGNORE_IF_LOCKED))
return new_handler;
return NULL;
ha_partition *new_handler;
DBUG_ENTER("ha_partition::clone");
new_handler= new (mem_root) ha_partition(ht, table_share, m_part_info,
this, mem_root);
if (!new_handler)
DBUG_RETURN(NULL);
/*
Allocate new_handler->ref here because otherwise ha_open will allocate it
on this->table->mem_root and we will not be able to reclaim that memory
when the clone handler object is destroyed.
*/
new_handler->ref= (uchar*) alloc_root(mem_root, ALIGN_SIZE(m_ref_length)*2);
if (!new_handler->ref)
DBUG_RETURN(NULL);
if (new_handler->ha_open(table, name,
table->db_stat, HA_OPEN_IGNORE_IF_LOCKED))
DBUG_RETURN(NULL);
DBUG_RETURN((handler*) new_handler);
}
......@@ -2695,7 +2809,7 @@ int ha_partition::close(void)
DBUG_ASSERT(table->s == table_share);
delete_queue(&m_queue);
bitmap_free(&m_bulk_insert_started);
if (!is_clone)
if (!m_is_clone_of)
bitmap_free(&(m_part_info->used_partitions));
file= m_file;
......
sql/ha_partition.h
View file @ a6b70da9
......@@ -133,6 +133,13 @@ private:
bool m_is_sub_partitioned; // Is subpartitioned
bool m_ordered_scan_ongoing;
/*
If set, this object was created with ha_partition::clone and doesn't
"own" the m_part_info structure.
*/
ha_partition *m_is_clone_of;
MEM_ROOT *m_clone_mem_root;
/*
We keep track if all underlying handlers are MyISAM since MyISAM has a
great number of extra flags not needed by other handlers.
......@@ -169,11 +176,6 @@ private:
PARTITION_SHARE *share; /* Shared lock info */
#endif
/*
TRUE <=> this object was created with ha_partition::clone and doesn't
"own" the m_part_info structure.
*/
bool is_clone;
bool auto_increment_lock; /**< lock reading/updating auto_inc */
/**
Flag to keep the auto_increment lock through out the statement.
......@@ -186,7 +188,7 @@ private:
/** used for prediction of start_bulk_insert rows */
enum_monotonicity_info m_part_func_monotonicity_info;
public:
handler *clone(MEM_ROOT *mem_root);
handler *clone(const char *name, MEM_ROOT *mem_root);
virtual void set_part_info(partition_info *part_info)
{
m_part_info= part_info;
......@@ -205,6 +207,10 @@ public:
*/
ha_partition(handlerton *hton, TABLE_SHARE * table);
ha_partition(handlerton *hton, partition_info * part_info);
ha_partition(handlerton *hton, TABLE_SHARE *share,
partition_info *part_info_arg,
ha_partition *clone_arg,
MEM_ROOT *clone_mem_root_arg);
~ha_partition();
/*
A partition handler has no characteristics in itself. It only inherits
......@@ -275,7 +281,7 @@ private:
And one method to read it in.
*/
bool create_handler_file(const char *name);
bool get_from_handler_file(const char *name, MEM_ROOT *mem_root);
bool get_from_handler_file(const char *name, MEM_ROOT *mem_root, bool clone);
bool new_handlers_from_part_info(MEM_ROOT *mem_root);
bool create_handlers(MEM_ROOT *mem_root);
void clear_handler_file();
......
sql/handler.cc
View file @ a6b70da9
......@@ -2037,9 +2037,9 @@ int ha_delete_table(THD *thd, handlerton *table_type, const char *path,
/****************************************************************************
** General handler functions
****************************************************************************/
handler *handler::clone(MEM_ROOT *mem_root)
handler *handler::clone(const char *name, MEM_ROOT *mem_root)
{
handler *new_handler= get_new_handler(table->s, mem_root, table->s->db_type());
handler *new_handler= get_new_handler(table->s, mem_root, ht);
/*
Allocate handler->ref here because otherwise ha_open will allocate it
on this->table->mem_root and we will not be able to reclaim that memory
......@@ -2048,7 +2048,7 @@ handler *handler::clone(MEM_ROOT *mem_root)
if (!(new_handler->ref= (uchar*) alloc_root(mem_root, ALIGN_SIZE(ref_length)*2)))
return NULL;
if (new_handler && !new_handler->ha_open(table,
table->s->normalized_path.str,
name,
table->db_stat,
HA_OPEN_IGNORE_IF_LOCKED))
return new_handler;
......
sql/handler.h
View file @ a6b70da9
......@@ -1166,7 +1166,7 @@ public:
DBUG_ASSERT(locked == FALSE);
/* TODO: DBUG_ASSERT(inited == NONE); */
}
virtual handler *clone(MEM_ROOT *mem_root);
virtual handler *clone(const char *name, MEM_ROOT *mem_root);
/** This is called after create to allow us to set up cached variables */
void init()
{
......
sql/opt_range.cc
View file @ a6b70da9
......@@ -1335,7 +1335,7 @@ int QUICK_RANGE_SELECT::init_ror_merged_scan(bool reuse_handler)
}
thd= head->in_use;
if (!(file= head->file->clone(thd->mem_root)))
if (!(file= head->file->clone(head->s->normalized_path.str, thd->mem_root)))
{
/*
Manually set the error flag. Note: there seems to be quite a few
......
storage/heap/ha_heap.cc
View file @ a6b70da9
......@@ -142,11 +142,11 @@ int ha_heap::close(void)
DESCRIPTION
Do same as default implementation but use file->s->name instead of
table->s->path. This is needed by Windows where the clone() call sees
'/'-delimited path in table->s->path, while ha_peap::open() was called
'/'-delimited path in table->s->path, while ha_heap::open() was called
with '\'-delimited path.
*/
handler *ha_heap::clone(MEM_ROOT *mem_root)
handler *ha_heap::clone(const char *name, MEM_ROOT *mem_root)
{
handler *new_handler= get_new_handler(table->s, mem_root, table->s->db_type());
if (new_handler && !new_handler->ha_open(table, file->s->name, table->db_stat,
......
storage/heap/ha_heap.h
View file @ a6b70da9
......@@ -34,7 +34,7 @@ class ha_heap: public handler
public:
ha_heap(handlerton *hton, TABLE_SHARE *table);
~ha_heap() {}
handler *clone(MEM_ROOT *mem_root);
handler *clone(const char *name, MEM_ROOT *mem_root);
const char *table_type() const
{
return (table->in_use->variables.sql_mode & MODE_MYSQL323) ?
......
storage/myisam/ha_myisam.cc
View file @ a6b70da9
......@@ -552,9 +552,10 @@ ha_myisam::ha_myisam(handlerton *hton, TABLE_SHARE *table_arg)
can_enable_indexes(1)
{}
handler *ha_myisam::clone(MEM_ROOT *mem_root)
handler *ha_myisam::clone(const char *name, MEM_ROOT *mem_root)
{
ha_myisam *new_handler= static_cast <ha_myisam *>(handler::clone(mem_root));
ha_myisam *new_handler= static_cast <ha_myisam *>(handler::clone(name,
mem_root));
if (new_handler)
new_handler->file->state= file->state;
return new_handler;
......
storage/myisam/ha_myisam.h
View file @ a6b70da9
......@@ -44,7 +44,7 @@ class ha_myisam: public handler
public:
ha_myisam(handlerton *hton, TABLE_SHARE *table_arg);
~ha_myisam() {}
handler *clone(MEM_ROOT *mem_root);
handler *clone(const char *name, MEM_ROOT *mem_root);
const char *table_type() const { return "MyISAM"; }
const char *index_type(uint key_number);
const char **bas_ext() const;
......
storage/myisammrg/ha_myisammrg.cc
View file @ a6b70da9
......@@ -459,8 +459,7 @@ int ha_myisammrg::open(const char *name, int mode __attribute__((unused)),
problem because all locking is handled by the original MERGE table
from which this is cloned of.
*/
if (!(file= myrg_open(table->s->normalized_path.str, table->db_stat,
HA_OPEN_IGNORE_IF_LOCKED)))
if (!(file= myrg_open(name, table->db_stat, HA_OPEN_IGNORE_IF_LOCKED)))
{
DBUG_PRINT("error", ("my_errno %d", my_errno));
DBUG_RETURN(my_errno ? my_errno : -1);
......@@ -484,7 +483,7 @@ int ha_myisammrg::open(const char *name, int mode __attribute__((unused)),
@return A cloned handler instance.
*/
handler *ha_myisammrg::clone(MEM_ROOT *mem_root)
handler *ha_myisammrg::clone(const char *name, MEM_ROOT *mem_root)
{
MYRG_TABLE *u_table,*newu_table;
ha_myisammrg *new_handler=
......@@ -505,7 +504,7 @@ handler *ha_myisammrg::clone(MEM_ROOT *mem_root)
return NULL;
}
if (new_handler->ha_open(table, table->s->normalized_path.str, table->db_stat,
if (new_handler->ha_open(table, name, table->db_stat,
HA_OPEN_IGNORE_IF_LOCKED))
{
delete new_handler;
......
storage/myisammrg/ha_myisammrg.h
View file @ a6b70da9
......@@ -62,7 +62,7 @@ class ha_myisammrg: public handler
int open(const char *name, int mode, uint test_if_locked);
int attach_children(void);
int detach_children(void);
virtual handler *clone(MEM_ROOT *mem_root);
virtual handler *clone(const char *name, MEM_ROOT *mem_root);
int close(void);
int write_row(uchar * buf);
int update_row(const uchar * old_data, uchar * new_data);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7