Bug #54360 Deadlock DROP/ALTER/CREATE DATABASE with open HANDLER

This deadlock happened if DROP DATABASE was blocked due to an open
HANDLER table from a different connection. While DROP DATABASE
is blocked, it holds the LOCK_mysql_create_db mutex. This results
in a deadlock if the connection with the open HANDLER table tries
to execute a CREATE/ALTER/DROP DATABASE statement as they all
try to acquire LOCK_mysql_create_db.

This patch makes this deadlock scenario very unlikely by closing and
marking for re-open all HANDLER tables for which there are pending
conflicing locks, before LOCK_mysql_create_db is acquired.
However, there is still a very slight possibility that a connection
could access one of these HANDLER tables between closing/marking for
re-open and the acquisition of LOCK_mysql_create_db.

This patch is for 5.1 only, a separate and complete fix will be
made for 5.5+.

Test case added to schema.test.

mysql-test/r/schema.result

@@ -11,3 +11,22 @@ mtr
 mysql
 test
 drop schema foo;
+#
+# Bug#54360 Deadlock DROP/ALTER/CREATE DATABASE with open HANDLER
+#
+CREATE DATABASE db1;
+CREATE TABLE db1.t1 (a INT);
+INSERT INTO db1.t1 VALUES (1), (2);
+# Connection con1
+HANDLER db1.t1 OPEN;
+# Connection default
+# Sending:
+DROP DATABASE db1;
+# Connection con2
+# Waiting for 'DROP DATABASE db1' to sync in.
+# Connection con1
+CREATE DATABASE db2;
+ALTER DATABASE db2 DEFAULT CHARACTER SET utf8;
+DROP DATABASE db2;
+# Connection default
+# Reaping: DROP DATABASE db1

mysql-test/t/schema.test

@@ -4,6 +4,9 @@
 # Drop mysqltest1 database, as it can left from the previous tests.
 #
 
+# Save the initial number of concurrent sessions.
+--source include/count_sessions.inc
+
 --disable_warnings
 drop database if exists mysqltest1;
 --enable_warnings
@@ -12,3 +15,47 @@ create schema foo;
 show create schema foo;
 show schemas;
 drop schema foo;
+
+
+--echo #
+--echo # Bug#54360 Deadlock DROP/ALTER/CREATE DATABASE with open HANDLER
+--echo #
+
+CREATE DATABASE db1;
+CREATE TABLE db1.t1 (a INT);
+INSERT INTO db1.t1 VALUES (1), (2);
+
+--echo # Connection con1
+connect (con1, localhost, root);
+HANDLER db1.t1 OPEN;
+
+--echo # Connection default
+connection default;
+--echo # Sending:
+--send DROP DATABASE db1
+
+--echo # Connection con2
+connect (con2, localhost, root);
+--echo # Waiting for 'DROP DATABASE db1' to sync in.
+let $wait_condition=SELECT COUNT(*)=1 FROM information_schema.processlist
+  WHERE state='Waiting for table' AND info='DROP DATABASE db1';
+--source include/wait_condition.inc
+
+--echo # Connection con1
+connection con1;
+# All these statements before resulted in deadlock.
+CREATE DATABASE db2;
+ALTER DATABASE db2 DEFAULT CHARACTER SET utf8;
+DROP DATABASE db2;
+
+--echo # Connection default
+connection default;
+--echo # Reaping: DROP DATABASE db1
+--reap
+disconnect con1;
+disconnect con2;
+
+
+# Check that all connections opened by test cases in this file are really
+# gone so execution of other tests won't be affected by their presence.
+--source include/wait_until_count_sessions.inc

sql/sql_db.cc

@@ -642,6 +642,18 @@ int mysql_create_db(THD *thd, char *db, HA_CREATE_INFO *create_info,
     goto exit2;
   }
 
+  /*
+    Close and mark for re-open all HANDLER tables which are marked for flush
+    or which there are pending conflicing locks against. This is needed to
+    prevent deadlocks.
+  */
+  if (thd->handler_tables_hash.records)
+  {
+    pthread_mutex_lock(&LOCK_open);
+    mysql_ha_flush(thd);
+    pthread_mutex_unlock(&LOCK_open);
+  }
+
   VOID(pthread_mutex_lock(&LOCK_mysql_create_db));
 
   /* Check directory */
@@ -788,6 +800,18 @@ bool mysql_alter_db(THD *thd, const char *db, HA_CREATE_INFO *create_info)
   if ((error=wait_if_global_read_lock(thd,0,1)))
     goto exit2;
 
+  /*
+    Close and mark for re-open all HANDLER tables which are marked for flush
+    or which there are pending conflicing locks against. This is needed to
+    prevent deadlocks.
+  */
+  if (thd->handler_tables_hash.records)
+  {
+    pthread_mutex_lock(&LOCK_open);
+    mysql_ha_flush(thd);
+    pthread_mutex_unlock(&LOCK_open);
+  }
+
   VOID(pthread_mutex_lock(&LOCK_mysql_create_db));
 
   /* 
@@ -886,6 +910,18 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent)
     goto exit2;
   }
 
+  /*
+    Close and mark for re-open all HANDLER tables which are marked for flush
+    or which there are pending conflicing locks against. This is needed to
+    prevent deadlocks.
+  */
+  if (thd->handler_tables_hash.records)
+  {
+    pthread_mutex_lock(&LOCK_open);
+    mysql_ha_flush(thd);
+    pthread_mutex_unlock(&LOCK_open);
+  }
+
   VOID(pthread_mutex_lock(&LOCK_mysql_create_db));
 
   length= build_table_filename(path, sizeof(path) - 1, db, "", "", 0);