MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key

Analysis: When a page is read from encrypted table and page can't be
decrypted because of bad key (or incorrect encryption algorithm or
method) page was incorrectly left on buffer pool.

Fix: Remove page from buffer pool and from pending IO.

mysql-test/suite/encryption/r/innodb-bad-key-change.result

@@ -50,7 +50,6 @@ ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be
 SHOW WARNINGS;
 Level	Code	Message
 Warning	192	 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
-Warning	192	 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
 Warning	192	Table test/t2 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
 Error	1296	Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
 SELECT * FROM t2 where id = 1;

mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result

+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+call mtr.add_suppression("Couldn't load plugins from 'file_key_management*");
+# 
+# Restart the server with key 4 in the key file
+# 
+CREATE TABLE t1 (i INT, KEY(i)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1);
+# 
+# Restart the server with a different value for key 4 in the key file
+# 
+SELECT * FROM t1;
+ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
+SELECT * FROM t1;
+i
+1
+DROP TABLE t1;

mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test

+#
+# MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key
+#
+
+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+# Suppression for builds where file_key_management plugin is linked statically
+call mtr.add_suppression("Couldn't load plugins from 'file_key_management*");
+
+--echo # 
+--echo # Restart the server with key 4 in the key file
+--echo # 
+
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys1.txt
+1;770A8A65DA156D24EE2A093277530142
+4;18420B5CBA31CCDFFE9716E91EB61374D05914F3ADE23E03
+EOF
+
+--exec echo "restart:--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+CREATE TABLE t1 (i INT, KEY(i)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1);
+
+--echo # 
+--echo # Restart the server with a different value for key 4 in the key file
+--echo # 
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys2.txt
+1;770A8A65DA156D24EE2A093277530142
+4;22222222222222222222222222222222
+EOF
+
+--exec echo "restart:--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--error 1296
+SELECT * FROM t1;
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--exec echo "restart:--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+SELECT * FROM t1;
+DROP TABLE t1;
+
+--remove_file $MYSQLTEST_VARDIR/keys2.txt
+--remove_file $MYSQLTEST_VARDIR/keys1.txt

storage/innobase/buf/buf0buf.cc

@@ -2868,18 +2868,24 @@ loop:
 			/* Do not try again for encrypted pages */
 			if (!corrupted) {
 				ib_mutex_t* pmutex = buf_page_get_mutex(bpage);
+
+				buf_pool = buf_pool_from_bpage(bpage);
 				buf_pool_mutex_enter(buf_pool);
 				mutex_enter(pmutex);
-				buf_block_t* block = buf_page_get_block(bpage);
+
+				ut_ad(buf_pool->n_pend_reads > 0);
+				os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1);
 				buf_page_set_io_fix(bpage, BUF_IO_NONE);
-				buf_block_set_state(block, BUF_BLOCK_NOT_USED);
-				buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
-				buf_pool_mutex_exit(buf_pool);
 				mutex_exit(pmutex);
+				buf_LRU_free_page(bpage, true);
+				buf_pool_mutex_exit(buf_pool);
+				rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock,
+					     BUF_IO_READ);
 
 				if (err) {
 					*err = DB_DECRYPTION_FAILED;
 				}
+
 				return (NULL);
 			}
 
@@ -2914,18 +2920,24 @@ loop:
 				ut_error;
 			} else {
 				ib_mutex_t* pmutex = buf_page_get_mutex(bpage);
+
+				buf_pool = buf_pool_from_bpage(bpage);
 				buf_pool_mutex_enter(buf_pool);
 				mutex_enter(pmutex);
-				buf_block_t* block = buf_page_get_block(bpage);
-				buf_page_set_io_fix(bpage, BUF_IO_NONE);
-				buf_block_set_state(block, BUF_BLOCK_NOT_USED);
-				buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
-				buf_pool_mutex_exit(buf_pool);
+
+				ut_ad(buf_pool->n_pend_reads > 0);
+				os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1);
+ 				buf_page_set_io_fix(bpage, BUF_IO_NONE);
 				mutex_exit(pmutex);
+				buf_LRU_free_page(bpage, true);
+				buf_pool_mutex_exit(buf_pool);
+				rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock,
+					     BUF_IO_READ);
 
 				if (err) {
 					*err = DB_DECRYPTION_FAILED;
 				}
+
 				return (NULL);
 			}
 		}

storage/innobase/dict/dict0stats.cc

@@ -962,6 +962,11 @@ dict_stats_update_transient(
 			continue;
 		}
 
+		/* Do not continue if table decryption has failed. */
+		if (index->table->is_encrypted) {
+			break;
+		}
+
 		dict_stats_update_transient_for_index(index);
 
 		sum_of_index_sizes += index->stat_index_size;

storage/xtradb/buf/buf0buf.cc

@@ -2911,16 +2911,23 @@ loop:
 				ib_mutex_t* pmutex = buf_page_get_mutex(bpage);
 				mutex_enter(&buf_pool->LRU_list_mutex);
 				mutex_enter(pmutex);
-				buf_block_t* block = buf_page_get_block(bpage);
+
+				ut_ad(buf_pool->n_pend_reads > 0);
+				os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1);
 				buf_page_set_io_fix(bpage, BUF_IO_NONE);
-				buf_block_set_state(block, BUF_BLOCK_NOT_USED);
-				buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
-				mutex_exit(&buf_pool->LRU_list_mutex);
+
+				if (!buf_LRU_free_page(bpage, true)) {
+					mutex_exit(&buf_pool->LRU_list_mutex);
+				}
+
 				mutex_exit(pmutex);
+				rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock,
+					     BUF_IO_READ);
 
 				if (err) {
 					*err = DB_DECRYPTION_FAILED;
 				}
+
 				return (NULL);
 			}
 
@@ -2957,16 +2964,23 @@ loop:
 				ib_mutex_t* pmutex = buf_page_get_mutex(bpage);
 				mutex_enter(&buf_pool->LRU_list_mutex);
 				mutex_enter(pmutex);
-				buf_block_t* block = buf_page_get_block(bpage);
+
+				ut_ad(buf_pool->n_pend_reads > 0);
+				os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1);
 				buf_page_set_io_fix(bpage, BUF_IO_NONE);
-				buf_block_set_state(block, BUF_BLOCK_NOT_USED);
-				buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
-				mutex_exit(&buf_pool->LRU_list_mutex);
+
+				if (!buf_LRU_free_page(bpage, true)) {
+					mutex_exit(&buf_pool->LRU_list_mutex);
+				}
+
 				mutex_exit(pmutex);
+				rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock,
+					     BUF_IO_READ);
 
 				if (err) {
 					*err = DB_DECRYPTION_FAILED;
 				}
+
 				return (NULL);
 			}
 		}

storage/xtradb/dict/dict0stats.cc

@@ -962,6 +962,11 @@ dict_stats_update_transient(
 			continue;
 		}
 
+		/* Do not continue if table decryption has failed. */
+		if (index->table->is_encrypted) {
+			break;
+		}
+
 		dict_stats_update_transient_for_index(index);
 
 		sum_of_index_sizes += index->stat_index_size;