MDEV-8772: Assertion failure in file ha_innodb.cc line 20027 when importing...

MDEV-8772: Assertion failure in file ha_innodb.cc line 20027 when importing page compressed and encrypted tablespace using incorrect keys

Add error handling to decryp function when decrypt fails during
import.

mysql-test/suite/encryption/r/innodb-bad-key-change3.result

+call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded");
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+set global innodb_compression_algorithm = 1;
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(255)) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+SHOW WARNINGS;
+Level	Code	Message
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `pk` int(11) NOT NULL,
+  `f` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`pk`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 `PAGE_COMPRESSED`=1 `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=4
+INSERT INTO t1 VALUES (1,'foobar'),(2,'barfoo');
+FLUSH TABLE t1 FOR EXPORT;
+# List before copying files
+t1.cfg
+t1.frm
+t1.ibd
+UNLOCK TABLES;
+# Tablespaces should be still encrypted
+# t1 yes on expecting NOT FOUND
+NOT FOUND /foobar/ in t1.ibd
+ALTER TABLE t1 DISCARD TABLESPACE;
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+# List after t1 DISCARD
+t1.frm
+ALTER TABLE t1 IMPORT TABLESPACE;
+ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `pk` int(11) NOT NULL,
+  `f` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`pk`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 `PAGE_COMPRESSED`=1 `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=4
+SELECT * FROM t1;
+ERROR HY000: Tablespace has been discarded for table 't1'
+# Tablespaces should be still encrypted
+# t1 yes on expecting NOT FOUND
+NOT FOUND /foobar/ in t1.ibd
+DROP TABLE t1;

mysql-test/suite/encryption/t/innodb-bad-key-change3.test

+--source include/have_innodb.inc
+# embedded does not support restart
+-- source include/not_embedded.inc
+-- source include/not_valgrind.inc
+# Avoid CrashReporter popup on Mac
+-- source include/not_crashrep.inc
+
+#
+# MDEV-8772: Assertion failure in file ha_innodb.cc line 20027 when importing page compressed and encrypted tablespace using incorrect keys
+#
+call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded");
+
+--disable_query_log
+let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
+let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
+--enable_query_log
+
+--let $MYSQLD_TMPDIR = `SELECT @@tmpdir`
+--let $MYSQLD_DATADIR = `SELECT @@datadir`
+--let SEARCH_RANGE = 10000000
+--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys1.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530143
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+set global innodb_compression_algorithm = 1;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(255)) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+SHOW WARNINGS;
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES (1,'foobar'),(2,'barfoo');
+FLUSH TABLE t1 FOR EXPORT;
+--echo # List before copying files
+--list_files $MYSQLD_DATADIR/test
+--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1.cfg
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1.ibd
+UNLOCK TABLES;
+
+--sleep 5
+--echo # Tablespaces should be still encrypted
+--let SEARCH_PATTERN=foobar
+--echo # t1 yes on expecting NOT FOUND
+-- let SEARCH_FILE=$t1_IBD
+-- source include/search_pattern_in_file.inc
+
+ALTER TABLE t1 DISCARD TABLESPACE;
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys2.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530144
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--source include/restart_mysqld.inc
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+--echo # List after t1 DISCARD
+--list_files $MYSQLD_DATADIR/test
+--copy_file $MYSQLD_TMPDIR/t1.cfg $MYSQLD_DATADIR/test/t1.cfg
+--copy_file $MYSQLD_TMPDIR/t1.ibd $MYSQLD_DATADIR/test/t1.ibd
+
+--error ER_GET_ERRMSG
+ALTER TABLE t1 IMPORT TABLESPACE;
+SHOW CREATE TABLE t1;
+--error ER_TABLESPACE_DISCARDED
+SELECT * FROM t1;
+--sleep 5
+--echo # Tablespaces should be still encrypted
+-- let SEARCH_FILE=$t1_IBD
+--let SEARCH_PATTERN=foobar
+--echo # t1 yes on expecting NOT FOUND
+-- let SEARCH_FILE=$t1_IBD
+-- source include/search_pattern_in_file.inc
+DROP TABLE t1;
+
+# reset system
+--disable_query_log
+EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
+EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
+--enable_query_log
+
+--remove_file $MYSQLTEST_VARDIR/keys1.txt
+--remove_file $MYSQLTEST_VARDIR/keys2.txt

storage/innobase/fil/fil0crypt.cc

@@ -714,12 +714,16 @@ fil_space_decrypt(
 	fil_space_crypt_t*	crypt_data,	/*!< in: crypt data */
 	byte*			tmp_frame,	/*!< in: temporary buffer */
 	ulint			page_size,	/*!< in: page size */
-	byte*			src_frame)	/*!< in:out: page buffer */
+	byte*			src_frame,	/*!< in: out: page buffer */
+	dberr_t*		err)		/*!< in: out: DB_SUCCESS or
+						error code */
 {
 	ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
 	uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
 	bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
 
+	*err = DB_SUCCESS;
+
 	if (key_version == ENCRYPTION_KEY_NOT_ENCRYPTED) {
 		return false;
 	}
@@ -756,6 +760,12 @@ fil_space_decrypt(
 					   space, offset, lsn);
 
 	if (! ((rc == MY_AES_OK) && ((ulint) dstlen == srclen))) {
+
+		if (rc == -1) {
+			*err = DB_DECRYPTION_FAILED;
+			return false;
+		}
+
 		ib_logf(IB_LOG_LEVEL_FATAL,
 			"Unable to decrypt data-block "
 			" src: %p srclen: %ld buf: %p buflen: %d."
@@ -797,11 +807,14 @@ fil_space_decrypt(
 	ulint		page_size,	/*!< in: page size */
 	byte*		src_frame)	/*!< in/out: page buffer */
 {
+	dberr_t err = DB_SUCCESS;
+
 	bool encrypted = fil_space_decrypt(
 				fil_space_get_crypt_data(space),
 				tmp_frame,
 				page_size,
-				src_frame);
+				src_frame,
+				&err);
 
 	if (encrypted) {
 		/* Copy the decrypted page back to page buffer, not

storage/innobase/fil/fil0fil.cc

@@ -6425,6 +6425,7 @@ fil_iterate(
 		if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
 				(srv_encrypt_tables &&
 					iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
+
 			encrypted = true;
 			readptr = iter.crypt_io_buffer;
 			writeptr = iter.crypt_io_buffer;
@@ -6444,6 +6445,7 @@ fil_iterate(
 
 		for (ulint i = 0; i < n_pages_read; ++i) {
 			ulint size = iter.page_size;
+			dberr_t	err = DB_SUCCESS;
 
 			/* If tablespace is encrypted, we need to decrypt
 			the page. */
@@ -6452,10 +6454,14 @@ fil_iterate(
 							iter.crypt_data,
 							io_buffer + i * size, //dst
 							iter.page_size,
-							readptr + i * size); // src
+							readptr + i * size, // src
+							&err); // src
+
+				if (err != DB_SUCCESS) {
+					return(err);
+				}
 
 				if (decrypted) {
-					/* write back unencrypted page */
 					updated = true;
 				} else {
 					/* TODO: remove unnecessary memcpy's */
@@ -6465,7 +6471,6 @@ fil_iterate(
 
 			buf_block_set_file_page(block, space_id, page_no++);
 
-			dberr_t	err;
 
 			if ((err = callback(page_off, block)) != DB_SUCCESS) {
 

storage/innobase/include/fil0crypt.h

@@ -211,7 +211,10 @@ fil_space_decrypt(
 	fil_space_crypt_t*	crypt_data,	/*!< in: crypt data */
 	byte*			tmp_frame,	/*!< in: temporary buffer */
 	ulint			page_size,	/*!< in: page size */
-	byte*			src_frame);	/*!< in:out: page buffer */
+	byte*			src_frame,	/*!< in:out: page buffer */
+	dberr_t*		err);		/*!< in: out: DB_SUCCESS or
+						error code */
+
 
 /*********************************************************************
 Encrypt buffer page

storage/innobase/row/row0import.cc

 /*****************************************************************************
 
 Copyright (c) 2012, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2015, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -3597,10 +3598,13 @@ row_import_for_mysql(
 		innobase_format_name(
 			table_name, sizeof(table_name), table->name, FALSE);
 
+		if (err != DB_DECRYPTION_FAILED) {
+
 			ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR,
 				ER_INTERNAL_ERROR,
 				"Cannot reset LSNs in table '%s' : %s",
 				table_name, ut_strerr(err));
+		}
 
 		return(row_import_cleanup(prebuilt, trx, err));
 	}

storage/xtradb/fil/fil0crypt.cc

@@ -714,12 +714,16 @@ fil_space_decrypt(
 	fil_space_crypt_t*	crypt_data,	/*!< in: crypt data */
 	byte*			tmp_frame,	/*!< in: temporary buffer */
 	ulint			page_size,	/*!< in: page size */
-	byte*			src_frame)	/*!< in:out: page buffer */
+	byte*			src_frame,	/*!< in: out: page buffer */
+	dberr_t*		err)		/*!< in: out: DB_SUCCESS or
+						error code */
 {
 	ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
 	uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
 	bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
 
+	*err = DB_SUCCESS;
+
 	if (key_version == ENCRYPTION_KEY_NOT_ENCRYPTED) {
 		return false;
 	}
@@ -756,6 +760,12 @@ fil_space_decrypt(
 					   space, offset, lsn);
 
 	if (! ((rc == MY_AES_OK) && ((ulint) dstlen == srclen))) {
+
+		if (rc == -1) {
+			*err = DB_DECRYPTION_FAILED;
+			return false;
+		}
+
 		ib_logf(IB_LOG_LEVEL_FATAL,
 			"Unable to decrypt data-block "
 			" src: %p srclen: %ld buf: %p buflen: %d."
@@ -797,11 +807,14 @@ fil_space_decrypt(
 	ulint		page_size,	/*!< in: page size */
 	byte*		src_frame)	/*!< in/out: page buffer */
 {
+	dberr_t err = DB_SUCCESS;
+
 	bool encrypted = fil_space_decrypt(
 				fil_space_get_crypt_data(space),
 				tmp_frame,
 				page_size,
-				src_frame);
+				src_frame,
+				&err);
 
 	if (encrypted) {
 		/* Copy the decrypted page back to page buffer, not

storage/xtradb/fil/fil0fil.cc

@@ -6482,6 +6482,7 @@ fil_iterate(
 		if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
 				(srv_encrypt_tables &&
 					iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
+
 			encrypted = true;
 			readptr = iter.crypt_io_buffer;
 			writeptr = iter.crypt_io_buffer;
@@ -6501,6 +6502,7 @@ fil_iterate(
 
 		for (ulint i = 0; i < n_pages_read; ++i) {
 			ulint size = iter.page_size;
+			dberr_t	err = DB_SUCCESS;
 
 			/* If tablespace is encrypted, we need to decrypt
 			the page. */
@@ -6509,10 +6511,14 @@ fil_iterate(
 							iter.crypt_data,
 							io_buffer + i * size, //dst
 							iter.page_size,
-							readptr + i * size); // src
+							readptr + i * size, // src
+							&err); // src
+
+				if (err != DB_SUCCESS) {
+					return(err);
+				}
 
 				if (decrypted) {
-					/* write back unencrypted page */
 					updated = true;
 				} else {
 					/* TODO: remove unnecessary memcpy's */
@@ -6522,7 +6528,6 @@ fil_iterate(
 
 			buf_block_set_file_page(block, space_id, page_no++);
 
-			dberr_t	err;
 
 			if ((err = callback(page_off, block)) != DB_SUCCESS) {
 

storage/xtradb/include/fil0crypt.h

@@ -211,7 +211,10 @@ fil_space_decrypt(
 	fil_space_crypt_t*	crypt_data,	/*!< in: crypt data */
 	byte*			tmp_frame,	/*!< in: temporary buffer */
 	ulint			page_size,	/*!< in: page size */
-	byte*			src_frame);	/*!< in:out: page buffer */
+	byte*			src_frame,	/*!< in:out: page buffer */
+	dberr_t*		err);		/*!< in: out: DB_SUCCESS or
+						error code */
+
 
 /*********************************************************************
 Encrypt buffer page

storage/xtradb/row/row0import.cc

 /*****************************************************************************
 
 Copyright (c) 2012, 2013, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2015, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -3620,10 +3621,13 @@ row_import_for_mysql(
 		innobase_format_name(
 			table_name, sizeof(table_name), table->name, FALSE);
 
+		if (err != DB_DECRYPTION_FAILED) {
+
 			ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR,
 				ER_INTERNAL_ERROR,
 				"Cannot reset LSNs in table '%s' : %s",
 				table_name, ut_strerr(err));
+		}
 
 		return(row_import_cleanup(prebuilt, trx, err));
 	}