MDEV-21168: Active XA transactions stop slave from working after backup

was restored.

Optionally rollback prepared XA's on "mariabackup --prepare".

The fix MUST NOT be ported on 10.5+, as MDEV-742 fix solves the issue for
slaves.

mysql-test/suite/mariabackup/innodb_xa_rollback.result

+CALL mtr.add_suppression("Found 1 prepared XA transactions");
+RESET MASTER;
+CREATE TABLE t1 (a INT) ENGINE=INNODB;
+XA START 'test1';
+INSERT t1 VALUES (10);
+XA END 'test1';
+XA PREPARE 'test1';
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	5	0	test1
+# xtrabackup backup
+XA ROLLBACK 'test1';
+# xtrabackup prepare and rollback prepared XA
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+# xtrabackup prepare and DO NOT rollback prepared XA
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	5	0	test1
+XA ROLLBACK 'test1';
+# xtrabackup prepare for export and rollback prepared XA
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+# xtrabackup prepare for export and DO NOT rollback prepared XA
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	5	0	test1
+XA ROLLBACK 'test1';
+DROP TABLE t1;

mysql-test/suite/mariabackup/innodb_xa_rollback.test

+#
+# Optionally rollback prepared XA when backup is prepared
+#
+--source include/have_innodb.inc
+--source include/have_binlog_format_mixed.inc
+
+CALL mtr.add_suppression("Found 1 prepared XA transactions");
+
+RESET MASTER;
+
+let targetdir1=$MYSQLTEST_VARDIR/tmp/backup1;
+let targetdir2=$MYSQLTEST_VARDIR/tmp/backup2;
+let targetdir3=$MYSQLTEST_VARDIR/tmp/backup3;
+let targetdir4=$MYSQLTEST_VARDIR/tmp/backup4;
+
+CREATE TABLE t1 (a INT) ENGINE=INNODB;
+XA START 'test1';
+INSERT t1 VALUES (10);
+XA END 'test1';
+XA PREPARE 'test1';
+XA RECOVER;
+
+--echo # xtrabackup backup
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir1;
+--enable_result_log
+
+perl;
+use lib "lib";
+use My::Handles { suppress_init_messages => 1 };
+use My::File::Path;
+copytree($ENV{'targetdir1'}, $ENV{'targetdir2'});
+copytree($ENV{'targetdir1'}, $ENV{'targetdir3'});
+copytree($ENV{'targetdir1'}, $ENV{'targetdir4'});
+EOF
+
+XA ROLLBACK 'test1';
+
+--echo # xtrabackup prepare and rollback prepared XA
+--disable_result_log
+exec $XTRABACKUP  --prepare --rollback_xa --target-dir=$targetdir1;
+--let $targetdir = $targetdir1
+--source include/restart_and_restore.inc
+--enable_result_log
+XA RECOVER;
+
+--echo # xtrabackup prepare and DO NOT rollback prepared XA
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir2;
+--let $targetdir = $targetdir2
+--source include/restart_and_restore.inc
+--enable_result_log
+XA RECOVER;
+XA ROLLBACK 'test1';
+
+--echo # xtrabackup prepare for export and rollback prepared XA
+--disable_result_log
+exec $XTRABACKUP  --prepare --rollback_xa --export --target-dir=$targetdir3;
+--let $targetdir = $targetdir3
+--source include/restart_and_restore.inc
+--enable_result_log
+XA RECOVER;
+
+--echo # xtrabackup prepare for export and DO NOT rollback prepared XA
+--disable_result_log
+exec $XTRABACKUP  --prepare --export --target-dir=$targetdir4;
+--let $targetdir = $targetdir4
+--source include/restart_and_restore.inc
+--enable_result_log
+XA RECOVER;
+XA ROLLBACK 'test1';
+
+DROP TABLE t1;
+rmdir $targetdir1;
+rmdir $targetdir2;
+rmdir $targetdir3;
+rmdir $targetdir4;

sql/handler.cc

@@ -1806,29 +1806,33 @@ int ha_commit_or_rollback_by_xid(XID *xid, bool commit)
 
 
 #ifndef DBUG_OFF
-/**
-  @note
-    This does not need to be multi-byte safe or anything
-*/
-static char* xid_to_str(char *buf, XID *xid)
+/** Converts XID to string.
+
+@param[out] buf output buffer
+@param[in] xid XID to convert
+
+@return pointer to converted string
+
+@note This does not need to be multi-byte safe or anything */
+char *xid_to_str(char *buf, const XID &xid)
 {
   int i;
   char *s=buf;
   *s++='\'';
-  for (i=0; i < xid->gtrid_length+xid->bqual_length; i++)
+  for (i= 0; i < xid.gtrid_length + xid.bqual_length; i++)
   {
-    uchar c=(uchar)xid->data[i];
+    uchar c= (uchar) xid.data[i];
     /* is_next_dig is set if next character is a number */
     bool is_next_dig= FALSE;
     if (i < XIDDATASIZE)
     {
-      char ch= xid->data[i+1];
+      char ch= xid.data[i + 1];
       is_next_dig= (ch >= '0' && ch <='9');
     }
-    if (i == xid->gtrid_length)
+    if (i == xid.gtrid_length)
     {
       *s++='\'';
-      if (xid->bqual_length)
+      if (xid.bqual_length)
       {
         *s++='.';
         *s++='\'';
@@ -1949,7 +1953,8 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
         {
 #ifndef DBUG_OFF
           char buf[XIDDATASIZE*4+6]; // see xid_to_str
-          DBUG_PRINT("info", ("ignore xid %s", xid_to_str(buf, info->list+i)));
+          DBUG_PRINT("info",
+                     ("ignore xid %s", xid_to_str(buf, info->list[i])));
 #endif
           xid_cache_insert(info->list+i, XA_PREPARED);
           info->found_foreign_xids++;
@@ -1979,7 +1984,8 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
           if (rc == 0)
           {
             char buf[XIDDATASIZE*4+6]; // see xid_to_str
-            DBUG_PRINT("info", ("commit xid %s", xid_to_str(buf, info->list+i)));
+            DBUG_PRINT("info",
+                       ("commit xid %s", xid_to_str(buf, info->list[i])));
           }
 #endif
         }
@@ -1993,8 +1999,8 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
           if (rc == 0)
           {
             char buf[XIDDATASIZE*4+6]; // see xid_to_str
-            DBUG_PRINT("info", ("rollback xid %s",
-                                xid_to_str(buf, info->list+i)));
+            DBUG_PRINT("info",
+                       ("rollback xid %s", xid_to_str(buf, info->list[i])));
           }
 #endif
         }

sql/handler.h

@@ -4387,4 +4387,16 @@ void print_keydup_error(TABLE *table, KEY *key, myf errflag);
 
 int del_global_index_stat(THD *thd, TABLE* table, KEY* key_info);
 int del_global_table_stat(THD *thd, LEX_STRING *db, LEX_STRING *table);
+#ifndef DBUG_OFF
+/** Converts XID to string.
+
+@param[out] buf output buffer
+@param[in] xid XID to convert
+
+@return pointer to converted string
+
+@note This does not need to be multi-byte safe or anything */
+char *xid_to_str(char *buf, const XID &xid);
+#endif // !DBUG_OFF
+
 #endif /* HANDLER_INCLUDED */

storage/innobase/fil/fil0fil.cc

@@ -4253,6 +4253,7 @@ fil_ibd_discover(
 		case SRV_OPERATION_RESTORE_DELTA:
 			ut_ad(0);
 			break;
+		case SRV_OPERATION_RESTORE_ROLLBACK_XA:
 		case SRV_OPERATION_RESTORE_EXPORT:
 		case SRV_OPERATION_RESTORE:
 			break;
@@ -4350,7 +4351,7 @@ fil_ibd_load(
 		return(FIL_LOAD_OK);
 	}
 
-	if (srv_operation == SRV_OPERATION_RESTORE) {
+	if (is_mariabackup_restore()) {
 		/* Replace absolute DATA DIRECTORY file paths with
 		short names relative to the backup directory. */
 		if (const char* name = strrchr(filename, OS_PATH_SEPARATOR)) {

storage/innobase/handler/ha_innodb.cc

@@ -1408,18 +1408,6 @@ innobase_commit_by_xid(
 	handlerton*	hton,		/*!< in: InnoDB handlerton */
 	XID*		xid);		/*!< in: X/Open XA transaction
 					identification */
-/*******************************************************************//**
-This function is used to rollback one X/Open XA distributed transaction
-which is in the prepared state
-@return 0 or error number */
-static
-int
-innobase_rollback_by_xid(
-/*=====================*/
-	handlerton*	hton,		/*!< in: InnoDB handlerton */
-	XID*		xid);		/*!< in: X/Open XA transaction
-					identification */
-
 /** Remove all tables in the named database inside InnoDB.
 @param[in]	hton	handlerton from InnoDB
 @param[in]	path	Database path; Inside InnoDB the name of the last
@@ -3591,12 +3579,8 @@ ha_innobase::init_table_handle_for_HANDLER(void)
 	reset_template();
 }
 
-/*********************************************************************//**
-Free tablespace resources allocated. */
-static
-void
-innobase_space_shutdown()
-/*=====================*/
+/** Free tablespace resources allocated. */
+void innobase_space_shutdown()
 {
 	DBUG_ENTER("innobase_space_shutdown");
 
@@ -17495,17 +17479,14 @@ innobase_commit_by_xid(
 	}
 }
 
-/*******************************************************************//**
-This function is used to rollback one X/Open XA distributed transaction
+/** This function is used to rollback one X/Open XA distributed transaction
 which is in the prepared state
+
+@param[in] hton InnoDB handlerton
+@param[in] xid X/Open XA transaction identification
+
 @return 0 or error number */
-static
-int
-innobase_rollback_by_xid(
-/*=====================*/
-	handlerton*	hton,	/*!< in: InnoDB handlerton */
-	XID*		xid)	/*!< in: X/Open XA transaction
-				identification */
+int innobase_rollback_by_xid(handlerton* hton, XID* xid)
 {
 	DBUG_ASSERT(hton == innodb_hton_ptr);
 

storage/innobase/handler/ha_innodb.h

@@ -970,3 +970,15 @@ ib_push_frm_error(
 @return true if index column length exceeds limit */
 MY_ATTRIBUTE((warn_unused_result))
 bool too_big_key_part_length(size_t max_field_len, const KEY& key);
+
+/** This function is used to rollback one X/Open XA distributed transaction
+which is in the prepared state
+
+@param[in] hton InnoDB handlerton
+@param[in] xid X/Open XA transaction identification
+
+@return 0 or error number */
+int innobase_rollback_by_xid(handlerton* hton, XID* xid);
+
+/** Free tablespace resources allocated. */
+void innobase_space_shutdown();

storage/innobase/include/srv0srv.h

@@ -518,6 +518,8 @@ enum srv_operation_mode {
 	SRV_OPERATION_BACKUP,
 	/** Mariabackup restoring a backup for subsequent --copy-back */
 	SRV_OPERATION_RESTORE,
+	/** Mariabackup restoring a backup with rolling back prepared XA's*/
+	SRV_OPERATION_RESTORE_ROLLBACK_XA,
 	/** Mariabackup restoring the incremental part of a backup */
 	SRV_OPERATION_RESTORE_DELTA,
 	/** Mariabackup restoring a backup for subsequent --export */
@@ -527,6 +529,21 @@ enum srv_operation_mode {
 /** Current mode of operation */
 extern enum srv_operation_mode srv_operation;
 
+inline bool is_mariabackup_restore()
+{
+	/* To rollback XA's trx_sys must be initialized, the rest is the same
+	as regular backup restore, that is why we join this two operations in
+	the most cases. */
+	return srv_operation == SRV_OPERATION_RESTORE
+	       || srv_operation == SRV_OPERATION_RESTORE_ROLLBACK_XA;
+}
+
+inline bool is_mariabackup_restore_or_export()
+{
+	return is_mariabackup_restore()
+	       || srv_operation == SRV_OPERATION_RESTORE_EXPORT;
+}
+
 extern my_bool	srv_print_innodb_monitor;
 extern my_bool	srv_print_innodb_lock_monitor;
 extern ibool	srv_print_verbose_log;

storage/innobase/log/log0recv.cc

@@ -406,8 +406,7 @@ fil_name_process(
 	}
 
 	ut_ad(srv_operation == SRV_OPERATION_NORMAL
-	      || srv_operation == SRV_OPERATION_RESTORE
-	      || srv_operation == SRV_OPERATION_RESTORE_EXPORT);
+	      || is_mariabackup_restore_or_export());
 
 	/* We will also insert space=NULL into the map, so that
 	further checks can ensure that a MLOG_FILE_NAME record was
@@ -2392,8 +2391,7 @@ buf_block_t* recv_recovery_create_page_low(const page_id_t page_id)
 void recv_apply_hashed_log_recs(bool last_batch)
 {
 	ut_ad(srv_operation == SRV_OPERATION_NORMAL
-	      || srv_operation == SRV_OPERATION_RESTORE
-	      || srv_operation == SRV_OPERATION_RESTORE_EXPORT);
+	      || is_mariabackup_restore_or_export());
 
 	mutex_enter(&recv_sys->mutex);
 
@@ -2411,9 +2409,8 @@ void recv_apply_hashed_log_recs(bool last_batch)
 
 	ut_ad(!last_batch == log_mutex_own());
 
-	recv_no_ibuf_operations = !last_batch
-		|| srv_operation == SRV_OPERATION_RESTORE
-		|| srv_operation == SRV_OPERATION_RESTORE_EXPORT;
+	recv_no_ibuf_operations
+		= !last_batch || is_mariabackup_restore_or_export();
 
 	ut_d(recv_no_log_write = recv_no_ibuf_operations);
 
@@ -3519,8 +3516,7 @@ static
 dberr_t
 recv_init_missing_space(dberr_t err, const recv_spaces_t::const_iterator& i)
 {
-	if (srv_operation == SRV_OPERATION_RESTORE
-	    || srv_operation == SRV_OPERATION_RESTORE_EXPORT) {
+	if (is_mariabackup_restore_or_export()) {
 		ib::warn() << "Tablespace " << i->first << " was not"
 			" found at " << i->second.name << " when"
 			" restoring a (partial?) backup. All redo log"
@@ -3697,8 +3693,7 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 	dberr_t		err = DB_SUCCESS;
 
 	ut_ad(srv_operation == SRV_OPERATION_NORMAL
-	      || srv_operation == SRV_OPERATION_RESTORE
-	      || srv_operation == SRV_OPERATION_RESTORE_EXPORT);
+	      || is_mariabackup_restore_or_export());
 
 	/* Initialize red-black tree for fast insertions into the
 	flush_list during recovery process. */

storage/innobase/srv/srv0start.cc

@@ -874,6 +874,7 @@ srv_undo_tablespaces_init(bool create_new_db)
 			break;
 		}
 		/* fall through */
+	case SRV_OPERATION_RESTORE_ROLLBACK_XA:
 	case SRV_OPERATION_RESTORE:
 	case SRV_OPERATION_RESTORE_EXPORT:
 		ut_ad(!create_new_db);
@@ -1280,6 +1281,7 @@ srv_shutdown_all_bg_threads()
 		case SRV_OPERATION_RESTORE_DELTA:
 			break;
 		case SRV_OPERATION_NORMAL:
+		case SRV_OPERATION_RESTORE_ROLLBACK_XA:
 		case SRV_OPERATION_RESTORE:
 		case SRV_OPERATION_RESTORE_EXPORT:
 			if (!buf_page_cleaner_is_active
@@ -1471,8 +1473,7 @@ innobase_start_or_create_for_mysql()
 	unsigned	i = 0;
 
 	ut_ad(srv_operation == SRV_OPERATION_NORMAL
-	      || srv_operation == SRV_OPERATION_RESTORE
-	      || srv_operation == SRV_OPERATION_RESTORE_EXPORT);
+	      || is_mariabackup_restore_or_export());
 
 	if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) {
 		srv_read_only_mode = true;
@@ -1950,14 +1951,9 @@ innobase_start_or_create_for_mysql()
 				srv_read_only_mode);
 
 			if (err == DB_NOT_FOUND) {
-				if (i == 0) {
-					if (srv_operation
-					    == SRV_OPERATION_RESTORE
-					    || srv_operation
-					    == SRV_OPERATION_RESTORE_EXPORT) {
-						return(DB_SUCCESS);
-					}
-				}
+				if (i == 0
+				    && is_mariabackup_restore_or_export())
+					return (DB_SUCCESS);
 
 				/* opened all files */
 				break;
@@ -1984,10 +1980,7 @@ innobase_start_or_create_for_mysql()
 
 			if (i == 0) {
 				if (size == 0
-				    && (srv_operation
-					== SRV_OPERATION_RESTORE
-					|| srv_operation
-					== SRV_OPERATION_RESTORE_EXPORT)) {
+				    && is_mariabackup_restore_or_export()) {
 					/* Tolerate an empty ib_logfile0
 					from a previous run of
 					mariabackup --prepare. */
@@ -2217,6 +2210,7 @@ innobase_start_or_create_for_mysql()
 
 		switch (srv_operation) {
 		case SRV_OPERATION_NORMAL:
+		case SRV_OPERATION_RESTORE_ROLLBACK_XA:
 		case SRV_OPERATION_RESTORE_EXPORT:
 			/* Initialize the change buffer. */
 			err = dict_boot();
@@ -2321,8 +2315,7 @@ innobase_start_or_create_for_mysql()
 
 		recv_recovery_from_checkpoint_finish();
 
-		if (srv_operation == SRV_OPERATION_RESTORE
-		    || srv_operation == SRV_OPERATION_RESTORE_EXPORT) {
+		if (is_mariabackup_restore_or_export()) {
 			/* After applying the redo log from
 			SRV_OPERATION_BACKUP, flush the changes
 			to the data files and truncate or delete the log.
@@ -2337,8 +2330,7 @@ innobase_start_or_create_for_mysql()
 			fil_close_log_files(true);
 			log_group_close_all();
 			if (err == DB_SUCCESS) {
-				bool trunc = srv_operation
-					== SRV_OPERATION_RESTORE;
+				bool trunc = is_mariabackup_restore();
 				/* Delete subsequent log files. */
 				delete_log_files(logfilename, dirnamelen,
 						 srv_n_log_files_found, trunc);
@@ -2632,7 +2624,9 @@ innobase_start_or_create_for_mysql()
 		srv_start_state_set(SRV_START_STATE_MASTER);
 	}
 
-	if (!srv_read_only_mode && srv_operation == SRV_OPERATION_NORMAL
+	if (!srv_read_only_mode
+	    && (srv_operation == SRV_OPERATION_NORMAL
+		|| srv_operation == SRV_OPERATION_RESTORE_ROLLBACK_XA)
 	    && srv_force_recovery < SRV_FORCE_NO_BACKGROUND) {
 		srv_undo_sources = true;
 		/* Create the dict stats gathering thread */
@@ -2787,6 +2781,7 @@ innodb_shutdown()
 	case SRV_OPERATION_RESTORE:
 	case SRV_OPERATION_RESTORE_DELTA:
 	case SRV_OPERATION_RESTORE_EXPORT:
+	case SRV_OPERATION_RESTORE_ROLLBACK_XA:
 		fil_close_all_files();
 		break;
 	case SRV_OPERATION_NORMAL:

storage/innobase/trx/trx0trx.cc

@@ -584,9 +584,7 @@ trx_free_prepared(
 	     || (trx->is_recovered
 		 && (trx_state_eq(trx, TRX_STATE_ACTIVE)
 		     || trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY))
-		 && (!srv_was_started
-		     || srv_operation == SRV_OPERATION_RESTORE
-		     || srv_operation == SRV_OPERATION_RESTORE_EXPORT
+		 && (!srv_was_started || is_mariabackup_restore_or_export()
 		     || srv_read_only_mode
 		     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO)));
 	ut_a(trx->magic_n == TRX_MAGIC_N);