MDEV-29401 InnoDB history list length increased in 10.6 compared to 10.5

The InnoDB buffer pool and locking were heavily refactored in
MariaDB Server 10.6. Among other things, dict_sys.mutex was removed,
and the contended lock_sys.mutex was replaced with a combination of
lock_sys.latch and distributed latches in hash tables. Also, a
default value was changed to innodb_flush_method=O_DIRECT to improve
performance in write-heavy workloads.

One thing where an adjustment was missing is around the parameters
innodb_max_purge_lag (number of committed transactions waiting to
be purged), and innodb_max_purge_lag_delay
(maximum number of microseconds to delay a DML operation).

purge_coordinator_state::do_purge(): Pass the history_size to trx_purge()
and reset srv_dml_needed_delay if the history is empty.
Keep executing the loop non-stop as long as srv_dml_needed_delay is set.

trx_purge_dml_delay(): Made part of trx_purge().
Set srv_dml_needed_delay=0 when nothing can be purged (!n_pages_handled).

row_mysql_delay_if_needed(): Mimic the logic of
innodb_max_purge_lag_wait_update().

Reviewed by: Thirunarayanan Balathandayuthapani

storage/innobase/include/trx0purge.h

@@ -41,10 +41,11 @@ void
 trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr);
 /**
 Run a purge batch.
-@param n_tasks   number of purge tasks to submit to the queue
-@param truncate  whether to truncate the history at the end of the batch
+@param n_tasks       number of purge tasks to submit to the queue
+@param history_size  trx_sys.history_size()
+@param truncate      whether to truncate the history at the end of the batch
 @return number of undo log pages handled in the batch */
-ulint trx_purge(ulint n_tasks, bool truncate);
+ulint trx_purge(ulint n_tasks, ulint history_size, bool truncate);
 
 /** Rollback segements from a given transaction with trx-no
 scheduled for purge. */

storage/innobase/row/row0mysql.cc

@@ -67,17 +67,23 @@ Created 9/17/2000 Heikki Tuuri
 #include <thread>
 
 
-/*******************************************************************//**
-Delays an INSERT, DELETE or UPDATE operation if the purge is lagging. */
-static
-void
-row_mysql_delay_if_needed(void)
-/*===========================*/
+/** Delay an INSERT, DELETE or UPDATE operation if the purge is lagging. */
+static void row_mysql_delay_if_needed()
 {
-	if (srv_dml_needed_delay) {
-		std::this_thread::sleep_for(
-			std::chrono::microseconds(srv_dml_needed_delay));
-	}
+  const auto delay= srv_dml_needed_delay;
+  if (UNIV_UNLIKELY(delay != 0))
+  {
+    /* Adjust for purge_coordinator_state::refresh() */
+    mysql_mutex_lock(&log_sys.mutex);
+    const lsn_t last= log_sys.last_checkpoint_lsn,
+      max_age= log_sys.max_checkpoint_age;
+    mysql_mutex_unlock(&log_sys.mutex);
+    const lsn_t lsn= log_sys.get_lsn();
+    if ((lsn - last) / 4 >= max_age / 5)
+      buf_flush_ahead(last + max_age / 5, false);
+    srv_wake_purge_thread_if_not_active();
+    std::this_thread::sleep_for(std::chrono::microseconds(delay));
+  }
 }
 
 /*******************************************************************//**

storage/innobase/srv/srv0mon.cc

@@ -752,7 +752,8 @@ static monitor_info_t	innodb_counter_info[] =
 
 	{"purge_dml_delay_usec", "purge",
 	 "Microseconds DML to be delayed due to purge lagging",
-	 MONITOR_DISPLAY_CURRENT,
+	 static_cast<monitor_type_t>(
+	 MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT),
 	 MONITOR_DEFAULT_START, MONITOR_DML_PURGE_DELAY},
 
 	{"purge_stop_count", "purge",
@@ -1710,6 +1711,9 @@ srv_mon_process_existing_counter(
 	case MONITOR_RSEG_CUR_SIZE:
 		value = srv_mon_get_rseg_size();
 		break;
+	case MONITOR_DML_PURGE_DELAY:
+		value = srv_max_purge_lag_delay;
+		break;
 	case MONITOR_NUM_UNDO_SLOT_USED:
 		value = srv_mon_get_rseg_used();
 		break;

storage/innobase/srv/srv0srv.cc

@@ -1689,27 +1689,32 @@ inline void purge_coordinator_state::do_purge()
 
     m_history_length= history_size;
 
-    if (history_size &&
-        trx_purge(n_use_threads,
-                  !(++count % srv_purge_rseg_truncate_frequency) ||
-                  purge_sys.truncate.current ||
-                  (srv_shutdown_state != SRV_SHUTDOWN_NONE &&
-                   srv_fast_shutdown == 0)))
+    if (!history_size)
+      srv_dml_needed_delay= 0;
+    else if (trx_purge(n_use_threads, history_size,
+                       !(++count % srv_purge_rseg_truncate_frequency) ||
+                       purge_sys.truncate.current ||
+                       (srv_shutdown_state != SRV_SHUTDOWN_NONE &&
+                        srv_fast_shutdown == 0)))
       continue;
 
-    if (m_running == sigcount)
+    if (srv_dml_needed_delay);
+    else if (m_running == sigcount)
     {
       /* Purge was not woken up by srv_wake_purge_thread_if_not_active() */
 
       /* The magic number 5000 is an approximation for the case where we have
       cached undo log records which prevent truncate of rollback segments. */
-      wakeup= history_size &&
-        (history_size >= 5000 ||
-         history_size != trx_sys.history_size_approx());
+      wakeup= history_size >= 5000 ||
+        (history_size && history_size != trx_sys.history_size_approx());
       break;
     }
-    else if (!trx_sys.history_exists())
+
+    if (!trx_sys.history_exists())
+    {
+      srv_dml_needed_delay= 0;
       break;
+    }
 
     if (!srv_purge_should_exit())
       goto loop;

storage/innobase/trx/trx0purge.cc

@@ -1243,43 +1243,6 @@ trx_purge_attach_undo_recs(ulint n_purge_threads)
 	return(n_pages_handled);
 }
 
-/*******************************************************************//**
-Calculate the DML delay required.
-@return delay in microseconds or ULINT_MAX */
-static
-ulint
-trx_purge_dml_delay(void)
-/*=====================*/
-{
-	/* Determine how much data manipulation language (DML) statements
-	need to be delayed in order to reduce the lagging of the purge
-	thread. */
-	ulint	delay = 0; /* in microseconds; default: no delay */
-
-	/* If purge lag is set then calculate the new DML delay. */
-
-	if (srv_max_purge_lag > 0) {
-		double ratio = static_cast<double>(trx_sys.history_size()) /
-			static_cast<double>(srv_max_purge_lag);
-
-		if (ratio > 1.0) {
-			/* If the history list length exceeds the
-			srv_max_purge_lag, the data manipulation
-			statements are delayed by at least 5000
-			microseconds. */
-			delay = (ulint) ((ratio - .5) * 10000);
-		}
-
-		if (delay > srv_max_purge_lag_delay) {
-			delay = srv_max_purge_lag_delay;
-		}
-
-		MONITOR_SET(MONITOR_DML_PURGE_DELAY, delay);
-	}
-
-	return(delay);
-}
-
 extern tpool::waitable_task purge_worker_task;
 
 /** Wait for pending purge jobs to complete. */
@@ -1323,18 +1286,18 @@ TRANSACTIONAL_INLINE void purge_sys_t::clone_end_view()
 
 /**
 Run a purge batch.
-@param n_tasks   number of purge tasks to submit to the queue
-@param truncate  whether to truncate the history at the end of the batch
+@param n_tasks       number of purge tasks to submit to the queue
+@param history_size  trx_sys.history_size()
+@param truncate      whether to truncate the history at the end of the batch
 @return number of undo log pages handled in the batch */
-TRANSACTIONAL_TARGET ulint trx_purge(ulint n_tasks, bool truncate)
+TRANSACTIONAL_TARGET
+ulint trx_purge(ulint n_tasks, ulint history_size, bool truncate)
 {
 	que_thr_t*	thr = NULL;
 	ulint		n_pages_handled;
 
 	ut_ad(n_tasks > 0);
 
-	srv_dml_needed_delay = trx_purge_dml_delay();
-
 	purge_sys.clone_oldest_view();
 
 #ifdef UNIV_DEBUG
@@ -1346,6 +1309,24 @@ TRANSACTIONAL_TARGET ulint trx_purge(ulint n_tasks, bool truncate)
 	/* Fetch the UNDO recs that need to be purged. */
 	n_pages_handled = trx_purge_attach_undo_recs(n_tasks);
 
+	{
+		ulint delay = n_pages_handled ? srv_max_purge_lag : 0;
+		if (UNIV_UNLIKELY(delay)) {
+			if (delay >= history_size) {
+		no_throttle:
+				delay = 0;
+			} else if (const ulint max_delay =
+				   srv_max_purge_lag_delay) {
+				delay = std::min(max_delay,
+						 10000 * history_size / delay
+						 - 5000);
+			} else {
+				goto no_throttle;
+			}
+		}
+		srv_dml_needed_delay = delay;
+	}
+
 	/* Submit tasks to workers queue if using multi-threaded purge. */
 	for (ulint i = n_tasks; --i; ) {
 		thr = que_fork_scheduler_round_robin(purge_sys.query, thr);