[storage/innobase] PS-7949: Stop buf_lru_manager_thread during calls …

…to buf_pool_invalidate_instance

https://jira.percona.com/browse/PS-7949

---

*Problem*:

For buffer pool invalidation to proceed we must ensure there is no
LRU write activity happening.

PS does not implement this guarantee as PS implements a multi-threaded
LRU Manager unlike MySQL that uses a single thread.

This error manifests when running the test `clone.remote_ddl_basic`. It
is a not deterministic error due to the interleaving of execution of
several threads.

*Solution*:

`buf_pool_invalidate_instance` reset the  the event
`buf_pool_t->run_lru` and `buf_lru_manager_thread` waits for the event
`buf_pool_t->run_lru` to be set.

storage/innobase/buf/buf0buf.cc

@@ -56,6 +56,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "lock0lock.h"
 #include "log0buf.h"
 #include "log0chkp.h"
+#include "scope_guard.h"
 #include "sync0rw.h"
 #include "trx0purge.h"
 #include "trx0undo.h"
@@ -1354,6 +1355,9 @@ static void buf_pool_create(buf_pool_t *buf_pool, ulint buf_pool_size,
     buf_pool->no_flush[i] = os_event_create();
   }
 
+  buf_pool->run_lru = os_event_create();
+  os_event_set(buf_pool->run_lru);
+
   buf_pool->watch = (buf_page_t *)ut::zalloc_withkey(
       UT_NEW_THIS_FILE_PSI_KEY, sizeof(*buf_pool->watch) * BUF_POOL_WATCH_SIZE);
   for (i = 0; i < BUF_POOL_WATCH_SIZE; i++) {
@@ -1449,6 +1453,8 @@ static void buf_pool_free_instance(buf_pool_t *buf_pool) {
     os_event_destroy(buf_pool->no_flush[i]);
   }
 
+  os_event_destroy(buf_pool->run_lru);
+
   ut::free(buf_pool->chunks);
   mutex_exit(&buf_pool->chunks_mutex);
   mutex_free(&buf_pool->chunks_mutex);
@@ -6020,27 +6026,28 @@ static void buf_pool_invalidate_instance(buf_pool_t *buf_pool) {
 
   ut_ad(!mutex_own(&buf_pool->LRU_list_mutex));
 
+  os_event_reset(buf_pool->run_lru);
+  auto guard = create_scope_guard([&]() { os_event_set(buf_pool->run_lru); });
+
   mutex_enter(&buf_pool->flush_state_mutex);
 
   for (i = BUF_FLUSH_LRU; i < BUF_FLUSH_N_TYPES; i++) {
-    /* As this function is called during startup and
-    during redo application phase during recovery, InnoDB
-    is single threaded (apart from IO helper threads) at
-    this stage. No new write batch can be in initialization
-    stage at this point. */
-    ut_ad(buf_pool->init_flush[i] == false);
+    /* Although this function is called during startup and
+    during redo application phase during recovery, Percona InnoDB
+    might be running several LRU manager threads at this stage.
+    Hence, a new write batch can be in initialization stage at this point. */
 
-    /* However, it is possible that a write batch that has
-    been posted earlier is still not complete. For buffer
-    pool invalidation to proceed we must ensure there is NO
+    /* For buffer pool invalidation to proceed we must ensure there is NO
     write activity happening. */
-    if (buf_pool->n_flush[i] > 0) {
+    if (buf_pool->n_flush[i] > 0 || buf_pool->init_flush[i]) {
       buf_flush_t type = static_cast<buf_flush_t>(i);
 
       mutex_exit(&buf_pool->flush_state_mutex);
       buf_flush_wait_batch_end(buf_pool, type);
       mutex_enter(&buf_pool->flush_state_mutex);
     }
+
+    ut_ad(buf_pool->init_flush[i] == false);
   }
 
   mutex_exit(&buf_pool->flush_state_mutex);

storage/innobase/buf/buf0flu.cc

@@ -3748,6 +3748,8 @@ static void buf_lru_manager_thread(size_t buf_pool_instance) {
          srv_shutdown_state.load() == SRV_SHUTDOWN_CLEANUP) {
     ut_d(buf_flush_page_cleaner_disabled_loop());
 
+    os_event_wait(buf_pool->run_lru);
+
     buf_lru_manager_sleep_if_needed(next_loop_time);
 
     buf_lru_manager_adapt_sleep_time(buf_pool, lru_n_flushed, lru_sleep_time);

storage/innobase/include/buf0buf.h

@@ -2314,6 +2314,12 @@ struct buf_pool_t {
   running. Protected by flush_state_mutex. */
   os_event_t no_flush[BUF_FLUSH_N_TYPES];
 
+  /* This event is always set at startup, so LRU threads do not wait for this
+  event. Before invalidating bufferpool, this event is reset, so the next LRU
+  batch flushing will wait for the event. Bufferpool invalidation needs LRU
+  flushing to be stopped. */
+  os_event_t run_lru;
+
   /** A red-black tree is used exclusively during recovery to speed up
   insertions in the flush_list. This tree contains blocks in order of
   oldest_modification LSN and is kept in sync with the flush_list.  Each