From 5a1cc775bbdd7394b244ad7400377061a0f6b837 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 31 Mar 2023 21:35:30 +0200 Subject: [PATCH 1/2] Give mysqlctld time to shutdown properly before restarting to account for delays in CI Signed-off-by: Rohit Nayak --- go/test/endtoend/cluster/mysqlctld_process.go | 5 +++++ go/test/endtoend/mysqlctld/mysqlctld_test.go | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/go/test/endtoend/cluster/mysqlctld_process.go b/go/test/endtoend/cluster/mysqlctld_process.go index 673870e62bb..c94856e1976 100644 --- a/go/test/endtoend/cluster/mysqlctld_process.go +++ b/go/test/endtoend/cluster/mysqlctld_process.go @@ -168,3 +168,8 @@ func (mysqlctld *MysqlctldProcess) IsHealthy() bool { _, err := mysql.Connect(context.Background(), ¶ms) return err == nil } + +// HasShutdown checks if the process has been set to nil +func (mysqlctld *MysqlctldProcess) HasShutdown() bool { + return mysqlctld.process == nil +} diff --git a/go/test/endtoend/mysqlctld/mysqlctld_test.go b/go/test/endtoend/mysqlctld/mysqlctld_test.go index dd6efb65c86..3da37323ffd 100644 --- a/go/test/endtoend/mysqlctld/mysqlctld_test.go +++ b/go/test/endtoend/mysqlctld/mysqlctld_test.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/stretchr/testify/require" @@ -134,10 +135,29 @@ func initCluster(shardNames []string, totalTabletsRequired int) error { return nil } +const defaultOperationTimeout = 60 * time.Second +const defeaultRetryDelay = 3 * time.Second + +func waitForMysqlctldShutdown(t *testing.T, tab *cluster.Vttablet) bool { + tmr := time.NewTimer(defaultOperationTimeout) + for { + if tab.MysqlctldProcess.HasShutdown() { + return true + } + select { + case <-tmr.C: + return false + default: + } + time.Sleep(defeaultRetryDelay) + } +} + func TestRestart(t *testing.T) { defer cluster.PanicHandler(t) err := primaryTablet.MysqlctldProcess.Stop() require.Nil(t, err) + require.Truef(t, waitForMysqlctldShutdown(t, primaryTablet), "Mysqlctld has not stopped after %s", defaultOperationTimeout) primaryTablet.MysqlctldProcess.CleanupFiles(primaryTablet.TabletUID) err = primaryTablet.MysqlctldProcess.Start() require.Nil(t, err) From 1c693d95d0c62f1eb3af353f80a8725fdd833c6a Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sun, 2 Apr 2023 22:23:25 +0200 Subject: [PATCH 2/2] Address review comments Signed-off-by: Rohit Nayak --- go/test/endtoend/mysqlctld/mysqlctld_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go/test/endtoend/mysqlctld/mysqlctld_test.go b/go/test/endtoend/mysqlctld/mysqlctld_test.go index 3da37323ffd..49ec6e6ba4d 100644 --- a/go/test/endtoend/mysqlctld/mysqlctld_test.go +++ b/go/test/endtoend/mysqlctld/mysqlctld_test.go @@ -135,11 +135,12 @@ func initCluster(shardNames []string, totalTabletsRequired int) error { return nil } -const defaultOperationTimeout = 60 * time.Second -const defeaultRetryDelay = 3 * time.Second +const defaultOperationTimeout = 300 * time.Second +const defaultRetryDelay = 3 * time.Second func waitForMysqlctldShutdown(t *testing.T, tab *cluster.Vttablet) bool { tmr := time.NewTimer(defaultOperationTimeout) + defer tmr.Stop() for { if tab.MysqlctldProcess.HasShutdown() { return true @@ -149,7 +150,7 @@ func waitForMysqlctldShutdown(t *testing.T, tab *cluster.Vttablet) bool { return false default: } - time.Sleep(defeaultRetryDelay) + time.Sleep(defaultRetryDelay) } }