From aedb6275f12384318a8c248f4863c464489c7194 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 25 Mar 2024 20:01:43 +0100 Subject: [PATCH] Draft reproduce issue with graceful snapshot Signed-off-by: Marek Siarkowicz --- tests/robustness/failpoint/failpoint.go | 36 ++++++++++++++----------- tests/robustness/failpoint/kill.go | 3 +++ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/tests/robustness/failpoint/failpoint.go b/tests/robustness/failpoint/failpoint.go index a8adde8e945..5ad714a58df 100644 --- a/tests/robustness/failpoint/failpoint.go +++ b/tests/robustness/failpoint/failpoint.go @@ -37,21 +37,22 @@ const ( var ( allFailpoints = []Failpoint{ - GracefulShutdown, ForcefulShutdown, - BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic, - DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic, - BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic, - BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic, - CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic, - CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork, - RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, - RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot, - BeforeApplyOneConfChangeSleep, - MemberReplace, - DropPeerNetwork, - RaftBeforeSaveSleep, - RaftAfterSaveSleep, - ApplyBeforeOpenSnapshot, + GracefulShutdown, + //ForcefulShutdown, + //BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic, + //DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic, + //BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic, + //BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic, + //CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic, + //CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork, + //RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, + //RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot, + //BeforeApplyOneConfChangeSleep, + //MemberReplace, + //DropPeerNetwork, + //RaftBeforeSaveSleep, + //RaftAfterSaveSleep, + //ApplyBeforeOpenSnapshot, } ) @@ -90,9 +91,12 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro time.Sleep(waitBetweenFailpointTriggers) lg.Info("Verifying cluster health before failpoint", zap.String("failpoint", failpoint.Name())) - if err = verifyClusterHealth(ctx, t, clus); err != nil { + vctx, cancel := context.WithTimeout(ctx, time.Second*5) + if err = verifyClusterHealth(vctx, t, clus); err != nil { + cancel() return fmt.Errorf("failed to verify cluster health before failpoint injection, err: %v", err) } + cancel() lg.Info("Triggering failpoint", zap.String("failpoint", failpoint.Name())) err = failpoint.Inject(ctx, t, lg, clus) diff --git a/tests/robustness/failpoint/kill.go b/tests/robustness/failpoint/kill.go index 9c7b4bac8a3..1926c94fdbd 100644 --- a/tests/robustness/failpoint/kill.go +++ b/tests/robustness/failpoint/kill.go @@ -20,6 +20,7 @@ import ( "math/rand" "strings" "testing" + "time" "go.uber.org/zap" @@ -84,6 +85,8 @@ func (f gracefulShutdownFailpoint) Inject(ctx context.Context, t *testing.T, lg return fmt.Errorf("failed to terminate the process within %s, err: %w", triggerTimeout, err) } } + ctx, cancel := context.WithTimeout(ctx, time.Second*2) + defer cancel() return member.Start(ctx) }