Reproduce etcd-io#18089

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
serathius · Jun 10, 2024 · 68120e9 · 68120e9
1 parent 8a0054f
commit 68120e9
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 31 deletions.
diff --git a/.github/workflows/robustness.yaml b/.github/workflows/robustness.yaml
@@ -7,19 +7,19 @@ jobs:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: main
-      count: 12
+      count: 300
       testTimeout: 30m
       artifactName: main
       runs-on: "['ubuntu-latest-8-cores']"
-      scenario: TestRobustness
+      scenario: TestRobustnessExploratory/Etcd/LowTraffic/ClusterOfSize1
       lazyfsEnabled: true
   main-arm64:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: main
-      count: 12
+      count: 300
       testTimeout: 30m
       artifactName: main-arm64
       runs-on: "['actuated-arm64-12cpu-8gb']"
-      scenario: TestRobustness
+      scenario: TestRobustnessExploratory/Etcd/LowTraffic/ClusterOfSize1
       lazyfsEnabled: false
diff --git a/client/v3/watch.go b/client/v3/watch.go
@@ -765,12 +765,19 @@ func (w *watchGRPCStream) serveWatchClient(wc pb.Watch_WatchClient) {
 	for {
 		resp, err := wc.Recv()
 		if err != nil {
+			fmt.Printf("Adding new error response to watch, err: %s \n", err)
 			select {
 			case w.errc <- err:
 			case <-w.donec:
 			}
 			return
 		}
+
+		if len(resp.Events) > 0 {
+			fmt.Printf("Adding new event response to watch, revs: %d...%d \n", resp.Events[0].Kv.ModRevision, resp.Events[len(resp.Events)-1].Kv.ModRevision)
+		} else {
+			fmt.Printf("Adding response watch: %+v\n", resp)
+		}
 		select {
 		case w.respc <- resp:
 		case <-w.donec:

diff --git a/tests/robustness/failpoint/failpoint.go b/tests/robustness/failpoint/failpoint.go
@@ -36,20 +36,7 @@ const (
 
 var (
 	allFailpoints = []Failpoint{
-		KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
-		DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
-		BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
-		BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
-		CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
-		CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
-		RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
-		RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
-		BeforeApplyOneConfChangeSleep,
-		MemberReplace,
-		DropPeerNetwork,
-		RaftBeforeSaveSleep,
-		RaftAfterSaveSleep,
-		ApplyBeforeOpenSnapshot,
+		RaftBeforeLeaderSendPanic,
 	}
 )
 

diff --git a/tests/robustness/traffic/etcd.go b/tests/robustness/traffic/etcd.go
@@ -34,18 +34,11 @@ var (
 		leaseTTL:     DefaultLeaseTTL,
 		largePutSize: 32769,
 		requests: []choiceWeight[etcdRequestType]{
-			{choice: Get, weight: 15},
-			{choice: List, weight: 15},
-			{choice: StaleGet, weight: 10},
-			{choice: StaleList, weight: 10},
-			{choice: Delete, weight: 5},
-			{choice: MultiOpTxn, weight: 5},
-			{choice: PutWithLease, weight: 5},
-			{choice: LeaseRevoke, weight: 5},
-			{choice: CompareAndSet, weight: 5},
-			{choice: Put, weight: 15},
-			{choice: LargePut, weight: 5},
-			{choice: Compact, weight: 5},
+			{choice: Get, weight: 25},
+			{choice: List, weight: 25},
+			{choice: Delete, weight: 20},
+			{choice: Put, weight: 20},
+			{choice: Compact, weight: 10},
 		},
 	}
 	EtcdPut = etcdTraffic{

diff --git a/tests/robustness/validate/watch.go b/tests/robustness/validate/watch.go
@@ -16,6 +16,7 @@ package validate
 
 import (
 	"errors"
+	"fmt"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -206,7 +207,8 @@ func validateReliable(lg *zap.Logger, replay *model.EtcdReplay, report report.Cl
 			}
 		}
 		if diff := cmp.Diff(wantEvents, gotEvents, cmpopts.IgnoreFields(model.PersistedEvent{}, "IsCreate")); diff != "" {
-			lg.Error("Broke watch guarantee", zap.String("guarantee", "reliable"), zap.Int("client", report.ClientID), zap.String("diff", diff))
+			lg.Error("Broke watch guarantee", zap.String("guarantee", "reliable"), zap.Int("client", report.ClientID))
+			fmt.Println(diff)
 			err = errBrokeReliable
 		}
 	}