tests: Reproduce issue 13766

.github/workflows/linearizability.yaml

@@ -13,7 +13,7 @@ jobs:
         make build
         mkdir -p /tmp/linearizability
         cat server/etcdserver/raft.fail.go
-        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
+        EXPECT_DEBUG=true TMPDIR=$HOME GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
     - uses: actions/upload-artifact@v2
       if: always()
       with:

tests/framework/e2e/etcd_process.go

@@ -32,7 +32,9 @@ import (
 )
 
 var (
-	EtcdServerReadyLines = []string{"ready to serve client requests"}
+	EtcdServerReadyLines                    = []string{EtcdServerReadyLog}
+	EtcdServerReadyLog                      = "ready to serve client requests"
+	EtcdServerInitialCorruptCheckFailureLog = "checkInitialHashKV failed"
 )
 
 // EtcdProcess is a process that serves etcd requests.
@@ -185,7 +187,16 @@ func (ep *EtcdServerProcess) Close() error {
 
 func (ep *EtcdServerProcess) waitReady(ctx context.Context) error {
 	defer close(ep.donec)
-	return WaitReadyExpectProc(ctx, ep.proc, EtcdServerReadyLines)
+	err := WaitReadyExpectProc(ctx, ep.proc, []string{EtcdServerReadyLog, EtcdServerInitialCorruptCheckFailureLog})
+	if err != nil {
+		return err
+	}
+	for _, log := range ep.proc.Lines() {
+		if strings.Contains(log, EtcdServerInitialCorruptCheckFailureLog) {
+			return fmt.Errorf("initial check failed: %s", log)
+		}
+	}
+	return nil
 }
 
 func (ep *EtcdServerProcess) Config() *EtcdServerProcessConfig { return ep.cfg }

tests/linearizability/linearizability_test.go

@@ -44,6 +44,7 @@ func TestLinearizability(t *testing.T) {
 		name      string
 		failpoint Failpoint
 		config    e2e.EtcdProcessClusterConfig
+		traffic   *trafficConfig
 	}{
 		{
 			name:      "ClusterOfSize1",
@@ -70,22 +71,38 @@ func TestLinearizability(t *testing.T) {
 				e2e.WithGoFailEnabled(true),
 			),
 		},
+		{
+			name:      "Issue13766",
+			failpoint: KillFailpoint,
+			config: e2e.EtcdProcessClusterConfig{
+				ClusterSize:   3,
+				SnapshotCount: 100,
+			},
+			traffic: &trafficConfig{
+				minimalQPS:  1000,
+				maximalQPS:  2000,
+				clientCount: 40,
+				traffic:     DefaultTraffic,
+			},
+		},
 	}
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
+			if tc.traffic == nil {
+				tc.traffic = &trafficConfig{
+					minimalQPS:  minimalQPS,
+					maximalQPS:  maximalQPS,
+					clientCount: 8,
+					traffic:     DefaultTraffic,
+				}
+			}
 			failpoint := FailpointConfig{
 				failpoint:           tc.failpoint,
 				count:               1,
 				retries:             3,
 				waitBetweenTriggers: waitBetweenFailpointTriggers,
 			}
-			traffic := trafficConfig{
-				minimalQPS:  minimalQPS,
-				maximalQPS:  maximalQPS,
-				clientCount: 8,
-				traffic:     DefaultTraffic,
-			}
-			testLinearizability(context.Background(), t, tc.config, failpoint, traffic)
+			testLinearizability(context.Background(), t, tc.config, failpoint, *tc.traffic)
 		})
 	}
 }

tests/linearizability/traffic.go

@@ -60,7 +60,7 @@ func (t readWriteSingleKey) Run(ctx context.Context, c *recordingClient, limiter
 }
 
 func (t readWriteSingleKey) Read(ctx context.Context, c *recordingClient, limiter *rate.Limiter) error {
-	getCtx, cancel := context.WithTimeout(ctx, 20*time.Millisecond)
+	getCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
 	err := c.Get(getCtx, t.key)
 	cancel()
 	if err == nil {