Merge pull request #17083 from siyuanfoundation/livez-bp-step3

[3.5] Backport e2e tests for livez/readyz.

.github/workflows/e2e.yaml

@@ -25,7 +25,8 @@ jobs:
         echo "${TARGET}"
         case "${TARGET}" in
           linux-amd64-e2e)
-            CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' make test-e2e-release
+            make install-gofail
+            CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' FAILPOINTS='true' make test-e2e-release
             ;;
           linux-386-e2e)
             GOARCH=386 CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' make test-e2e

Makefile

@@ -565,3 +565,9 @@ pull-docker-functional:
 	$(info GO_VERSION: $(GO_VERSION))
 	$(info ETCD_VERSION: $(ETCD_VERSION))
 	docker pull gcr.io/etcd-development/etcd-functional:go$(GO_VERSION)
+
+# Failpoints
+GOFAIL_VERSION = $(shell cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail)
+.PHONY: install-gofail
+install-gofail:
+	go install go.etcd.io/gofail@${GOFAIL_VERSION}

build.sh

@@ -20,11 +20,27 @@ GOARCH=${GOARCH:-$(go env GOARCH)}
 GO_LDFLAGS=(${GO_LDFLAGS:-} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
 GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS:-}" "GOOS=${GOOS}" "GOARCH=${GOARCH}")
 
+GOFAIL_VERSION=$(cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail)
 # enable/disable failpoints
 toggle_failpoints() {
   mode="$1"
   if command -v gofail >/dev/null 2>&1; then
-    run gofail "$mode" server/etcdserver/ server/mvcc/backend/ server/wal/
+    run gofail "$mode" server/etcdserver/ server/mvcc/ server/wal/ server/mvcc/backend/
+    if [[ "$mode" == "enable" ]]; then
+      go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ./server && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../etcdutl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../etcdctl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../tests && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../
+    else
+      go mod tidy
+      cd ./server && go mod tidy
+      cd ../etcdutl && go mod tidy
+      cd ../etcdctl && go mod tidy
+      cd ../tests && go mod tidy
+      cd ../
+    fi
   elif [[ "$mode" != "disable" ]]; then
     log_error "FAILPOINTS set but gofail not found"
     exit 1

pkg/expect/expect.go

@@ -203,3 +203,7 @@ func (ep *ExpectProcess) Lines() []string {
 	defer ep.mu.Unlock()
 	return ep.lines
 }
+
+func (ep *ExpectProcess) IsRunning() bool {
+	return ep.cmd != nil
+}

server/etcdserver/server.go

@@ -2152,6 +2152,7 @@ func (s *EtcdServer) apply(
 			zap.Stringer("type", e.Type))
 		switch e.Type {
 		case raftpb.EntryNormal:
+			// gofail: var beforeApplyOneEntryNormal struct{}
 			s.applyEntryNormal(&e)
 			s.setAppliedIndex(e.Index)
 			s.setTerm(e.Term)

server/mvcc/backend/batch_tx.go

@@ -307,6 +307,7 @@ func newBatchTxBuffered(backend *backend) *batchTxBuffered {
 func (t *batchTxBuffered) Unlock() {
 	if t.pending != 0 {
 		t.backend.readTx.Lock() // blocks txReadBuffer for writing.
+		// gofail: var beforeWritebackBuf struct{}
 		t.buf.writeback(&t.backend.readTx.buf)
 		t.backend.readTx.Unlock()
 		if t.pending >= t.backend.batchLimit {

tests/e2e/etcdctl.go

@@ -55,6 +55,15 @@ func (ctl *Etcdctl) Put(key, value string) error {
 	return e2e.SpawnWithExpectWithEnv(args, ctl.env(), "OK")
 }
 
+func (ctl *Etcdctl) PutWithAuth(key, value, username, password string) error {
+	if ctl.v2 {
+		panic("Unsupported method for v2")
+	}
+	args := ctl.cmdArgs()
+	args = append(args, "--user", fmt.Sprintf("%s:%s", username, password), "put", key, value)
+	return e2e.SpawnWithExpectWithEnv(args, ctl.env(), "OK")
+}
+
 func (ctl *Etcdctl) Set(key, value string) error {
 	if !ctl.v2 {
 		panic("Unsupported method for v3")
@@ -72,6 +81,32 @@ func (ctl *Etcdctl) Set(key, value string) error {
 	return nil
 }
 
+func (ctl *Etcdctl) AuthEnable() error {
+	args := ctl.cmdArgs("auth", "enable")
+	return e2e.SpawnWithExpectWithEnv(args, ctl.env(), "Authentication Enabled")
+}
+
+func (ctl *Etcdctl) UserGrantRole(user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) {
+	var resp clientv3.AuthUserGrantRoleResponse
+	err := ctl.spawnJsonCmd(&resp, "user", "grant-role", user, role)
+	return &resp, err
+}
+
+func (ctl *Etcdctl) UserAdd(name, password string) (*clientv3.AuthUserAddResponse, error) {
+	args := []string{"user", "add"}
+	if password == "" {
+		args = append(args, name)
+		args = append(args, "--no-password")
+	} else {
+		args = append(args, fmt.Sprintf("%s:%s", name, password))
+	}
+	args = append(args, "--interactive=false")
+
+	var resp clientv3.AuthUserAddResponse
+	err := ctl.spawnJsonCmd(&resp, args...)
+	return &resp, err
+}
+
 func (ctl *Etcdctl) AlarmList() (*clientv3.AlarmResponse, error) {
 	if ctl.v2 {
 		panic("Unsupported method for v2")