test: enable gofail in release e2e test.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

.github/workflows/e2e.yaml

@@ -25,7 +25,8 @@ jobs:
         echo "${TARGET}"
         case "${TARGET}" in
           linux-amd64-e2e)
-            CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' make test-e2e-release
+            make install-gofail
+            CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' FAILPOINTS='true' make test-e2e-release
             ;;
           linux-386-e2e)
             GOARCH=386 CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' make test-e2e

Makefile

@@ -565,3 +565,9 @@ pull-docker-functional:
 	$(info GO_VERSION: $(GO_VERSION))
 	$(info ETCD_VERSION: $(ETCD_VERSION))
 	docker pull gcr.io/etcd-development/etcd-functional:go$(GO_VERSION)
+
+# Failpoints
+GOFAIL_VERSION = $(shell cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail)
+.PHONY: install-gofail
+install-gofail:
+	go install go.etcd.io/gofail@${GOFAIL_VERSION}

build.sh

@@ -20,11 +20,27 @@ GOARCH=${GOARCH:-$(go env GOARCH)}
 GO_LDFLAGS=(${GO_LDFLAGS:-} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
 GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS:-}" "GOOS=${GOOS}" "GOARCH=${GOARCH}")
 
+GOFAIL_VERSION=$(cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail)
 # enable/disable failpoints
 toggle_failpoints() {
   mode="$1"
   if command -v gofail >/dev/null 2>&1; then
-    run gofail "$mode" server/etcdserver/ server/mvcc/backend/ server/wal/
+    run gofail "$mode" server/etcdserver/ server/mvcc/ server/wal/ server/mvcc/backend/
+    if [[ "$mode" == "enable" ]]; then
+      go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ./server && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../etcdutl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../etcdctl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../tests && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+      cd ../
+    else
+      go mod tidy
+      cd ./server && go mod tidy
+      cd ../etcdutl && go mod tidy
+      cd ../etcdctl && go mod tidy
+      cd ../tests && go mod tidy
+      cd ../
+    fi
   elif [[ "$mode" != "disable" ]]; then
     log_error "FAILPOINTS set but gofail not found"
     exit 1

tools/mod/go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/mgechev/revive v1.0.2
 	github.com/mikefarah/yq/v3 v3.0.0-20201125113350-f42728eef735
 	github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f // indirect
+	go.etcd.io/gofail v0.1.0
 	go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116
 	google.golang.org/genproto v0.0.0-20201008135153-289734e2e40c // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect

tools/mod/tools.go

@@ -33,6 +33,7 @@ import (
 	_ "github.com/mdempsky/unconvert"
 	_ "github.com/mgechev/revive"
 	_ "github.com/mikefarah/yq/v3"
+	_ "go.etcd.io/gofail"
 	_ "go.etcd.io/protodoc"
 	_ "honnef.co/go/tools/cmd/staticcheck"
 	_ "mvdan.cc/unparam"