Add integration test

Signed-off-by: Chun-Hung Tseng <henrybear327@gmail.com>
Co-authored-by: Benjamin Wang <benjamin.wang@broadcom.com>

.github/workflows/govuln.yaml

@@ -3,7 +3,7 @@ name: Go Vulnerability Checker
 on: [push, pull_request]
 permissions: read-all
 jobs:
-  test:
+  govuln:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5

.github/workflows/integration_tests.yml

@@ -0,0 +1,27 @@
+---
+name: Integration Tests
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+      - main
+  pull_request:
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+    steps:
+      - id: goversion
+        run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+        with:
+          go-version: ${{ steps.goversion.outputs.goversion }}
+      - uses: actions/checkout@v4
+      - name: tests
+        run: |
+          make run-all-integration-tests

Makefile

@@ -1,4 +1,5 @@
 SRCS=$(filter-out %_test.go, $(wildcard *.go */*.go))
+include integration/makefile.mk
 
 .PHONY: all
 all: gofail

integration/README.md

@@ -0,0 +1,5 @@
+# Integration Tests
+
+Each directory contains a scenario
+
+- sleep: the enabling and disabling of a failpoint won't be delayed due to an ongoing sleep() action

integration/makefile.mk

@@ -0,0 +1,24 @@
+GOFAIL_BINARY = $(shell pwd)/gofail
+
+# run all integration tests
+.PHONY: run-all-integration-tests
+run-all-integration-tests: run-integration-test-sleep
+
+# run integration test - sleep
+.PHONY: run-integration-test-sleep
+run-integration-test-sleep: build-gofail execute-integration-test-sleep remove-gofail
+
+.PHONY: execute-integration-test-sleep
+execute-integration-test-sleep:
+	@echo "GOFAIL_BINARY = ${GOFAIL_BINARY}"
+	cd ./integration/sleep/failpoints && ${GOFAIL_BINARY} enable && cd .. && go build -o integration_test_sleep . && cd failpoints && ${GOFAIL_BINARY} disable
+	cd ./integration/sleep && ./integration_test_sleep && rm integration_test_sleep
+
+# helper: build/remove gofail binaries
+.PHONY: build-gofail
+build-gofail:
+	make clean && make gofail
+
+.PHONY: remove-gofail
+remove-gofail:
+	make clean

integration/sleep/failpoints/failpoints.go

@@ -0,0 +1,40 @@
+package failpoints
+
+import (
+	"log"
+	"sync"
+	"time"
+)
+
+func Worker1(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	log.Println("worker1 in")
+	defer log.Println("worker1 out")
+
+	// gofail: var worker1Failpoint struct{}
+
+	time.Sleep(3 * time.Second)
+}
+
+func Worker2(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	log.Println("worker2 in")
+	defer log.Println("worker2 out")
+
+	// gofail: var worker2Failpoint struct{}
+
+	time.Sleep(3 * time.Second)
+}
+
+func Worker3(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	log.Println("worker3 in")
+	defer log.Println("worker3 out")
+
+	// gofail: var worker3Failpoint struct{}
+
+	time.Sleep(3 * time.Second)
+}

integration/sleep/go.mod

@@ -0,0 +1,9 @@
+module go.etcd.io/gofail/integration/sleep
+
+go 1.21
+
+toolchain go1.21.11
+
+require go.etcd.io/gofail v0.1.1-0.20240328162059-93c579a86c46
+
+replace go.etcd.io/gofail => ./../../

integration/sleep/go.sum

@@ -0,0 +1,8 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration/sleep/main.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"log"
+	"sync"
+	"time"
+
+	"go.etcd.io/gofail/integration/sleep/failpoints"
+	gofail "go.etcd.io/gofail/runtime"
+)
+
+func main() {
+	{
+		// expectation: this part of the code will take about 3s to execute, because all go routines will be executing concurrently
+		start := time.Now()
+
+		log.Println("Stage 1: Run 3 workers under normal logic")
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go failpoints.Worker1(&wg)
+
+		wg.Add(1)
+		go failpoints.Worker2(&wg)
+
+		wg.Add(1)
+		go failpoints.Worker3(&wg)
+
+		wg.Wait()
+
+		elapsed := time.Since(start)
+		if elapsed > (3*time.Second + 100*time.Millisecond) {
+			log.Fatalln("invalid execution time", elapsed)
+		}
+
+		log.Println("Stage 1: Done")
+	}
+
+	{
+		// expectation: this part of the code will take about 6s to execute only,
+		// because all go routines will be executing concurrently, with both the sleep
+		// from failpoint and the original sleep actions
+		//
+		// The gofail implementation up till commit 93c579a86c46 is executing the
+		// program sequentially, due to the failpoint action execution and enable/disable
+		// flows are under the same locking mechanism, only one of the actions can make
+		// progress at a given moment
+		log.Println("Stage 2: Run 3 workers under failpoint logic")
+
+		start := time.Now()
+
+		var wg sync.WaitGroup
+		gofail.Enable("worker1Failpoint", `sleep("3s")`)
+		wg.Add(1)
+		go failpoints.Worker1(&wg)
+		time.Sleep(10 * time.Millisecond)
+
+		gofail.Enable("worker2Failpoint", `sleep("3s")`)
+		wg.Add(1)
+		go failpoints.Worker2(&wg)
+		time.Sleep(10 * time.Millisecond)
+
+		gofail.Enable("worker3Failpoint", `sleep("3s")`)
+		wg.Add(1)
+		go failpoints.Worker3(&wg)
+		time.Sleep(10 * time.Millisecond)
+
+		// the failpoint can be disabled during failpoint execution
+		gofail.Disable("worker1Failpoint")
+		gofail.Disable("worker2Failpoint")
+		gofail.Disable("worker3Failpoint")
+
+		wg.Wait()
+
+		elapsed := time.Since(start)
+		if elapsed > (6*time.Second + 100*time.Millisecond) {
+			log.Fatalln("invalid execution time", elapsed)
+		}
+
+		log.Println("Stage 2: Done")
+	}
+}