feat(ci): retry individual e2e tests

Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com>

Makefile

@@ -386,9 +386,9 @@ test: test-tools-image
 .PHONY: test-local
 test-local:
 	if test "$(TEST_MODULE)" = ""; then \
-		./hack/test.sh -coverprofile=coverage.out `go list ./... | grep -v 'test/e2e'`; \
+		./hack/test.sh -failfast -coverprofile=coverage.out `go list ./... | grep -v 'test/e2e'`; \
 	else \
-		./hack/test.sh -coverprofile=coverage.out "$(TEST_MODULE)"; \
+		./hack/test.sh -failfast -coverprofile=coverage.out "$(TEST_MODULE)"; \
 	fi
 
 .PHONY: test-race
@@ -400,9 +400,9 @@ test-race: test-tools-image
 .PHONY: test-race-local
 test-race-local:
 	if test "$(TEST_MODULE)" = ""; then \
-		./hack/test.sh -race -coverprofile=coverage.out `go list ./... | grep -v 'test/e2e'`; \
+		./hack/test.sh -failfast -race -coverprofile=coverage.out `go list ./... | grep -v 'test/e2e'`; \
 	else \
-		./hack/test.sh -race -coverprofile=coverage.out "$(TEST_MODULE)"; \
+		./hack/test.sh -failfast -race -coverprofile=coverage.out "$(TEST_MODULE)"; \
 	fi
 
 # Run the E2E test suite. E2E test servers (see start-e2e target) must be
@@ -416,7 +416,7 @@ test-e2e:
 test-e2e-local: cli-local
 	# NO_PROXY ensures all tests don't go out through a proxy if one is configured on the test system
 	export GO111MODULE=off
-	ARGOCD_E2E_RECORD=${ARGOCD_E2E_RECORD} ARGOCD_GPG_ENABLED=true NO_PROXY=* ./hack/test.sh -timeout $(ARGOCD_E2E_TEST_TIMEOUT) -v ./test/e2e
+	DIST_DIR=${DIST_DIR} ARGOCD_E2E_RECORD=${ARGOCD_E2E_RECORD} ARGOCD_GPG_ENABLED=true NO_PROXY=* ./hack/test.sh -timeout $(ARGOCD_E2E_TEST_TIMEOUT) -v ./test/e2e
 
 # Spawns a shell in the test server container for debugging purposes
 debug-test-server: test-tools-image
@@ -556,7 +556,8 @@ install-tools-local: install-test-tools-local install-codegen-tools-local instal
 .PHONY: install-test-tools-local
 install-test-tools-local:
 	./hack/install.sh kustomize
-	./hack/install.sh helm-linux
+
+	./hack/install.sh gotestsum
 
 # Installs all tools required for running codegen (Linux packages)
 .PHONY: install-codegen-tools-local

hack/installers/install-gotestsum.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# Code from: https://github.com/argoproj/argo-rollouts/blob/f650a1fd0ba7beb2125e1598410515edd572776f/hack/installers/install-dev-tools.sh
+
+PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/../..; pwd)
+DIST_PATH="${PROJECT_ROOT}/dist"
+PATH="${DIST_PATH}:${PATH}"
+
+mkdir -p ${DIST_PATH}
+
+gotestsum_version=1.11.0
+
+OS=$(go env GOOS)
+ARCH=$(go env GOARCH)
+
+export TARGET_FILE=gotestsum_${gotestsum_version}_${OS}_${ARCH}.tar.gz
+temp_path="/tmp/${TARGET_FILE}"
+url=https://github.com/gotestyourself/gotestsum/releases/download/v${gotestsum_version}/gotestsum_${gotestsum_version}_${OS}_${ARCH}.tar.gz
+[ -e ${temp_path} ] || curl -sLf --retry 3 -o ${temp_path} ${url}
+
+mkdir -p /tmp/gotestsum-${gotestsum_version}
+tar -xvzf ${temp_path} -C /tmp/gotestsum-${gotestsum_version}
+cp /tmp/gotestsum-${gotestsum_version}/gotestsum ${DIST_PATH}/gotestsum
+chmod +x ${DIST_PATH}/gotestsum
+gotestsum --version

hack/test.sh

@@ -15,12 +15,4 @@ fi
 
 mkdir -p $TEST_RESULTS
 
-report() {
-	set -eux -o pipefail
-
-	go-junit-report < $TEST_RESULTS/test.out > $TEST_RESULTS/junit.xml
-}
-
-trap 'report' EXIT
-
-GODEBUG="tarinsecurepath=0,zipinsecurepath=0" go test $TEST_FLAGS -failfast $* 2>&1 | tee $TEST_RESULTS/test.out
+GODEBUG="tarinsecurepath=0,zipinsecurepath=0" ${DIST_DIR}/gotestsum --rerun-fails-report=rerunreport.txt --junitfile=$TEST_RESULTS/junit.xml --format=testname --packages="./test/e2e" --rerun-fails=5 -- $TEST_FLAGS $*