chore: run bash shell in strict mode

According to Github Actions doc [1], linux platform will use `bash -e {0}` mode to run script. But we also need `pipefail` and `nounset` in CI. This patch is aimed to add strict mode setting before any run. And remove the `| tee` and egrep commands since we don't need this workaround to show error. REF: 1: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell ISSUE: etcd-io#15514 Signed-off-by: Wei Fu <fuweid89@gmail.com>
fuweid · Mar 20, 2023 · 7837008 · 7837008
1 parent 6d01ab6
commit 7837008
Show file tree

Hide file tree

Showing 63 changed files with 192 additions and 76 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -26,6 +26,8 @@ jobs:
       - env:
           TARGET: ${{ matrix.target }}
         run: |
+          set -euo pipefail
+
           echo "${TARGET}"
           case "${TARGET}" in
             linux-amd64)

diff --git a/.github/workflows/contrib.yaml b/.github/workflows/contrib.yaml
@@ -9,4 +9,7 @@ jobs:
     - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
       with:
         go-version: "1.19.7"
-    - run: make -C contrib/mixin tools test
+    - run: |
+      set -euo pipefail
+
+      make -C contrib/mixin tools test
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
@@ -17,6 +17,8 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         mkdir "${TARGET}"
         case "${TARGET}" in
           linux-amd64-coverage)

diff --git a/.github/workflows/e2e-arm64.yaml b/.github/workflows/e2e-arm64.yaml
@@ -23,11 +23,12 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         echo "${TARGET}"
         case "${TARGET}" in
           linux-arm64-e2e)
-            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           *)
             echo "Failed to find target"

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -19,15 +19,15 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         echo "${TARGET}"
         case "${TARGET}" in
           linux-amd64-e2e)
-            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           linux-386-e2e)
-            GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           *)
             echo "Failed to find target"

diff --git a/.github/workflows/fuzzing.yaml b/.github/workflows/fuzzing.yaml
@@ -13,7 +13,10 @@ jobs:
     - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
       with:
         go-version: "1.19.7"
-    - run: GOARCH=amd64 CPU=4 make fuzz
+    - run: |
+      set -euo pipefail
+
+      GOARCH=amd64 CPU=4 make fuzz
     - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
       if: failure()
       with:

diff --git a/.github/workflows/govuln.yaml b/.github/workflows/govuln.yaml
@@ -10,4 +10,7 @@ jobs:
         with:
           go-version: "1.19.7"
       - run: date
-      - run: go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./...
+      - run: |
+        set -euo pipefail
+
+        go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./...
diff --git a/.github/workflows/grpcproxy.yaml b/.github/workflows/grpcproxy.yaml
@@ -19,15 +19,15 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         echo "${TARGET}"
         case "${TARGET}" in
           linux-amd64-grpcproxy-integration)
-            PASSES='build grpcproxy_integration'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build grpcproxy_integration'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           linux-amd64-grpcproxy-e2e)
-            PASSES='build grpcproxy_e2e'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build grpcproxy_e2e'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           *)
             echo "Failed to find target"

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -10,6 +10,8 @@ jobs:
       with:
         go-version: "1.19.7"
     - run: |
+        set -euo pipefail
+
         git config --global user.email "github-action@etcd.io"
         git config --global user.name "Github Action"
         gpg --batch --gen-key <<EOF

diff --git a/.github/workflows/robustness-template.yaml b/.github/workflows/robustness-template.yaml
@@ -29,6 +29,8 @@ jobs:
       env:
         GITHUB_REF: ${{ inputs.ref }}
       run: |
+        set -euo pipefail
+
         case "${GITHUB_REF}" in
           release-3.5)
             make build-failpoints-release-3.5
@@ -45,6 +47,8 @@ jobs:
           esac
     - name: test-robustness
       run: |
+        set -euo pipefail
+
         # Use --failfast to avoid overriding report generated by failed test
         EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestRobustness' RESULTS_DIR=/tmp/results make test-robustness
     - uses: actions/upload-artifact@v2

diff --git a/.github/workflows/static-analysis.yaml b/.github/workflows/static-analysis.yaml
@@ -18,5 +18,11 @@ jobs:
         with:
           version: '3.14.0'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make verify
-      - run: make fix
+      - run: |
+        set -euo pipefail
+
+        make verify
+      - run: |
+        set -euo pipefail
+
+        make fix
diff --git a/.github/workflows/tests-arm64.yaml b/.github/workflows/tests-arm64.yaml
@@ -25,6 +25,8 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         mkdir "${TARGET}"
         export JUNIT_REPORT_DIR=$(realpath ${TARGET})
         case "${TARGET}" in

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -22,6 +22,8 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         mkdir "${TARGET}"
         export JUNIT_REPORT_DIR=$(realpath ${TARGET})
         case "${TARGET}" in

diff --git a/hack/benchmark/bench.sh b/hack/benchmark/bench.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+
+set -euo pipefail
 
 leader=http://localhost:2379
 # assume three servers

diff --git a/pkg/proxy/fixtures/gencerts.sh b/pkg/proxy/fixtures/gencerts.sh
@@ -1,4 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
+set -euo pipefail
 
 if ! [[ "$0" =~ "./gencerts.sh" ]]; then
   echo "must be run from 'fixtures'"

diff --git a/scripts/build-binary.sh b/scripts/build-binary.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 source ./scripts/test_lib.sh
 

diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 VERSION" >&2

diff --git a/scripts/build-release.sh b/scripts/build-release.sh
@@ -3,7 +3,7 @@
 # Build all release binaries and images to directory ./release.
 # Run from repository root.
 #
-set -e
+set -euo pipefail
 
 source ./scripts/test_lib.sh
 

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -3,6 +3,8 @@
 # This scripts build the etcd binaries
 # To build the tools, run `build_tools.sh`
 
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh
 

diff --git a/scripts/build_lib.sh b/scripts/build_lib.sh
@@ -1,18 +1,25 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 
 GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
 VERSION_SYMBOL="${ROOT_MODULE}/api/v3/version.GitSHA"
 
+# use go env if noset
+GOOS=${GOOS:-$(go env GOOS)}
+GOARCH=${GOARCH:-$(go env GOARCH)}
+GO_BUILD_FLAGS=${GO_BUILD_FLAGS:-}
+
 # Set GO_LDFLAGS="-s" for building without symbols for debugging.
 # shellcheck disable=SC2206
-GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
+GO_LDFLAGS=(${GO_LDFLAGS:-} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
 GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}")
 
 etcd_build() {
   out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi
 
   run rm -f "${out}/etcd"
   (
@@ -62,7 +69,7 @@ etcd_build() {
 
 tools_build() {
   out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi
   tools_path="tools/benchmark
     tools/etcd-dump-db
     tools/etcd-dump-logs

diff --git a/scripts/build_tools.sh b/scripts/build_tools.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh
 

diff --git a/scripts/codecov_upload.sh b/scripts/codecov_upload.sh
@@ -3,7 +3,7 @@
 # Script used to collect and upload test coverage (mostly by travis).
 # Usage ./test_coverage_upload.sh [log_file]
 
-set -o pipefail
+set -euo pipefail
 
 LOG_FILE=${1:-test-coverage.log}
 

diff --git a/scripts/fix.sh b/scripts/fix.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 # Top level problems with modules can lead to test_lib being not functional
 go mod tidy

diff --git a/scripts/fuzzing.sh b/scripts/fuzzing.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
-set -e
+
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 
 GO_CMD="go"

diff --git a/scripts/genproto.sh b/scripts/genproto.sh
@@ -3,7 +3,8 @@
 # Generate all etcd protobuf bindings.
 # Run from repository root directory named etcd.
 #
-set -e
+set -euo pipefail
+
 shopt -s globstar
 
 if ! [[ "$0" =~ scripts/genproto.sh ]]; then

diff --git a/scripts/install-marker.sh b/scripts/install-marker.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 ARCH=$1
 

diff --git a/scripts/measure-test-flakiness.sh b/scripts/measure-test-flakiness.sh
@@ -1,7 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-set -e
-set -o pipefail
+set -euo pipefail
 
 if [[ -z ${GITHUB_TOKEN} ]]
 then

diff --git a/scripts/release_mod.sh b/scripts/release_mod.sh
@@ -10,7 +10,9 @@
 #
 # % DRY_RUN=false REMOTE_REPO="origin" ./scripts/release_mod.sh push_mod_tags
 
-set -e
+set -o pipefail
+set -o errexit
+set -o nounset
 
 source ./scripts/test_lib.sh
 

diff --git a/scripts/test.sh b/scripts/test.sh
@@ -36,6 +36,7 @@ set -e
 # Consider command as failed when any component of the pipe fails:
 # https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
 set -o pipefail
+set -o nounset
 
 # The test script is not supposed to make any changes to the files
 # e.g. add/update missing dependencies. Such divergences should be 
@@ -46,6 +47,8 @@ export ETCD_VERIFY=all
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh
 
+OUTPUT_FILE=${OUTPUT_FILE:-""}
+
 if [ -n "${OUTPUT_FILE}" ]; then
   log_callout "Dumping output to: ${OUTPUT_FILE}"
   exec > >(tee -a "${OUTPUT_FILE}") 2>&1
@@ -55,12 +58,12 @@ PASSES=${PASSES:-"gofmt bom dep build unit"}
 PKG=${PKG:-}
 SHELLCHECK_VERSION=${SHELLCHECK_VERSION:-"v0.8.0"}
 
-if [ -z "$GOARCH" ]; then
+if [ -z "$GOARCH:-" ]; then
   GOARCH=$(go env GOARCH);
 fi
 
 # determine whether target supports race detection
-if [ -z "${RACE}" ] ; then
+if [ -z "${RACE:-}" ] ; then
   if [ "$GOARCH" == "amd64" ]; then
     RACE="--race"
   else
@@ -72,14 +75,14 @@ fi
 
 # This options make sense for cases where SUT (System Under Test) is compiled by test.
 COMMON_TEST_FLAGS=("${RACE}")
-if [[ -n "${CPU}" ]]; then
+if [[ -n "${CPU:-}" ]]; then
   COMMON_TEST_FLAGS+=("--cpu=${CPU}")
 fi 
 
 log_callout "Running with ${COMMON_TEST_FLAGS[*]}"
 
 RUN_ARG=()
-if [ -n "${TESTCASE}" ]; then
+if [ -n "${TESTCASE:-}" ]; then
   RUN_ARG=("-run=${TESTCASE}")
 fi
 
@@ -566,7 +569,7 @@ function release_pass {
   rm -f ./bin/etcd-last-release
   # to grab latest patch release; bump this up for every minor release
   UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.5.*" | head -1 | cut -d- -f1)
-  if [ -n "$MANUAL_VER" ]; then
+  if [ -n "$MANUAL_VER:-" ]; then
     # in case, we need to test against different version
     UPGRADE_VER=$MANUAL_VER
   fi

diff --git a/scripts/test_lib.sh b/scripts/test_lib.sh
@@ -232,7 +232,7 @@ function produce_junit_xmlreport {
 
   # Ensure that gotestsum is run without cross-compiling
   run_go_tool gotest.tools/gotestsum --junitfile "${junit_xml_filename}" --raw-command cat "${junit_filename_prefix}"*.stdout || exit 1
-  if [ "${VERBOSE}" != "1" ]; then
+  if [ "${VERBOSE:-}" != "1" ]; then
     rm "${junit_filename_prefix}"*.stdout
   fi
 
@@ -290,7 +290,7 @@ function go_test {
 
   junit_filename_prefix=$(junitFilenamePrefix)
 
-  if [ "${VERBOSE}" == "1" ]; then
+  if [ "${VERBOSE:-}" == "1" ]; then
     goTestFlags="-v"
   fi