chore: introduce strict bash mode for scripts,.github

REF: etcd-io#15514 Signed-off-by: Wei Fu <fuweid89@gmail.com>
fuweid · Mar 22, 2023 · 1fcb782 · 1fcb782
1 parent 6d01ab6
commit 1fcb782
Show file tree

Hide file tree

Showing 29 changed files with 137 additions and 69 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -26,6 +26,8 @@ jobs:
       - env:
           TARGET: ${{ matrix.target }}
         run: |
+          set -euo pipefail
+
           echo "${TARGET}"
           case "${TARGET}" in
             linux-amd64)

diff --git a/.github/workflows/contrib.yaml b/.github/workflows/contrib.yaml
@@ -9,4 +9,7 @@ jobs:
     - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
       with:
         go-version: "1.19.7"
-    - run: make -C contrib/mixin tools test
+    - run: |
+        set -euo pipefail
+
+        make -C contrib/mixin tools test
diff --git a/.github/workflows/e2e-arm64.yaml b/.github/workflows/e2e-arm64.yaml
@@ -23,11 +23,12 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         echo "${TARGET}"
         case "${TARGET}" in
           linux-arm64-e2e)
-            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           *)
             echo "Failed to find target"

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -19,15 +19,15 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         echo "${TARGET}"
         case "${TARGET}" in
           linux-amd64-e2e)
-            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           linux-386-e2e)
-            GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           *)
             echo "Failed to find target"

diff --git a/.github/workflows/fuzzing.yaml b/.github/workflows/fuzzing.yaml
@@ -13,7 +13,10 @@ jobs:
     - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
       with:
         go-version: "1.19.7"
-    - run: GOARCH=amd64 CPU=4 make fuzz
+    - run: |
+        set -euo pipefail
+
+        GOARCH=amd64 CPU=4 make fuzz
     - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
       if: failure()
       with:

diff --git a/.github/workflows/govuln.yaml b/.github/workflows/govuln.yaml
@@ -10,4 +10,7 @@ jobs:
         with:
           go-version: "1.19.7"
       - run: date
-      - run: go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./...
+      - run: |
+          set -euo pipefail
+
+          go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./...
diff --git a/.github/workflows/grpcproxy.yaml b/.github/workflows/grpcproxy.yaml
@@ -19,15 +19,15 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         echo "${TARGET}"
         case "${TARGET}" in
           linux-amd64-grpcproxy-integration)
-            PASSES='build grpcproxy_integration'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build grpcproxy_integration'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           linux-amd64-grpcproxy-e2e)
-            PASSES='build grpcproxy_e2e'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build grpcproxy_e2e'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh
             ;;
           *)
             echo "Failed to find target"

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -10,6 +10,8 @@ jobs:
       with:
         go-version: "1.19.7"
     - run: |
+        set -euo pipefail
+
         git config --global user.email "github-action@etcd.io"
         git config --global user.name "Github Action"
         gpg --batch --gen-key <<EOF

diff --git a/.github/workflows/robustness-template.yaml b/.github/workflows/robustness-template.yaml
@@ -29,6 +29,8 @@ jobs:
       env:
         GITHUB_REF: ${{ inputs.ref }}
       run: |
+        set -euo pipefail
+
         case "${GITHUB_REF}" in
           release-3.5)
             make build-failpoints-release-3.5
@@ -45,6 +47,8 @@ jobs:
           esac
     - name: test-robustness
       run: |
+        set -euo pipefail
+
         # Use --failfast to avoid overriding report generated by failed test
         EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestRobustness' RESULTS_DIR=/tmp/results make test-robustness
     - uses: actions/upload-artifact@v2

diff --git a/.github/workflows/static-analysis.yaml b/.github/workflows/static-analysis.yaml
@@ -18,5 +18,11 @@ jobs:
         with:
           version: '3.14.0'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make verify
-      - run: make fix
+      - run: |
+          set -euo pipefail
+
+          make verify
+      - run: |
+          set -euo pipefail
+
+          make fix
diff --git a/.github/workflows/tests-arm64.yaml b/.github/workflows/tests-arm64.yaml
@@ -25,6 +25,8 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         mkdir "${TARGET}"
         export JUNIT_REPORT_DIR=$(realpath ${TARGET})
         case "${TARGET}" in

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -22,6 +22,8 @@ jobs:
     - env:
         TARGET: ${{ matrix.target }}
       run: |
+        set -euo pipefail
+
         mkdir "${TARGET}"
         export JUNIT_REPORT_DIR=$(realpath ${TARGET})
         case "${TARGET}" in

diff --git a/scripts/build-binary.sh b/scripts/build-binary.sh
@@ -1,19 +1,17 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 source ./scripts/test_lib.sh
 
-VER=$1
+VER=${1:-}
 REPOSITORY="${REPOSITORY:-git@github.com:etcd-io/etcd.git}"
 
-if [ -z "$1" ]; then
+if [ -z "$VER" ]; then
   echo "Usage: ${0} VERSION" >> /dev/stderr
   exit 255
 fi
 
-set -u
-
 function setup_env {
   local ver=${1}
   local proj=${2}
@@ -60,7 +58,7 @@ function main {
   cd release
   setup_env "${VER}" "${proj}"
 
-  tarcmd=tar
+  local tarcmd=tar
   if [[ $(go env GOOS) == "darwin" ]]; then
       echo "Please use linux machine for release builds."
     exit 1

diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh
@@ -1,17 +1,23 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 VERSION" >&2
   exit 1
 fi
 
+VERSION=${1}
+if [ -z "$VERSION" ]; then
+  echo "Usage: ${0} VERSION" >&2
+  exit 1
+fi
+
 ARCH=$(go env GOARCH)
-VERSION="${1}-${ARCH}"
+VERSION="${VERSION}-${ARCH}"
 DOCKERFILE="Dockerfile-release.${ARCH}"
 
-if [ -z "${BINARYDIR}" ]; then
+if [ -z "${BINARYDIR:-}" ]; then
   RELEASE="etcd-${1}"-$(go env GOOS)-$(go env GOARCH)
   BINARYDIR="${RELEASE}"
   TARFILE="${RELEASE}.tar.gz"
@@ -34,7 +40,7 @@ cp "${BINARYDIR}"/etcd "${BINARYDIR}"/etcdctl "${BINARYDIR}"/etcdutl "${IMAGEDIR
 
 cat ./"${DOCKERFILE}" > "${IMAGEDIR}"/Dockerfile
 
-if [ -z "$TAG" ]; then
+if [ -z "${TAG:-}" ]; then
     docker build -t "gcr.io/etcd-development/etcd:${VERSION}" "${IMAGEDIR}"
     docker build -t "quay.io/coreos/etcd:${VERSION}" "${IMAGEDIR}"
 else

diff --git a/scripts/build-release.sh b/scripts/build-release.sh
@@ -3,11 +3,11 @@
 # Build all release binaries and images to directory ./release.
 # Run from repository root.
 #
-set -e
+set -euo pipefail
 
 source ./scripts/test_lib.sh
 
-VERSION=$1
+VERSION=${1:-}
 if [ -z "${VERSION}" ]; then
   echo "Usage: ${0} VERSION" >> /dev/stderr
   exit 255

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -3,6 +3,8 @@
 # This scripts build the etcd binaries
 # To build the tools, run `build_tools.sh`
 
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh
 

diff --git a/scripts/build_lib.sh b/scripts/build_lib.sh
@@ -1,18 +1,26 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 
 GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
 VERSION_SYMBOL="${ROOT_MODULE}/api/v3/version.GitSHA"
 
+# use go env if noset
+GOOS=${GOOS:-$(go env GOOS)}
+GOARCH=${GOARCH:-$(go env GOARCH)}
+
+GO_BUILD_FLAGS=${GO_BUILD_FLAGS:-}
+
 # Set GO_LDFLAGS="-s" for building without symbols for debugging.
 # shellcheck disable=SC2206
-GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
+GO_LDFLAGS=(${GO_LDFLAGS:-} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
 GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}")
 
 etcd_build() {
   out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi
 
   run rm -f "${out}/etcd"
   (
@@ -62,7 +70,7 @@ etcd_build() {
 
 tools_build() {
   out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi
   tools_path="tools/benchmark
     tools/etcd-dump-db
     tools/etcd-dump-logs

diff --git a/scripts/build_tools.sh b/scripts/build_tools.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh
 

diff --git a/scripts/fix.sh b/scripts/fix.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 # Top level problems with modules can lead to test_lib being not functional
 go mod tidy

diff --git a/scripts/fuzzing.sh b/scripts/fuzzing.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
-set -e
+
+set -euo pipefail
+
 source ./scripts/test_lib.sh
 
 GO_CMD="go"

diff --git a/scripts/genproto.sh b/scripts/genproto.sh
@@ -3,7 +3,8 @@
 # Generate all etcd protobuf bindings.
 # Run from repository root directory named etcd.
 #
-set -e
+set -euo pipefail
+
 shopt -s globstar
 
 if ! [[ "$0" =~ scripts/genproto.sh ]]; then
@@ -98,7 +99,7 @@ log_callout -e "\\nRunning swagger ..."
 run_go_tool github.com/hexfusion/schwag -input=Documentation/dev-guide/apispec/swagger/rpc.swagger.json
 
 
-if [ "$1" != "--skip-protodoc" ]; then
+if [ "${1:-}" != "--skip-protodoc" ]; then
   log_callout "protodoc is auto-generating grpc API reference documentation..."
 
   # API reference

diff --git a/scripts/install-marker.sh b/scripts/install-marker.sh
@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
-ARCH=$1
+ARCH=${1:-}
 
-if [ -z "$1" ]; then
+if [ -z "$ARCH" ]; then
     echo "Usage: ${0} [amd64 or darwin], defaulting to 'amd64'" >> /dev/stderr
     ARCH=amd64
 fi

diff --git a/scripts/measure-test-flakiness.sh b/scripts/measure-test-flakiness.sh
@@ -1,9 +1,8 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-set -e
-set -o pipefail
+set -euo pipefail
 
-if [[ -z ${GITHUB_TOKEN} ]]
+if [[ -z ${GITHUB_TOKEN:-} ]]
 then
     echo "Please set the \$GITHUB_TOKEN environment variable for the script to work"
     exit 1

diff --git a/scripts/release_mod.sh b/scripts/release_mod.sh
@@ -10,7 +10,7 @@
 #
 # % DRY_RUN=false REMOTE_REPO="origin" ./scripts/release_mod.sh push_mod_tags
 
-set -e
+set -euo pipefail
 
 source ./scripts/test_lib.sh
 
@@ -55,7 +55,7 @@ function mod_tidy_fix {
 function update_versions_cmd() {
   assert_no_git_modifications || return 2
 
-  if [ -z "${TARGET_VERSION}" ]; then
+  if [ -z "${TARGET_VERSION:-}" ]; then
     log_error "TARGET_VERSION environment variable not set. Set it to e.g. v3.5.10-alpha.0"
     return 2
   fi
@@ -89,7 +89,7 @@ function get_gpg_key {
 function push_mod_tags_cmd {
   assert_no_git_modifications || return 2
 
-  if [ -z "${REMOTE_REPO}" ]; then
+  if [ -z "${REMOTE_REPO:-}" ]; then
     log_error "REMOTE_REPO environment variable not set"
     return 2
   fi