Merge pull request #178 from protocol/next

update to Go 1.17, update all repos with the updated workflows

.github/workflows/check-3rd-party.sh

@@ -7,7 +7,11 @@ status=0
 
 for line in `sed -ne 's/[[:space:]-]*uses:[[:space:]]*//p' $1 | sed -e 's/\s*#.*$//'`; do
   author=`echo $line | awk -F/ '{print $1}'`
-  if [[ $author == "actions" || $author == "protocol" ]]; then continue; fi
+  # We trust:
+  # - .: local workflows
+  # "actions": workflows authored by GitHub
+  # "protocol": workflows published in the protocol org
+  if [[ $author == "." || $author == "actions" || $author == "protocol" ]]; then continue; fi
   version=`echo $line | awk -F@ '{print $2}' | awk '{print $1}'`
   if ! [[ "$version" =~ ^[a-f0-9]{40}$ ]]; then
     status=1

.github/workflows/copy-workflow.yml

@@ -1,24 +1,34 @@
 # This workflow is triggered by the dispatch workflow.
 
-on:
-  repository_dispatch:
-    types: [ copy-workflow ]
-
 name: Deploy
 
+on:
+  workflow_dispatch:
+    inputs:
+      head_commit_url:
+        description: "github.event.head_commit.url of the dispatcher"
+        required: true
+      files:
+        description: "List of files to deploy"
+        required: true
+      targets:
+        description: "List of repositories to deploy to"
+        required: true
+
 jobs:
   copy:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cfg: ${{ github.event.client_payload.targets }}
+        cfg: ${{ fromJson(github.event.inputs.targets) }}
     env:
       TARGET_REPO_DIR: "target-repo"
       TEMPLATE_REPO_DIR: "template-repo"
       TEMPLATE_DIR: "templates"
       NEEDS_UPDATE: 0
       INITIAL_TEST_DEPLOYMENT: 0
+      GO_VERSION_BUMP: 0
       GITHUB_USER: "web3-bot"
       GITHUB_EMAIL: "web3-bot@users.noreply.github.com"
     name: ${{ matrix.cfg.target }}
@@ -38,10 +48,11 @@ jobs:
       with:
         # This should be the same Go version we use in the go-check workflow.
         # go mod tidy, go vet, staticcheck and gofmt might behave differently depending on the version.
-        go-version: "1.16.x"
+        stable: 'false'
+        go-version: "1.17.0-rc1"
     - name: git config
+      working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
-        cd $TARGET_REPO_DIR
         git config user.name ${{ env.GITHUB_USER }}
         git config user.email ${{ env.GITHUB_EMAIL }}
     - name: is initial test workflow deployment
@@ -50,58 +61,91 @@ jobs:
           echo "INITIAL_TEST_DEPLOYMENT=1" >> $GITHUB_ENV
         fi
     - name: remove the autorebase workflow # TODO: remove this step once all autorebase.yml files have been deleted
+      working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
-        cd $TARGET_REPO_DIR
         if [[ -f .github/workflows/autorebase.yml ]]; then
           git rm .github/workflows/autorebase.yml
           git commit -m "remove the autorebase workflow"
         fi
     - name: remove Travis (on initial deployment)
       if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
+      working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
-        cd $TARGET_REPO_DIR
         if [[ -f .travis.yml ]]; then
           git rm .travis.yml
           git commit -m "disable Travis"
         fi
     - name: remove CircleCI (on initial deployment)
       if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
+      working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
-        cd $TARGET_REPO_DIR
         if [[ -d .circleci ]]; then
           git rm -r .circleci
           git commit -m "disable CircleCI"
         fi
     - name: remove gx (on initial deployment)
       if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
+      working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
-        cd $TARGET_REPO_DIR
         if [[ -d .gx ]]; then
           git rm -r .gx
           git commit -m "remove .gx"
         fi
-    - name: go mod tidy (on initial deployment)
-      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
-      run: |
-        cd $TARGET_REPO_DIR
-        go mod edit -go 1.15
-        go mod tidy
-        if ! git diff --quiet; then
-          git add .
-          git commit -m "set Go version to 1.15 and run go mod tidy"
-        fi
-    - name: gofmt (on initial deployment)
-      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
+    - name: bump go.mod go version if needed
+      uses: protocol/multiple-go-modules@v1.2
+      with:
+        working-directory: ${{ env.TARGET_REPO_DIR }}
+        run: |
+          # We want our modules to support two Go versions at a time.
+          # As of August 2021, Go 1.17 is the latest stable.
+          # go.mod's Go version declares the language version being used.
+          # As such, it has to be the minimum of all Go versions supported.
+          # Bump this every six months, as new Go versions come out.
+          TARGET_VERSION=1.16
+
+          # Note that the "<" comparison doesn't understand semver,
+          # but it should be good enough for the foreseeable future.
+          CURRENT_VERSION=$(go list -m -f {{.GoVersion}})
+
+          if [[ $CURRENT_VERSION < $TARGET_VERSION ]]; then
+            echo "GO_VERSION_BUMP=1" >> $GITHUB_ENV
+
+            # Update the version in go.mod. This alone ensures there's a diff.
+            go mod edit -go $TARGET_VERSION
+
+            # In the future, "go fix" may make changes to Go code,
+            # such as to adapt to language changes or API deprecations.
+            # This is largely a no-op as of Go 1.17, and that's fine.
+            go fix
+            git add .
+
+            # We don't tidy, because the next step does that.
+            # Separate commits also help with reviews.
+            git commit -m "bump go.mod to Go $TARGET_VERSION and run go fix"
+          fi
+    - name: go mod tidy (on initial deployment and on new Go version)
+      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}}
+      uses: protocol/multiple-go-modules@v1.2
+      with:
+        working-directory: ${{ env.TARGET_REPO_DIR }}
+        run: |
+          go mod tidy
+          if ! git diff --quiet; then
+            git add .
+            git commit -m "run go mod tidy"
+          fi
+    - name: gofmt -s (on initial deployment and on new Go version)
+      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}}
+      working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
-        cd $TARGET_REPO_DIR
         gofmt -s -w .
         if ! git diff --quiet; then
           git add .
           git commit -m "run gofmt -s"
         fi
     - name: Add files
       run: |
-        for f in $(jq -r ".[]" <<< '${{ toJson(github.event.client_payload.files) }}'); do 
+        for f in $(jq -r ".[]" <<< ${{ toJson(github.event.inputs.files) }}); do
           echo -e "\nProcessing $f."
           # add DO NOT EDIT header
           tmp=$(mktemp)
@@ -131,16 +175,15 @@ jobs:
           popd > /dev/null
         done
     - name: Check if we need to create a PR
-      run: |
-        cd $TARGET_REPO_DIR
-        echo "NEEDS_UPDATE=$(git rev-list HEAD...origin/$(git rev-parse --abbrev-ref HEAD) --ignore-submodules --count)" >> $GITHUB_ENV
+      working-directory: ${{ env.TARGET_REPO_DIR }}
+      run: echo "NEEDS_UPDATE=$(git rev-list HEAD...origin/$(git rev-parse --abbrev-ref HEAD) --ignore-submodules --count)" >> $GITHUB_ENV
     - name: Create Pull Request
       if: ${{ env.NEEDS_UPDATE }}
-      uses: peter-evans/create-pull-request@89a67b1c69299ccc4dcc2253c3e91001adba29b5 # https://github.com/peter-evans/create-pull-request/pull/856
+      uses: peter-evans/create-pull-request@a3d36d978a8605d71fdb61c96bf0dfd9c9d8f897 # https://github.com/peter-evans/create-pull-request/pull/856
       with:
         path: ${{ env.TARGET_REPO_DIR }}
         title: "sync: update CI config files"
-        body: Syncing to commit ${{ github.event.client_payload.github_event.head_commit.url }}.
+        body: Syncing to commit ${{ github.event.inputs.head_commit_url }}.
         token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }}
         committer: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}>
         author: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}>

.github/workflows/dispatch.yml

@@ -33,11 +33,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # We end up with multiple "dispatch" jobs,
+        # one per TARGETS "key" chunk above with a "value" array.
+        # For each "dispatch" job, matrix.cfg.value is an array, like:
+        #
+        #   [{"target": "repo1", "target": "repo2"}]
+        #
+        # The triggered copy-workflow jobs use that final array as their matrix.
+        # As such, we'll end up with one copy-workflow parallel job per target.
         cfg: ${{ fromJson(needs.matrix.outputs.targets) }}
     name: Start copy workflow (batch ${{ matrix.cfg.key }})
     steps:
-      - uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4 # v1.1.3
+      - uses: benc-uk/workflow-dispatch@4c044c1613fabbe5250deadc65452d54c4ad4fc7 # v1.1.0
         with:
+          workflow: "Deploy" # "name" attribute of copy-workflow.yml
           token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }}
-          event-type: copy-workflow
-          client-payload: '{ "github_event": ${{ toJson(github.event) }}, "files": ${{ env.FILES }}, "targets": ${{ toJson(matrix.cfg.value) }} }'
+          inputs: '{ "head_commit_url": ${{ toJson(github.event.head_commit.url) }}, "files": ${{ toJson(env.FILES) }}, "targets": ${{ toJson(toJson(matrix.cfg.value)) }} }'

README.md

@@ -5,6 +5,43 @@ By storing them in a central place (here), and distributing them in an automated
 1. Consistency: Every participating repository uses the same workflow, ensuring that our code adheres to the same coding standards and is properly tested.
 2. Maintainability: Workflows change over time. We need to be able to make changes without manually updating dozens of repositories.
 
+## Customization
+
+### Additional Setup Steps
+
+Most repositories won't need any customization, and the workflows defined here will just work fine.
+Some repositories may require some pre-setup steps to be run before tests (or code checks) can be run. Setup steps for `go-test` are defined in `.github/actions/go-test-setup/action.yml`, and setup steps for `go-check` are defined in `.github/actions/go-check-setup/action.yml`, in the following format:
+
+```yml
+runs:
+  using: "composite"
+  steps:
+    - name: Step 1 
+      shell: bash
+      run: echo "do some initial setup"
+    - name: Step 2
+      shell: bash
+      run: echo "do some Linux-specific setup"
+      if: ${{ matrix.os == 'ubuntu' }}
+```
+
+These setup steps are run after the repository has been checked out and after Go has been installed, but before any tests or checks are run.
+
+### Configuration
+
+`go-check` contains an optional step that checks that running `go generate` doesn't change any files.
+This is useful to make sure that the generated code stays in sync.
+
+This check will be run in repositories that set `gogenerate` to `true` in `.github/workflows/go-check-config.json`:
+```json
+{
+  "gogenerate": true
+}
+```
+
+Note that depending on the code generators used, it might be necessary to [install those first](#additional-setup-steps).
+The generators must also be deterministic, to prevent CI from getting different results each time.
+
 ## Technical Details
 
 This repository currently defines two workflows for Go repositories:

templates/.github/workflows/automerge.yml

@@ -30,7 +30,9 @@ jobs:
   automerge:
     needs: automerge-check
     runs-on: ubuntu-latest
-    if: ${{ needs.automerge-check.outputs.status == 'true' }}
+    # The check for the user is redundant here, as this job depends on the automerge-check job,
+    # but it prevents this job from spinning up, just to be skipped shortly after.
+    if: github.event.pull_request.user.login == 'web3-bot' && needs.automerge-check.outputs.status == 'true'
     steps:
     - name: Wait on tests
       uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2

templates/.github/workflows/go-check.yml

@@ -5,17 +5,28 @@ jobs:
   unit:
     runs-on: ubuntu-latest
     name: All
+    env:
+      RUNGOGENERATE: false
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: recursive
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.16.x"
+          go-version: "1.17.x"
+      - name: Run repo-specific setup
+        uses: ./.github/actions/go-check-setup
+        if: hashFiles('./.github/actions/go-check-setup') != ''
+      - name: Read config
+        if: hashFiles('./.github/workflows/go-check-config.json') != ''
+        run: |
+          if jq -re .gogenerate ./.github/workflows/go-check-config.json; then
+            echo "RUNGOGENERATE=true" >> $GITHUB_ENV
+          fi
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@434f5f3816b358fe468fa83dcba62d794e7fe04b # 2021.1 (v0.2.0)
+        run: go install honnef.co/go/tools/cmd/staticcheck@df71e5d0e0ed317ebf43e6e59cf919430fa4b8f2 # 2021.1.1 (v0.2.1)
       - name: Check that go.mod is tidy
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: |
             go mod tidy
@@ -34,14 +45,27 @@ jobs:
           fi
       - name: go vet
         if: ${{ success() || failure() }} # run this step even if the previous one failed
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: go vet ./...
       - name: staticcheck
         if: ${{ success() || failure() }} # run this step even if the previous one failed
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: |
             set -o pipefail
             staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g'
+      - name: go generate
+        uses: protocol/multiple-go-modules@v1.2
+        if: (success() || failure()) && env.RUNGOGENERATE == 'true'
+        with:
+          run: |
+            git clean -fd # make sure there aren't untracked files / directories
+            go generate ./...
+            # check if go generate modified or added any files
+            if ! $(git add . && git diff-index HEAD --exit-code --quiet); then
+              echo "go generated caused changes to the repository:"
+              git status --short
+              exit 1
+            fi
 

templates/.github/workflows/go-test.yml

@@ -7,7 +7,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu", "windows", "macos" ]
-        go: [ "1.15.x", "1.16.x" ]
+        go: [ "1.16.x", "1.17.x" ]
+    env:
+      COVERAGES: ""
     runs-on: ${{ matrix.os }}-latest
     name: ${{ matrix.os}} (go ${{ matrix.go }})
     steps:
@@ -21,24 +23,30 @@ jobs:
         run: |
           go version
           go env
+      - name: Run repo-specific setup
+        uses: ./.github/actions/go-test-setup
+        if: hashFiles('./.github/actions/go-test-setup') != ''
       - name: Run tests
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
-          run: go test -v -coverprofile coverage.txt ./...
+          run: go test -v -coverprofile module-coverage.txt ./...
       - name: Run tests (32 bit)
         if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         env:
           GOARCH: 386
         with:
           run: go test -v ./...
       - name: Run tests with race detector
         if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: go test -v -race ./...
+      - name: Collect coverage files
+        shell: bash
+        run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 # v1.5.0
+        uses: codecov/codecov-action@51d810878be5422784e86451c0e7c14e5860ec47 # v2.0.2
         with:
-          file: coverage.txt
+          files: '${{ env.COVERAGES }}'
           env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}