ci: streamline CI job

.github/workflows/golangci-lint.yaml

@@ -1,4 +1,4 @@
-name: golangci-lint
+name: Run Go linter
 on:
   push:
     branches:
@@ -11,5 +11,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: golangci-lint
-        run: |
-          make go-lint-all
+        run: make go-lint

.github/workflows/release-wip.yaml

@@ -0,0 +1,132 @@
+name: Release WIP
+
+on:
+  push:
+    branches:
+      - joao/ci-overhaul
+  workflow_dispatch:
+
+jobs:
+  permission_check:
+    runs-on: ubuntu-latest
+    outputs:
+      can-write: ${{ steps.check.outputs.can-write }}
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+    steps:
+    - id: check
+      run: |
+        # If the AWS_ACCESS_KEY_ID secret is MIA we can't run tests
+        if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then
+            echo "can-write=false" >> $GITHUB_OUTPUT
+        else
+            echo "can-write=true" >> $GITHUB_OUTPUT
+        fi
+
+  discover:
+    needs: [permission_check]
+    if: needs.permission_check.outputs.can-write == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    outputs:
+      apps: ${{ steps.apps.outputs.matrix }}
+      regions: ${{ steps.regions.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Find apps
+      id: apps
+      run: |
+        echo "matrix=$(ls apps/ | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+
+    - name: Setup AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4.0.2
+      with:
+        role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+        aws-region: us-west-2
+
+    - name: AWS Info
+      run: aws sts get-caller-identity
+
+    - name: Fetch available AWS regions
+      id: fetch-regions
+      run: |
+        regions=$(aws ec2 describe-regions --query "Regions[].RegionName" --output text | tr '\t' '\n' | jq -R -s -c 'split("\n")[:-1]')
+        echo "Regions: $regions"
+        echo "regions_json=$regions" >> "$GITHUB_ENV"
+    
+    - name: Set Matrix for aws-release job
+      id: regions 
+      run: echo "matrix=${regions_json}" >> "$GITHUB_OUTPUT"
+
+  github-release:
+    needs: [permission_check]
+    runs-on: ubuntu-latest
+    if: >
+      (needs.permission_check.outputs.can-write == 'true' && github.event_name == 'push') 
+    outputs:
+      version: ${{ steps.release-version.outputs.VERSION }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+
+    - name: github release (beta)
+      if: github.event_name == 'push'
+      id: prerelease
+      uses: ahmadnassri/action-semantic-release@v2
+      with:
+        config: ${{ github.workspace }}/.releaserc.json
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: github release (stable)
+      if: github.event_name == 'workflow_dispatch'
+      id: fullrelease
+      uses: ahmadnassri/action-semantic-release@v2
+      with:
+        config: ${{ github.workspace }}/.releaserc-release.json
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Set version for aws-release job
+      id: release-version
+      run: |
+        echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT"
+      env:
+        VERSION: ${{ (steps.prerelease.outputs.release-version != '') && steps.prerelease.outputs.release-version || steps.fullrelease.outputs.release-version }}
+
+  build:
+    needs: [discover, github-release]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        app: ${{fromJson(needs.discover.outputs.apps)}}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Validate SAM app
+      run: make sam-validate
+      env:
+        APP: ${{ matrix.app }}
+
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: 'go.mod'
+
+    - name: Build SAM app
+      run: make sam-build
+      env:
+        APP: ${{ matrix.app }}
+        VERSION: ${{ needs.github-release.outputs.VERSION }}
+
+    - name: Archive build directory
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.app }}
+        path: |
+          ${{ github.workspace }}/.aws-sam/build/${{ matrix.app }}
+
+

.github/workflows/tests-integration.yaml

@@ -17,6 +17,9 @@ on:
   schedule:
     - cron:  '0 0 * * 1' # Monday at 00:00 UTC
 
+env:
+  SAM_CLI_TELEMETRY: 0
+
 jobs:
   permission_check:
     runs-on: ubuntu-latest
@@ -34,27 +37,11 @@ jobs:
             echo "can-write=true" >> $GITHUB_OUTPUT
         fi
 
-  prepare_matrix:
+  provision:
+    runs-on: ubuntu-latest
     needs: [permission_check]
     if: needs.permission_check.outputs.can-write == 'true'
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.find_hcl_files.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v4
-
-    - name: Setup the test matrix
-      id: find_hcl_files
-      run: |
-        cd integration && \
-        echo "matrix=$(ls tests/*.hcl | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
-
-    - uses: actions/checkout@v4
-
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: 'go.mod'
-
     - name: DCE Provision
       uses: observeinc/github-action-dce@1.0.1
       with:
@@ -63,71 +50,120 @@ jobs:
         budget-amount: ${{ vars.BUDGET_AMOUNT }}
         budget-currency: 'USD'
         expiry: '30m'
-        email: 'colin.hutchinson+gha@observeinc.com'
+        email: 'joao+gha@observeinc.com'
+
+    - name: Setup tmate session
+      uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      with:
+        limit-access-to-actor: true
 
     - name: Create S3 Bucket for Artifacts
       run: |
-        if ! aws s3api head-bucket --bucket "${{ github.run_id }}-$AWS_REGION" 2>/dev/null; then
-          aws s3 mb s3://"${{ github.run_id }}-$AWS_REGION" --region us-west-2
+        if ! aws s3api head-bucket --bucket "${{ env.S3_BUCKET_PREFIX }}-${{ env.AWS_REGION }}" 2>/dev/null; then
+          aws s3 mb s3://"${{ env.S3_BUCKET_PREFIX }}-${{ env.AWS_REGION }}" --region $AWS_REGION
         fi
       env:
         AWS_REGION: us-west-2
+        S3_BUCKET_PREFIX: ${{ github.event.repository.name }}-${{ github.run_id }}
 
-    - name: Package SAM Applications
-      run: make sam-package-all
-      env:
-        AWS_REGION: us-west-2
-        S3_BUCKET_PREFIX: ${{ github.run_id }}
+  discover:
+    needs: [permission_check]
+    if: needs.permission_check.outputs.can-write == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+      apps: ${{ steps.apps.outputs.matrix }}
+      tests: ${{ steps.tests.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v4
 
-    - name: Setup tmate session
-      uses: mxschmitt/action-tmate@v3
-      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
-      with:
-        limit-access-to-actor: true
+    - name: Find apps
+      id: apps
+      run: |
+        echo "matrix=$(ls apps/ | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
 
-    - name: Archive SAM directory
-      uses: actions/upload-artifact@v4
-      with:
-        name: repo-and-sam-build
-        path: |
-          ${{ github.workspace }}/.aws-sam/
+    - name: Find tests
+      id: tests
+      run: |
+        cd integration && \
+        echo "matrix=$(ls tests/*.hcl | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
 
-  test-integration:
+  build:
+    needs: [discover, provision]
     runs-on: ubuntu-latest
-    needs: [permission_check, prepare_matrix]
-    if: needs.permission_check.outputs.can-write == 'true'
     strategy:
       matrix:
-        testfile: ${{fromJson(needs.prepare_matrix.outputs.matrix)}}
+        app: ${{fromJson(needs.discover.outputs.apps)}}
     steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Validate SAM app
+      run: make sam-validate
+      env:
+        APP: ${{ matrix.app }}
+
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: 'go.mod'
+
+    - name: Build SAM app
+      run: make sam-build
+      env:
+        APP: ${{ matrix.app }}
+
     - name: DCE Use
       id: dce_setup
       uses: observeinc/github-action-dce@1.0.1
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+    - name: Package SAM app
+      run: make sam-package
+      env:
+        APP: ${{ matrix.app }}
+        S3_BUCKET_PREFIX: ${{ github.event.repository.name }}-${{ github.run_id }}
+        AWS_REGION: us-west-2
+
+    - name: Archive build directory
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.app }}
+        # we only need the packaged yaml, since other artifacts are already in S3
+        path: |
+          ${{ github.workspace }}/.aws-sam/build/${{ matrix.app }}/*.yaml
+
+  test:
+    runs-on: ubuntu-latest
+    needs: [discover, build]
+    strategy:
+      matrix:
+        testfile: ${{fromJson(needs.discover.outputs.tests)}}
+    steps:
     - name: checkout
       uses: actions/checkout@v4
 
-    - name: Download SAM directory
+    - name: Download build directory
       uses: actions/download-artifact@v4
       with:
-        name: repo-and-sam-build
-        path: ${{ github.workspace }}/.aws-sam/
+        path: .aws-sam/build/
 
-    - uses: actions/setup-go@v5
+    - name: DCE Use
+      id: dce_setup
+      uses: observeinc/github-action-dce@1.0.1
       with:
-        go-version-file: 'go.mod'
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
     - name: Integration test for ${{ matrix.testfile }}
-      run: S3_BUCKET_PREFIX=${S3_BUCKET_PREFIX} TEST_ARGS='-filter=${{ matrix.testfile }} -verbose' make integration-test
+      run: TEST_ARGS='-filter=${{ matrix.testfile }} -verbose' make integration-test
       env:
         AWS_REGION: us-west-2
-        S3_BUCKET_PREFIX: ${{ github.run_id }}
 
   cleanup:
-    needs: [permission_check, test-integration]
+    needs: [permission_check, test]
     runs-on: ubuntu-latest
     if: always()
     steps:

.github/workflows/tests-unit.yaml

@@ -12,15 +12,16 @@ jobs:
   test:
     strategy:
       matrix:
-        go: [ 1.22.x ]
         platform: [ ubuntu-latest ]
     runs-on: ${{ matrix.platform }}
     steps:
-    - name: Install Go
-      uses: actions/setup-go@v5
-      with:
-        go-version: ${{ matrix.go }}
     - name: Checkout code
       uses: actions/checkout@v4
+
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: 'go.mod'
+
     - name: Test
       run: make go-test