Workflow

.github/workflows/integration.yaml

@@ -0,0 +1,102 @@
+name: Run Integration Tests
+
+on:
+  workflow_call:
+    inputs:
+      release_version:
+        type: string
+        description: 'Release version'
+        required: true
+      s3_bucket_prefix:
+        type: string
+        description: 'Bucket prefix for SAM assets'
+        required: true
+      aws_region:
+        type: string
+        description: 'AWS region to run tests in'
+        default: 'us-west-2'
+  workflow_dispatch:
+    inputs:
+      release_version:
+        type: string
+        description: 'Release version'
+        required: true
+      s3_bucket_prefix:
+        type: string
+        description: 'Bucket prefix for SAM assets'
+        required: true
+      aws_region:
+        type: string
+        description: 'AWS region to run tests in'
+        default: 'us-west-2'
+
+env:
+  AWS_REGION: "${{ inputs.aws_region }}"
+  SAM_CLI_TELEMETRY: 0
+
+jobs:
+  provision:
+    name: Provision DCE for tests
+    runs-on: ubuntu-latest
+    outputs:
+      tests: ${{ steps.find_hcl_files.outputs.tests }}
+    steps:
+    - name: DCE Provision
+      uses: observeinc/github-action-dce@1.0.1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        budget-amount: ${{ vars.BUDGET_AMOUNT }}
+        budget-currency: 'USD'
+        expiry: '30m'
+        email: 'joao+gha@observeinc.com'
+
+    - name: checkout
+      uses: actions/checkout@v4
+
+    - name: Setup test matrix
+      id: find_hcl_files
+      run: |
+        echo "tests=$(ls integration/tests | awk -F. '{print $1}' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+
+
+  tests:
+    name: Run integration test
+    runs-on: ubuntu-latest
+    needs: provision
+    strategy:
+      matrix:
+        testfile: ${{fromJson(needs.provision.outputs.tests)}}
+    steps:
+    - name: DCE Use
+      uses: observeinc/github-action-dce@1.0.1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+    - name: checkout
+      uses: actions/checkout@v4
+
+    - name: Pull SAM manifests
+      run: |
+        make sam-pull-${AWS_REGION}
+      env:
+        S3_BUCKET_PREFIX: "${{ inputs.s3_bucket_prefix }}"
+        RELEASE_VERSION: "${{ inputs.release_version }}"
+
+    - name: Run ${{ matrix.testfile }} integration test
+      run: TEST_ARGS='-verbose' make test-integration-${{ matrix.testfile }}
+
+  cleanup:
+    name: Cleanup
+    needs: tests
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+    - name: DCE Cleanup
+      if: needs.permission_check.outputs.can-write == 'true'
+      uses: observeinc/github-action-dce@1.0.1
+      with:
+        action-type: 'decommission'
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/push.yaml

@@ -0,0 +1,43 @@
+on:
+  #push:
+  #branches:
+  #- main
+  #pull_request:
+  workflow_call:
+    inputs:
+      run_integration_tests:
+        type: boolean
+        description: 'Run integration tests'
+        default: true
+      release_version:
+        type: string
+        description: 'Release version'
+        default: ''
+
+jobs:
+  tests:
+    name: Run tests
+    uses: ./.github/workflows/tests.yaml
+    secrets: inherit
+
+  upload:
+    name: Upload SAM assets
+    needs: tests
+    uses: ./.github/workflows/upload.yaml
+    permissions:
+      id-token: write
+    secrets: inherit
+    with:
+      s3_bucket_prefix: "observeinc-"
+      global: ${{ inputs.release_version != '' }}
+      release_version: ${{ inputs.release_version }}
+
+  integration:
+    name: Run integration tests 
+    if: ${{ github.event_name != 'workflow_call' || inputs.run_integration_tests }}
+    needs: upload
+    uses: ./.github/workflows/integration.yaml
+    secrets: inherit
+    with:
+      s3_bucket_prefix: "observeinc-"
+      release_version: ${{ needs.upload.outputs.release_version }}

.github/workflows/release.yaml

@@ -1,162 +1,78 @@
 name: Release
 
 on:
-  push:
-    branches:
-      - main
   workflow_dispatch:
-
-env:
-  S3_BUCKET_PREFIX: observeinc
-
+    inputs:
+      dry_run:
+        type: boolean
+        description: 'Dry run. Compute release version only'
+        default: false
+      run_integration_tests:
+        type: boolean
+        description: 'Run integration tests'
+        default: true
+
 jobs:
-  permission_check:
+  version:
+    name: Compute release version
     runs-on: ubuntu-latest
     outputs:
-      can-write: ${{ steps.check.outputs.can-write }}
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-    steps:
-    - id: check
-      run: |
-        # If the AWS_ACCESS_KEY_ID secret is MIA we can't run tests
-        if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then
-            echo "can-write=false" >> $GITHUB_OUTPUT
-        else
-            echo "can-write=true" >> $GITHUB_OUTPUT
-        fi
-
-  tests:
-    needs: permission_check
-    uses: ./.github/workflows/tests-integration.yaml
-    if: needs.permission_check.outputs.can-write == 'true'
-    secrets: inherit
-
-  fetch-regions:
-    runs-on: ubuntu-latest
-    needs: permission_check
-    if: needs.permission_check.outputs.can-write == 'true'
+      version: ${{ steps.dryrun.outputs.release-version }}
+      tag: ${{ steps.dryrun.outputs.release-channel == 'main' && 'latest' || steps.dryrun.outputs.release-channel }}
     permissions:
-      id-token: write
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-    - name: Setup AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4.0.2
-      with:
-        role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
-        aws-region: us-west-2
-
-    - name: AWS Info
-      run: aws sts get-caller-identity
-
-    - name: Fetch available AWS regions
-      id: fetch-regions
-      run: |
-        regions=$(aws ec2 describe-regions --query "Regions[].RegionName" --output text | tr '\t' '\n' | jq -R -s -c 'split("\n")[:-1]')
-        echo "Regions: $regions"
-        echo "regions_json=$regions" >> "$GITHUB_ENV"
-    
-    - name: Set Matrix for aws-release job
-      id: set-matrix
-      run: echo "matrix=${regions_json}" >> "$GITHUB_OUTPUT"
-
-  github-release:
-    needs: [tests, permission_check]
-    runs-on: ubuntu-latest
-    if: >
-      (needs.permission_check.outputs.can-write == 'true' && github.event_name == 'push') || 
-      (github.event_name == 'workflow_dispatch' && needs.tests.result == 'success')
-    outputs:
-      version: ${{ steps.release-version.outputs.VERSION }}
+      contents: write
     steps:
     - name: checkout
       uses: actions/checkout@v4
 
-    - name: github release (beta)
-      if: github.event_name == 'push'
-      id: prerelease
+    - name: dryrun
+      id: dryrun
       uses: ahmadnassri/action-semantic-release@v2
       with:
         config: ${{ github.workspace }}/.releaserc.json
+        dry: true
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: github release (stable)
-      if: github.event_name == 'workflow_dispatch'
-      id: fullrelease
-      uses: ahmadnassri/action-semantic-release@v2
-      with:
-        config: ${{ github.workspace }}/.releaserc-release.json
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Set version for aws-release job
-      id: release-version
-      run: |
-        echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT"
-      env:
-        VERSION: ${{ (steps.prerelease.outputs.release-version != '') && steps.prerelease.outputs.release-version || steps.fullrelease.outputs.release-version }}
-
-  static-upload:
-    needs: [permission_check, github-release, tests]
-    uses: ./.github/workflows/static-upload.yaml
-    permissions:
-      id-token: write
-    if: |
-      github.actor != 'dependabot[bot]' &&
-      needs.github-release.outputs.version != ''
+
+  push:
+    needs: version
+    if: ${{ !inputs.dry_run && needs.version.outputs.version != '' }}
+    uses: ./.github/workflows/push.yaml
     secrets: inherit
     with:
-      version: ${{ needs.github-release.outputs.VERSION }}
-
-  aws-release:
-    needs: [fetch-regions, github-release, tests]
+      run_integration_tests: ${{ inputs.run_integration_tests }}
+      release_version: ${{ needs.version.outputs.version }}
+
+  publish:
+    needs: [version, push]
     runs-on: ubuntu-latest
-    if: |
-      github.actor != 'dependabot[bot]' &&
-      needs.github-release.outputs.version != ''
-    strategy:
-      matrix:
-        region: ${{fromJson(needs.fetch-regions.outputs.matrix)}}
     permissions:
       contents: write
       id-token: write
-      pull-requests: write
-
     steps:
-    - name: checkout
+    - name: Checkout
       uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: 'go.mod'
+    - name: Tag release 
+      id: build
+      if: ${{ needs.version.outputs.tag != '' }}
+      run: |
+          make tag
+      env:
+        MAKEFLAGS: "-j 4 --output-sync=target"
+        S3_BUCKET_PREFIX: "observeinc-"
+        RELEASE_VERSION: "${{ needs.version.outputs.version }}"
+        TAG: ${{ needs.version.outputs.tag }}
 
-    - name: Setup AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4.0.2
+    - name: Cut release
+      id: release
+      uses: ahmadnassri/action-semantic-release@v2
       with:
-        role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
-        aws-region: us-west-2
-
-    - name: AWS Info
-      run: aws sts get-caller-identity
-
-    - name: Set release tag (beta)
-      if: github.event_name == 'push'
-      run: echo "TAG=beta" >> $GITHUB_ENV
-
-    - name: Set release tag (latest)
-      if: github.event_name == 'workflow_dispatch'
-      run: echo "TAG=latest" >> $GITHUB_ENV
-
-    - name: aws sam release
-      run: make release-all
+        config: ${{ github.workspace }}/.releaserc.json
       env:
-        # TAG is set implicitly
-        VERSION: ${{ needs.github-release.outputs.VERSION }}
-        AWS_REGION: ${{ matrix.region }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: delete pre-releases
+    - name: Delete older pre-releases
       uses: dev-drprasad/delete-older-releases@v0.3.4
       with:
         keep_latest: 0