From 796e564684a910b04e4c756b081a3802a1b212cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Fri, 31 May 2024 20:21:13 -0700 Subject: [PATCH] ci: streamline CI job --- .github/workflows/golangci-lint.yaml | 5 +- .github/workflows/release-wip.yaml | 132 ++++++++++++++++++++++ .github/workflows/sam-validate.yaml | 20 ---- .github/workflows/tests-integration.yaml | 134 ++++++++++++++--------- .github/workflows/tests-unit.yaml | 11 +- .github/workflows/tests.yaml | 26 ----- DEVELOPER.md | 2 - Makefile | 82 ++++---------- integration/main.tf | 2 +- samconfig.yaml | 1 - 10 files changed, 248 insertions(+), 167 deletions(-) create mode 100644 .github/workflows/release-wip.yaml delete mode 100644 .github/workflows/sam-validate.yaml delete mode 100644 .github/workflows/tests.yaml diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 1342b390..3de9df4f 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -1,4 +1,4 @@ -name: golangci-lint +name: Run Go linter on: push: branches: @@ -11,5 +11,4 @@ jobs: steps: - uses: actions/checkout@v4 - name: golangci-lint - run: | - make go-lint-all + run: make go-lint diff --git a/.github/workflows/release-wip.yaml b/.github/workflows/release-wip.yaml new file mode 100644 index 00000000..22cccb97 --- /dev/null +++ b/.github/workflows/release-wip.yaml @@ -0,0 +1,132 @@ +name: Release WIP + +on: + push: + branches: + - joao/ci-overhaul + workflow_dispatch: + +jobs: + permission_check: + runs-on: ubuntu-latest + outputs: + can-write: ${{ steps.check.outputs.can-write }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + steps: + - id: check + run: | + # If the AWS_ACCESS_KEY_ID secret is MIA we can't run tests + if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then + echo "can-write=false" >> $GITHUB_OUTPUT + else + echo "can-write=true" >> $GITHUB_OUTPUT + fi + + discover: + needs: [permission_check] + if: needs.permission_check.outputs.can-write == 'true' + runs-on: ubuntu-latest + permissions: + id-token: write + outputs: + apps: ${{ steps.apps.outputs.matrix }} + regions: ${{ steps.regions.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + + - name: Find apps + id: apps + run: | + echo "matrix=$(ls apps/ | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + + - name: Setup AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-west-2 + + - name: AWS Info + run: aws sts get-caller-identity + + - name: Fetch available AWS regions + id: fetch-regions + run: | + regions=$(aws ec2 describe-regions --query "Regions[].RegionName" --output text | tr '\t' '\n' | jq -R -s -c 'split("\n")[:-1]') + echo "Regions: $regions" + echo "regions_json=$regions" >> "$GITHUB_ENV" + + - name: Set Matrix for aws-release job + id: regions + run: echo "matrix=${regions_json}" >> "$GITHUB_OUTPUT" + + github-release: + needs: [permission_check] + runs-on: ubuntu-latest + if: > + (needs.permission_check.outputs.can-write == 'true' && github.event_name == 'push') + outputs: + version: ${{ steps.release-version.outputs.VERSION }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: github release (beta) + if: github.event_name == 'push' + id: prerelease + uses: ahmadnassri/action-semantic-release@v2 + with: + config: ${{ github.workspace }}/.releaserc.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: github release (stable) + if: github.event_name == 'workflow_dispatch' + id: fullrelease + uses: ahmadnassri/action-semantic-release@v2 + with: + config: ${{ github.workspace }}/.releaserc-release.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set version for aws-release job + id: release-version + run: | + echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + env: + VERSION: ${{ (steps.prerelease.outputs.release-version != '') && steps.prerelease.outputs.release-version || steps.fullrelease.outputs.release-version }} + + build: + needs: [discover, github-release] + runs-on: ubuntu-latest + strategy: + matrix: + app: ${{fromJson(needs.discover.outputs.apps)}} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate SAM app + run: make sam-validate + env: + APP: ${{ matrix.app }} + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Build SAM app + run: make sam-build + env: + APP: ${{ matrix.app }} + VERSION: ${{ needs.github-release.outputs.VERSION }} + + - name: Archive build directory + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.app }} + path: | + ${{ github.workspace }}/.aws-sam/build/${{ matrix.app }} + + diff --git a/.github/workflows/sam-validate.yaml b/.github/workflows/sam-validate.yaml deleted file mode 100644 index da4c1e4b..00000000 --- a/.github/workflows/sam-validate.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Run Validate - -on: - push: - branches: - - main - pull_request: -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: aws-actions/setup-sam@v2 - with: - use-installer: true - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - run: pip install -r dev-requirements.txt - - run: make sam-validate-all diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index ff4bb758..61a839d7 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -17,6 +17,9 @@ on: schedule: - cron: '0 0 * * 1' # Monday at 00:00 UTC +env: + SAM_CLI_TELEMETRY: 0 + jobs: permission_check: runs-on: ubuntu-latest @@ -34,27 +37,11 @@ jobs: echo "can-write=true" >> $GITHUB_OUTPUT fi - prepare_matrix: + provision: + runs-on: ubuntu-latest needs: [permission_check] if: needs.permission_check.outputs.can-write == 'true' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.find_hcl_files.outputs.matrix }} steps: - - uses: actions/checkout@v4 - - - name: Setup the test matrix - id: find_hcl_files - run: | - cd integration && \ - echo "matrix=$(ls tests/*.hcl | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT - - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - - name: DCE Provision uses: observeinc/github-action-dce@1.0.1 with: @@ -63,43 +50,69 @@ jobs: budget-amount: ${{ vars.BUDGET_AMOUNT }} budget-currency: 'USD' expiry: '30m' - email: 'colin.hutchinson+gha@observeinc.com' + email: 'joao+gha@observeinc.com' + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + with: + limit-access-to-actor: true - name: Create S3 Bucket for Artifacts run: | - if ! aws s3api head-bucket --bucket "${{ github.run_id }}-$AWS_REGION" 2>/dev/null; then - aws s3 mb s3://"${{ github.run_id }}-$AWS_REGION" --region us-west-2 + if ! aws s3api head-bucket --bucket "${{ env.S3_BUCKET_PREFIX }}-${{ env.AWS_REGION }}" 2>/dev/null; then + aws s3 mb s3://"${{ env.S3_BUCKET_PREFIX }}-${{ env.AWS_REGION }}" --region $AWS_REGION fi env: AWS_REGION: us-west-2 + S3_BUCKET_PREFIX: ${{ github.event.repository.name }}-${{ github.run_id }} - - name: Package SAM Applications - run: make sam-package-all - env: - AWS_REGION: us-west-2 - S3_BUCKET_PREFIX: ${{ github.run_id }} + discover: + needs: [permission_check] + if: needs.permission_check.outputs.can-write == 'true' + runs-on: ubuntu-latest + outputs: + apps: ${{ steps.apps.outputs.matrix }} + tests: ${{ steps.tests.outputs.matrix }} + steps: + - uses: actions/checkout@v4 - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - with: - limit-access-to-actor: true + - name: Find apps + id: apps + run: | + echo "matrix=$(ls apps/ | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT - - name: Archive SAM directory - uses: actions/upload-artifact@v4 - with: - name: repo-and-sam-build - path: | - ${{ github.workspace }}/.aws-sam/ + - name: Find tests + id: tests + run: | + cd integration && \ + echo "matrix=$(ls tests/*.hcl | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT - test-integration: + build: + needs: [discover, provision] runs-on: ubuntu-latest - needs: [permission_check, prepare_matrix] - if: needs.permission_check.outputs.can-write == 'true' strategy: matrix: - testfile: ${{fromJson(needs.prepare_matrix.outputs.matrix)}} + app: ${{fromJson(needs.discover.outputs.apps)}} steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate SAM app + run: make sam-validate + env: + APP: ${{ matrix.app }} + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Build SAM app + run: make sam-build + env: + APP: ${{ matrix.app }} + - name: DCE Use id: dce_setup uses: observeinc/github-action-dce@1.0.1 @@ -107,27 +120,50 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Package SAM app + run: make sam-package + env: + APP: ${{ matrix.app }} + S3_BUCKET_PREFIX: ${{ github.event.repository.name }}-${{ github.run_id }} + AWS_REGION: us-west-2 + + - name: Archive build directory + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.app }} + # we only need the packaged yaml, since other artifacts are already in S3 + path: | + ${{ github.workspace }}/.aws-sam/build/${{ matrix.app }}/*.yaml + + test: + runs-on: ubuntu-latest + needs: [discover, build] + strategy: + matrix: + testfile: ${{fromJson(needs.discover.outputs.tests)}} + steps: - name: checkout uses: actions/checkout@v4 - - name: Download SAM directory + - name: Download build directory uses: actions/download-artifact@v4 with: - name: repo-and-sam-build - path: ${{ github.workspace }}/.aws-sam/ + path: .aws-sam/build/ - - uses: actions/setup-go@v5 + - name: DCE Use + id: dce_setup + uses: observeinc/github-action-dce@1.0.1 with: - go-version-file: 'go.mod' + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Integration test for ${{ matrix.testfile }} - run: S3_BUCKET_PREFIX=${S3_BUCKET_PREFIX} TEST_ARGS='-filter=${{ matrix.testfile }} -verbose' make integration-test + run: TEST_ARGS='-filter=${{ matrix.testfile }} -verbose' make integration-test env: AWS_REGION: us-west-2 - S3_BUCKET_PREFIX: ${{ github.run_id }} cleanup: - needs: [permission_check, test-integration] + needs: [permission_check, test] runs-on: ubuntu-latest if: always() steps: diff --git a/.github/workflows/tests-unit.yaml b/.github/workflows/tests-unit.yaml index 56b9f531..704658f4 100644 --- a/.github/workflows/tests-unit.yaml +++ b/.github/workflows/tests-unit.yaml @@ -12,15 +12,16 @@ jobs: test: strategy: matrix: - go: [ 1.22.x ] platform: [ ubuntu-latest ] runs-on: ${{ matrix.platform }} steps: - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go }} - name: Checkout code uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: Test run: make go-test diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml deleted file mode 100644 index 56b9f531..00000000 --- a/.github/workflows/tests.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Run Go tests - -on: - push: - tags: - - v* - branches: - - main - pull_request: - -jobs: - test: - strategy: - matrix: - go: [ 1.22.x ] - platform: [ ubuntu-latest ] - runs-on: ${{ matrix.platform }} - steps: - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go }} - - name: Checkout code - uses: actions/checkout@v4 - - name: Test - run: make go-test diff --git a/DEVELOPER.md b/DEVELOPER.md index 9aef6048..d3e2683b 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -34,8 +34,6 @@ Refer to `make help` for an authoritative list of Make targets - Sync: Sync your code changes to AWS rapidly (`make sam-sync APP=$APP`) - Validate: Check your SAM template for errors (`make sam-validate APP=$APP`) - Publish: Share your application via AWS Serverless Application Repository (`make sam-publish APP=$APP`) -- Multi-application Commands: Build, package, or publish all applications (`make sam-build-all, make sam-package-all, make sam-publish-all`) -- Multi-region Commands: Manage multi-region deployments (`make sam-package-all-regions`) ## Development Workflow diff --git a/Makefile b/Makefile index b63c6ad1..0eb2139f 100644 --- a/Makefile +++ b/Makefile @@ -19,23 +19,21 @@ endef SUBDIR = $(shell ls apps) -.PHONY: help go-lint go-lint-all go-test integration-test debug sam-validate sam-validate-all sam-build-all sam-build sam-publish sam-package-all sam-package release-all release sam-publish-all build-App build-Forwarder build-Subscriber clean-aws-sam +.PHONY: help go-lint go-test integration-test debug sam-validate sam-build sam-package sam-publish release build-App build-Forwarder build-Subscriber clean -clean-aws-sam: +clean: rm -rf $(SAM_BUILD_DIR) +## all: Build and package single app +all: sam-validate sam-build sam-package + ## help: Displays this help message listing all targets help: @echo "Usage: make [target]" @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /' -## go-lint: Runs Go linter for a specified directory, set with PACKAGE variable +## go-lint: Executes Go linter for all Go packages in the project go-lint: - $(call check_var,PACKAGE) - docker run --rm -v "`pwd`:/workspace:cached" -w "/workspace/$(PACKAGE)" golangci/golangci-lint:latest golangci-lint run - -## go-lint-all: Executes Go linter for all Go packages in the project -go-lint-all: docker run --rm -v "`pwd`:/workspace:cached" -w "/workspace/." golangci/golangci-lint:latest golangci-lint run ## go-test: Runs Go tests across all packages @@ -61,61 +59,33 @@ sam-validate: --config-file $(SAM_CONFIG_FILE) \ --config-env $(SAM_CONFIG_ENV) -## sam-validate-all: Validates all CloudFormation templates in the project -sam-validate-all: - @ for dir in $(SUBDIR); do \ - APP=$$dir $(MAKE) sam-validate || exit 1; \ - done - -## sam-build-all: Builds assets for all SAM applications across specified regions -sam-build-all: - @ for app in $(SUBDIR); do \ - for region in $(REGIONS); do \ - APP=$$app AWS_REGION=$$region $(MAKE) sam-build || exit 1; \ - done \ - done - ## sam-build: Builds assets for a specific SAM application, specified by APP variable -sam-build: +sam-build: sam-validate $(call check_var,APP) echo "VERSION=${VERSION}" > ${BUILD_MAKEFILE_ENV_VARS} sam build \ --template-file apps/$(APP)/template.yaml \ - --build-dir $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION) \ - --config-file $(SAM_CONFIG_FILE) \ - --config-env $(SAM_CONFIG_ENV) - -## sam-publish: Publishes a specific serverless repository application, after packaging -sam-publish: sam-package - sam publish \ - --template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION)/packaged.yaml \ - --region $(AWS_REGION) \ + --build-dir $(SAM_BUILD_DIR)/$(APP)/ \ --config-file $(SAM_CONFIG_FILE) \ --config-env $(SAM_CONFIG_ENV) -## sam-package-all: Packages and pushes all CloudFormation templates to S3 -sam-package-all: - @ for dir in $(SUBDIR); do \ - APP=$$dir $(MAKE) sam-package || exit 1; \ - done - ## sam-package: Packages a specific CloudFormation template and pushes assets to S3, specified by APP variable -sam-package: sam-build +sam-package: $(call check_var,APP) $(call check_var,VERSION) echo "Packaging for app: $(APP) in region: $(AWS_REGION)" ifeq ($(S3_BUCKET_PREFIX),) sam package \ - --template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION)/template.yaml \ - --output-template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION)/packaged.yaml \ + --template-file $(SAM_BUILD_DIR)/$(APP)/template.yaml \ + --output-template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION).yaml \ --region $(AWS_REGION) \ --resolve-s3 \ --config-file $(SAM_CONFIG_FILE) \ --config-env $(SAM_CONFIG_ENV) else sam package \ - --template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION)/template.yaml \ - --output-template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION)/packaged.yaml \ + --template-file $(SAM_BUILD_DIR)/$(APP)/template.yaml \ + --output-template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION).yaml \ --region $(AWS_REGION) \ --s3-bucket $(S3_BUCKET_PREFIX)-$(AWS_REGION) \ --s3-prefix apps/$(APP)/$(VERSION) \ @@ -123,14 +93,13 @@ else --config-env $(SAM_CONFIG_ENV) endif -## release-all: Releases all applications, ensuring S3_BUCKET_PREFIX is set -release-all: -ifeq ($(S3_BUCKET_PREFIX),) - $(error S3_BUCKET_PREFIX is empty. Cannot proceed with release-all.) -endif - for dir in $(SUBDIR); do \ - APP=$$dir $(MAKE) release || exit 1; \ - done +## sam-publish: Publishes a specific serverless repository application, after packaging +sam-publish: sam-package + sam publish \ + --template-file $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION).yaml \ + --region $(AWS_REGION) \ + --config-file $(SAM_CONFIG_FILE) \ + --config-env $(SAM_CONFIG_ENV) ## release: Packages, uploads, and sets ACL for a specific app, ensuring S3_BUCKET_PREFIX is set, specified by APP variable release: @@ -141,18 +110,12 @@ endif @echo "Resetting assets to be public readable" aws s3 cp --acl public-read --recursive s3://$(S3_BUCKET_PREFIX)-$(AWS_REGION)/apps/$(APP)/$(VERSION)/ s3://$(S3_BUCKET_PREFIX)-$(AWS_REGION)/apps/$(APP)/$(VERSION)/ @echo "Copying stack definition" - aws s3 cp --acl public-read $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION)/packaged.yaml s3://$(S3_BUCKET_PREFIX)-$(AWS_REGION)/apps/$(APP)/$(VERSION)/ + aws s3 cp --acl public-read $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION).yaml s3://$(S3_BUCKET_PREFIX)-$(AWS_REGION)/apps/$(APP)/$(VERSION)/packaged.yaml ifeq ($(TAG),) else - aws s3 cp --acl public-read $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION)/packaged.yaml s3://$(S3_BUCKET_PREFIX)-$(AWS_REGION)/apps/$(APP)/$(TAG)/ + aws s3 cp --acl public-read $(SAM_BUILD_DIR)/$(APP)/$(AWS_REGION).yaml s3://$(S3_BUCKET_PREFIX)-$(AWS_REGION)/apps/$(APP)/$(TAG)/packaged.yaml endif -## sam-publish-all: Publishes all serverless applications -sam-publish-all: - for dir in $(SUBDIR); do - APP=$$dir $(MAKE) sam-publish || exit 1; - done - build-App: $(call check_var,APP) $(call check_var,ARTIFACTS_DIR) @@ -181,4 +144,3 @@ static-upload: static-validate $(call check_var,S3_BUCKET_PREFIX) aws s3 sync static s3://$(S3_BUCKET_PREFIX)/ --acl public-read --metadata Version=$(VERSION) -.PHONY: help go-lint go-lint-all go-test sam-validate sam-validate-all sam-build sam-package sam-publish sam-package-all sam-publish-all build-App build-Forwarder diff --git a/integration/main.tf b/integration/main.tf index 853827c9..0b5167b7 100644 --- a/integration/main.tf +++ b/integration/main.tf @@ -2,7 +2,7 @@ data "aws_region" "current" {} resource "aws_cloudformation_stack" "this" { name = var.setup.stack_name - template_body = file("../.aws-sam/build/${var.app}/${data.aws_region.current.name}/packaged.yaml") + template_body = file("../.aws-sam/build/${var.app}/${data.aws_region.current.name}.yaml") parameters = var.parameters capabilities = var.capabilities iam_role_arn = var.install_policy_json == null ? null : aws_iam_role.this[0].arn diff --git a/samconfig.yaml b/samconfig.yaml index eb0dc8ca..8ff9e54f 100644 --- a/samconfig.yaml +++ b/samconfig.yaml @@ -3,7 +3,6 @@ version: 0.1 default: build: parameters: - debug: true cached: true parallel: true validate: