diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml new file mode 100644 index 00000000..dff81519 --- /dev/null +++ b/.github/workflows/integration.yaml @@ -0,0 +1,102 @@ +name: Run Integration Tests + +on: + workflow_call: + inputs: + release_version: + type: string + description: 'Release version' + required: true + s3_bucket_prefix: + type: string + description: 'Bucket prefix for SAM assets' + required: true + aws_region: + type: string + description: 'AWS region to run tests in' + default: 'us-west-2' + workflow_dispatch: + inputs: + release_version: + type: string + description: 'Release version' + required: true + s3_bucket_prefix: + type: string + description: 'Bucket prefix for SAM assets' + required: true + aws_region: + type: string + description: 'AWS region to run tests in' + default: 'us-west-2' + +env: + AWS_REGION: "${{ inputs.aws_region }}" + SAM_CLI_TELEMETRY: 0 + +jobs: + provision: + name: Provision DCE for tests + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.find_hcl_files.outputs.tests }} + steps: + - name: DCE Provision + uses: observeinc/github-action-dce@1.0.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + budget-amount: ${{ vars.BUDGET_AMOUNT }} + budget-currency: 'USD' + expiry: '30m' + email: 'joao+gha@observeinc.com' + + - name: checkout + uses: actions/checkout@v4 + + - name: Setup test matrix + id: find_hcl_files + run: | + echo "tests=$(ls integration/tests | awk -F. '{print $1}' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + + + tests: + name: Run integration test + runs-on: ubuntu-latest + needs: provision + strategy: + matrix: + testfile: ${{fromJson(needs.provision.outputs.tests)}} + steps: + - name: DCE Use + uses: observeinc/github-action-dce@1.0.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: checkout + uses: actions/checkout@v4 + + - name: Pull SAM manifests + run: | + make sam-pull-${AWS_REGION} + env: + S3_BUCKET_PREFIX: "${{ inputs.s3_bucket_prefix }}" + RELEASE_VERSION: "${{ inputs.release_version }}" + + - name: Run ${{ matrix.testfile }} integration test + run: TEST_ARGS='-verbose' make test-integration-${{ matrix.testfile }} + + cleanup: + name: Cleanup + needs: tests + runs-on: ubuntu-latest + if: always() + steps: + - name: DCE Cleanup + if: needs.permission_check.outputs.can-write == 'true' + uses: observeinc/github-action-dce@1.0.1 + with: + action-type: 'decommission' + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index 4d890268..00000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Run linters -on: - push: - branches: - - main - pull_request: -jobs: - lint: - name: golangci-lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: sam-validate - run: | - make sam-validate - - name: golangci-lint - run: | - make go-lint diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 00000000..ee570e9e --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,33 @@ +name: Push + +on: + #push: + #branches: + #- main + #pull_request: + +jobs: + tests: + name: Run tests + uses: ./.github/workflows/tests.yaml + secrets: inherit + + upload: + name: Upload SAM assets + needs: tests + uses: ./.github/workflows/upload.yaml + permissions: + id-token: write + secrets: inherit + with: + s3_bucket_prefix: "observeinc-" + + integration: + name: Run integration tests + if: ${{ !inputs.skip_integration_tests }} + needs: upload + uses: ./.github/workflows/integration.yaml + secrets: inherit + with: + s3_bucket_prefix: "observeinc-" + release_version: ${{ needs.upload.outputs.release_version }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1a17be16..cc6cd1aa 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,137 +1,78 @@ name: Release on: - push: - branches: - - main workflow_dispatch: - -env: - S3_BUCKET_PREFIX: observeinc - + inputs: + dry_run: + type: boolean + description: 'Dry run. Compute release version only' + default: false + skip_integration_tests: + type: boolean + description: 'Skip integration tests' + default: false + jobs: - permission_check: + version: + name: Compute release version runs-on: ubuntu-latest outputs: - can-write: ${{ steps.check.outputs.can-write }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - steps: - - id: check - run: | - # If the AWS_ACCESS_KEY_ID secret is MIA we can't run tests - if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then - echo "can-write=false" >> $GITHUB_OUTPUT - else - echo "can-write=true" >> $GITHUB_OUTPUT - fi - - tests: - needs: permission_check - uses: ./.github/workflows/tests-integration.yaml - if: needs.permission_check.outputs.can-write == 'true' - secrets: inherit - - fetch-regions: - runs-on: ubuntu-latest - needs: permission_check - if: needs.permission_check.outputs.can-write == 'true' + version: ${{ steps.dryrun.outputs.release-version }} + tag: ${{ steps.dryrun.outputs.release-channel == 'main' && 'latest' || steps.dryrun.outputs.release-channel }} permissions: - id-token: write - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Setup AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: us-west-2 - - - name: AWS Info - run: aws sts get-caller-identity - - - name: Fetch available AWS regions - id: fetch-regions - run: | - regions=$(aws ec2 describe-regions --query "Regions[].RegionName" --output text | tr '\t' '\n' | jq -R -s -c 'split("\n")[:-1]') - echo "Regions: $regions" - echo "regions_json=$regions" >> "$GITHUB_ENV" - - - name: Set Matrix for aws-release job - id: set-matrix - run: echo "matrix=${regions_json}" >> "$GITHUB_OUTPUT" - - github-release: - needs: [tests, permission_check] - runs-on: ubuntu-latest - if: > - (needs.permission_check.outputs.can-write == 'true' && github.event_name == 'push') || - (github.event_name == 'workflow_dispatch' && needs.tests.result == 'success') - outputs: - version: ${{ steps.release-version.outputs.VERSION }} + contents: write steps: - name: checkout uses: actions/checkout@v4 - - name: github release (beta) - if: github.event_name == 'push' - id: prerelease + - name: dryrun + id: dryrun uses: ahmadnassri/action-semantic-release@v2 with: config: ${{ github.workspace }}/.releaserc.json + dry: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + tests: + needs: version + name: Run tests + uses: ./.github/workflows/tests.yaml + secrets: inherit - - name: github release (stable) - if: github.event_name == 'workflow_dispatch' - id: fullrelease - uses: ahmadnassri/action-semantic-release@v2 - with: - config: ${{ github.workspace }}/.releaserc-release.json - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set version for aws-release job - id: release-version - run: | - echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" - env: - VERSION: ${{ (steps.prerelease.outputs.release-version != '') && steps.prerelease.outputs.release-version || steps.fullrelease.outputs.release-version }} - - static-upload: - needs: [permission_check, github-release, tests] - uses: ./.github/workflows/static-upload.yaml + upload: + needs: version + name: Upload SAM assets + needs: tests + uses: ./.github/workflows/upload.yaml permissions: id-token: write - if: | - github.actor != 'dependabot[bot]' && - needs.github-release.outputs.version != '' secrets: inherit with: - version: ${{ needs.github-release.outputs.VERSION }} - - aws-release: - needs: [fetch-regions, github-release, tests] + s3_bucket_prefix: "observeinc-" + global: ${{ needs.version.outputs.version != '' }} + release_version: ${{ needs.version.outputs.version }} + + integration: + needs: upload + name: Run integration tests + if: ${{ !inputs.skip_integration_tests }} + uses: ./.github/workflows/integration.yaml + secrets: inherit + with: + s3_bucket_prefix: "observeinc-" + release_version: ${{ needs.version.outputs.release_version }} + + publish: + needs: [version, integration] runs-on: ubuntu-latest - if: | - github.actor != 'dependabot[bot]' && - needs.github-release.outputs.version != '' - strategy: - matrix: - region: ${{fromJson(needs.fetch-regions.outputs.matrix)}} permissions: contents: write id-token: write - pull-requests: write - steps: - - name: checkout + - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - - name: Setup AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.2 with: @@ -141,22 +82,26 @@ jobs: - name: AWS Info run: aws sts get-caller-identity - - name: Set release tag (beta) - if: github.event_name == 'push' - run: echo "TAG=beta" >> $GITHUB_ENV - - - name: Set release tag (latest) - if: github.event_name == 'workflow_dispatch' - run: echo "TAG=latest" >> $GITHUB_ENV + - name: Tag release + id: build + if: ${{ needs.version.outputs.tag != '' }} + run: | + make tag + env: + MAKEFLAGS: "-j 4 --output-sync=target" + S3_BUCKET_PREFIX: "observeinc-" + RELEASE_VERSION: "${{ needs.version.outputs.version }}" + TAG: ${{ needs.version.outputs.tag }} - - name: aws sam release - run: make release-all + - name: Cut release + id: release + uses: ahmadnassri/action-semantic-release@v2 + with: + config: ${{ github.workspace }}/.releaserc.json env: - # TAG is set implicitly - VERSION: ${{ needs.github-release.outputs.VERSION }} - AWS_REGION: ${{ matrix.region }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: delete pre-releases + - name: Delete older pre-releases uses: dev-drprasad/delete-older-releases@v0.3.4 with: keep_latest: 0 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2d27b0df..2045745a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,15 +1,21 @@ name: Run tests on: - push: - tags: - - v* - branches: - - main - pull_request: + workflow_dispatch: + workflow_call: jobs: + go-lint: + name: Lint Go + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Lint + run: make go-lint + go-test: + name: Run Go tests runs-on: ubuntu-latest steps: - name: Checkout code @@ -17,7 +23,9 @@ jobs: - name: Go unit tests run: make go-test - integration: - needs: go-test - uses: ./.github/workflows/tests-integration.yaml - secrets: inherit + sam-validate: + name: Validate SAM templates + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: make sam-validate diff --git a/.github/workflows/upload.yaml b/.github/workflows/upload.yaml new file mode 100644 index 00000000..a59aa892 --- /dev/null +++ b/.github/workflows/upload.yaml @@ -0,0 +1,86 @@ +name: Upload assets + +on: + workflow_call: + inputs: + global: + type: boolean + description: 'Upload to all supported regions' + default: false + concurrency: + type: number + description: 'Number of concurrenct jobs' + default: 4 + s3_bucket_prefix: + type: string + description: 'S3 bucket prefix to upload SAM assets to' + required: true + release_version: + type: string + description: 'Release version to use. If omitted, version will be computed' + default: '' + outputs: + release_version: + description: "Release version used." + value: ${{ jobs.upload.outputs.release_version }} + +env: + SAM_CLI_TELEMETRY: 0 + +jobs: + check_permission: + name: Check permissions + runs-on: ubuntu-latest + outputs: + can-write: ${{ steps.check.outputs.can-write }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + steps: + - id: check + run: | + # If the AWS_ACCESS_KEY_ID secret is MIA we can't run tests + if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then + echo "can-write=false" >> $GITHUB_OUTPUT + else + echo "can-write=true" >> $GITHUB_OUTPUT + fi + + upload: + name: Package and upload SAM apps + needs: [check_permission] + if: needs.check_permission.outputs.can-write == 'true' + runs-on: ubuntu-latest + permissions: + id-token: write + outputs: + release_version: ${{ steps.build.outputs.version }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Fetch tags for versioning + if: inputs.release_version == '' + run: git fetch --prune --unshallow --tags + + - name: Setup AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-west-2 + + - name: AWS Info + run: aws sts get-caller-identity + + - name: Build and upload SAM apps + id: build + run: | + if [ "${{ inputs.global }}" = "true" ]; then + make sam-push + else + make sam-push-us-west-2 + fi + echo "version=`make version`" >> $GITHUB_OUTPUT + env: + MAKEFLAGS: "-j ${{ inputs.concurrency }} --output-sync=target" + S3_BUCKET_PREFIX: "${{ inputs.s3_bucket_prefix }}" + RELEASE_VERSION: "${{ inputs.release_version }}" diff --git a/.releaserc.json b/.releaserc.json index 984efe00..f51614b8 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -1,13 +1,10 @@ { "branches": [ { - "name": "release", - "channel": false, - "prerelease": false, - "type": "maintenance" + "name": "main" }, { - "name": "main", + "name": "joao/new-make", "channel": "beta", "prerelease": "beta" } @@ -19,4 +16,4 @@ "@semantic-release/github" ] } - \ No newline at end of file + diff --git a/DEVELOPER.md b/DEVELOPER.md index 9aef6048..348f19eb 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -2,14 +2,13 @@ This document details the processes and commands needed to develop, build, and deploy applications within this project. -## Setup +## Prequisites Before you begin, ensure that you have the following tools installed: -- AWS CLI -- SAM CLI -- Docker -- Terraform -- jq +- [aws-cli](https://github.com/aws/aws-cli) +- [aws-sam-cli](https://github.com/aws/aws-sam-cli) +- [Docker](https://docs.docker.com/engine/install/) +- [Terraform](https://developer.hashicorp.com/terraform/install) Set up your AWS credentials and configure the default region: @@ -18,14 +17,119 @@ export AWS_REGION=us-east-1 aws configure ``` -## Makefile Targets for SAM +## Repository organization -The project's Makefile streamlines the development process with various targets: +The most important directories: + +- `apps/` contains the SAM template definitions. This way lies Cloudformation. +- `cmd/` contains Go entrypoints used by the different Lambda functions. +- `docs/` contains repo documentation. +- `integration/` contains Terraform for integration testing +- `vendor/` contains vendored Golang dependencies. + +## Makefile Targets + +Our Makefile encodes all development and release workflows. To list all targets +and the most important variables, run `make help`, e.g: ``` -export APP=forwarder #As an example +VARIABLES: + APPS = config configsubscription firehose forwarder logwriter metricstream stack + AWS_REGION = us-west-2 + GO_BINS = forwarder subscriber + GO_BUILD_DIRS = bin/linux_arm64 .go/bin/linux_arm64 .go/cache .go/pkg + TF_TESTS = config configsubscription firehose forwarder forwarder_s3 logwriter metricstream simple stack + VERSION = v1.19.2-4-gb1238b5-dirty + +TARGETS: + clean removes built binaries and temporary files. + go-build build Go binaries. + go-clean clean Go temp files. + ... ``` +## Running tests + +This repository contains both Go code and SAM templates which can be quickly +validated locally: + +`make go-test` executes Go unit tests. You can use the `GOFLAGS` environment +variable to pass in additional flags. Tests are executed within a docker +container. During development you may prefer to run `go` directly in your local +environment. A dockerized environment is provided to ensure consistency in +builds and across CI. + +`make go-lint` lints Go code using [golangci-lint](https://github.com/golangci/golangci-lint). + +`make sam-validate` validates SAM templates in `apps/${APP}/template.yaml`. To +run validation for a specific app, run `make sam-validate-${APP}`. This command +will also lint data according to [yamllint] and [aws-sam-cli](https://github.com/aws/aws-sam-cli). + +## Packaging apps + +Once your tests pass, you are ready to package a SAM application by running `make sam-package`. +You will need AWS credentials allow you to write objects to an S3 bucket. + + + + + +### Building Lambda binaries + +`make go-build` is responsible for building Lambda binaries. The list of +binaries to compile are controlled through the `GO_BINS` variable. The target +architecture is set to `arm64` for compatibility with the `provided.al2` Lambda +runtime. + +A build is tagged with a version. By omission, the version is derived from +git. You can override the version by setting the `RELEASE_VERSION` environment +variable. + +Our build process follows [go-build-template](https://github.com/thockin/go-build-template) very +closely in order to minimize file changes that would otherwise confuse Make. + +Once build is successful, you should have a set of binaries under `bin/linux_arm64`. + +### SAM package + +`make sam-package` is responsible for running `sam build`, followed by `sam package`. + +`sam build` takes the SAM templates under `apps/${APP}/template.yaml`, and +produces a directory containing all necessary templates and artifacts for that +particular app. In our case, it does not actually build any binaries. It +invokes Make in the `CodeUri` directory specified in the template: + +``` + Forwarder: + Type: AWS::Serverless::Function + Metadata: + BuildMethod: makefile + Properties: + CodeUri: ../../bin/linux_arm64 +``` + +We create `bin/linux_arm64/Makefile` as part of the dependencies to `make +sam-package`. The code is in `lambda.mk`, and simply copies the binary from the +Go build directory, to the temporary build directory provided through the +`${ARTIFACTS_DIR}` environment variable. + +### Push assets + + +## CI workflows + +A `push` workflow is triggered on every push. It executes the following sequence: + +- Run tests + - `make go-test` + - `make go-lint` + - `make sam-validate` +- Upload SAM assets + - `make sam-push` +- Run integration tests + - `make sam-pull` + - `make test-integration` + Refer to `make help` for an authoritative list of Make targets - Build: Compile your application for deployment (`make sam-build APP=$APP`) diff --git a/Makefile b/Makefile index 07f19a80..73182ce0 100644 --- a/Makefile +++ b/Makefile @@ -16,44 +16,111 @@ MAKEFLAGS += --no-builtin-rules MAKEFLAGS += --warn-undefined-variables .SUFFIXES: -VERSION ?= $(shell git describe --tags --always --dirty) - --include golang.mk - - - -APPS := $(shell find apps/* -type d -maxdepth 0 -exec basename {} \;) -AWS_REGION ?= us-west-2 -AWS_REGIONS := us-west-2 \ - us-west-1 \ - us-east-2 \ - us-east-1 \ - sa-east-1 \ - eu-west-3 \ - eu-west-2 \ - eu-west-1 \ - eu-north-1 \ - eu-central-1 \ - ca-central-1 \ - ap-southeast-2 \ - ap-southeast-1 \ - ap-south-1 \ - ap-northeast-3 \ - ap-northeast-2 \ - ap-northeast-1 \ - -# leave this undefined for the purposes of development -S3_BUCKET_PREFIX ?= -SAM_BUILD_DIR ?= .aws-sam/build -SAM_CONFIG_FILE ?= $(shell pwd)/samconfig.yaml -SAM_CONFIG_ENV ?= default -TF_TESTS ?= $(shell ls integration/tests | awk -F. '{print $$1}') -TF_TEST_DEBUG ?= 0 -TF_TEST_ARGS ?= +-include variables.mk + +LAMBDA_MAKEFILE = bin/$(OS)_$(ARCH)/Makefile + +$(LAMBDA_MAKEFILE): $(GO_BUILD_DIRS) + cp lambda.mk $@ .PHONY: clean -clean: # @HELP removes built binaries and temporary files -clean: bin-clean +clean: # @HELP removes built binaries and temporary files. +clean: go-clean sam-clean + +$(GO_BUILD_DIRS): + mkdir -p $@ + +# The following structure defeats Go's (intentional) behavior to always touch +# result files, even if they have not changed. This will still run `go` but +# will not trigger further work if nothing has actually changed. +GO_OUTBINS = $(foreach bin,$(GO_BINS),bin/$(OS)_$(ARCH)/$(bin)) + +go-build: # @HELP build Go binaries. +go-build: $(GO_OUTBINS) + echo + +# Each outbin target is just a facade for the respective stampfile target. +# This `eval` establishes the dependencies for each. +$(foreach outbin,$(GO_OUTBINS),$(eval \ + $(outbin): .go/$(outbin).stamp \ +)) + +# This is the target definition for all outbins. +$(GO_OUTBINS): + true + +# Each stampfile target can reference an $(OUTBIN) variable. +$(foreach outbin,$(GO_OUTBINS),$(eval $(strip \ + .go/$(outbin).stamp: OUTBIN = $(outbin) \ +))) + +# This is the target definition for all stampfiles. +# This will build the binary under ./.go and update the real binary iff needed. +GO_STAMPS = $(foreach outbin,$(GO_OUTBINS),.go/$(outbin).stamp) +.PHONY: $(GO_STAMPS) +$(GO_STAMPS): go-build-bin + echo -ne "binary: $(OUTBIN) " + if ! cmp -s .go/$(OUTBIN) $(OUTBIN); then \ + mv .go/$(OUTBIN) $(OUTBIN); \ + date >$@; \ + echo; \ + else \ + echo "(cached)"; \ + fi + +# This runs the actual `go build` which updates all binaries. +go-build-bin: | $(GO_BUILD_DIRS) + echo "# building $(VERSION) for $(OS)/$(ARCH)" + docker run \ + -i \ + --rm \ + -u $$(id -u):$$(id -g) \ + -v $$(pwd):/src \ + -w /src \ + -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin \ + -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH) \ + -v $$(pwd)/.go/cache:/.cache \ + -v $$(pwd)/.go/pkg:/go/pkg \ + --env GOARCH=$(ARCH) \ + --env GOFLAGS="$(GOFLAGS) -mod=$(GO_MOD)" \ + --env GOOS=$(OS) \ + $(GO_BUILD_IMAGE) \ + /bin/sh -c " \ + go install \ + -tags lambda.norpc \ + -ldflags \"-X $$(go list -m)/pkg/version.Version=$(VERSION)\" \ + ./... \ + " + +go-clean: # @HELP clean Go temp files. +go-clean: + test -d .go && chmod -R u+w .go || true + rm -rf .go bin + +go-test: # @HELP run Go unit tests. +go-test: | $(GO_BUILD_DIRS) + docker run \ + -i \ + --rm \ + -u $$(id -u):$$(id -g) \ + -v $$(pwd):/src \ + -w /src \ + -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin \ + -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH) \ + -v $$(pwd)/.go/cache:/.cache \ + -v $$(pwd)/.go/pkg:/go/pkg \ + --env GOFLAGS="$(GOFLAGS) -mod=$(GO_MOD)" \ + $(GO_BUILD_IMAGE) \ + /bin/sh -c " \ + go test ./... \ + " + +go-lint: # @HELP lint Go workspace. +go-lint: + docker run --rm -v "$$(pwd):/workspace:cached" -w "/workspace/." golangci/golangci-lint:latest golangci-lint run --timeout 3m && echo "lint passed" + +sam-clean: # @HELP remove SAM build directory. +sam-clean: rm -rf $(SAM_BUILD_DIR) SAM_BUILD_TEMPLATES = $(foreach app,$(APPS), $(SAM_BUILD_DIR)/apps/$(app)/template.yaml) @@ -62,14 +129,14 @@ $(foreach template,$(SAM_BUILD_TEMPLATE),$(eval \ $(template): apps/$(call get_app, $(template))/template.yaml \ )) -$(SAM_BUILD_TEMPLATES): go-build bin/Makefile +$(SAM_BUILD_TEMPLATES): go-build $(LAMBDA_MAKEFILE) sam build \ - -p \ - -beta-features \ - --template-file $(patsubst $(SAM_BUILD_DIR)/%,%,$@) \ - --build-dir $(patsubst %template.yaml,%,$@) \ - --config-file $(SAM_CONFIG_FILE) \ - --config-env $(SAM_CONFIG_ENV) + -p \ + -beta-features \ + --template-file $(patsubst $(SAM_BUILD_DIR)/%,%,$@) \ + --build-dir $(patsubst %template.yaml,%,$@) \ + --config-file $(SAM_CONFIG_FILE) \ + --config-env $(SAM_CONFIG_ENV) SAM_PACKAGE_TARGETS = $(foreach app,$(APPS),sam-package-$(app)) @@ -79,6 +146,10 @@ $(foreach target,$(SAM_PACKAGE_TARGETS),$(eval \ $(target): $(SAM_BUILD_DIR)/regions/$(AWS_REGION)/$(lastword $(subst -, , $(target))).yaml \ )) +define check_var + @[[ -n "$($1)" ]] || (echo >&2 "The environment variable '$1' is not set." && exit 2) +endef + define get_region $(lastword $(subst /, ,$(basename $(dir $(1))))) endef @@ -100,26 +171,61 @@ $(SAM_PACKAGE_DIRS): $(SAM_PACKAGE_TEMPLATES): | $(SAM_PACKAGE_DIRS) ifeq ($(S3_BUCKET_PREFIX),) sam package \ - --template-file $(SAM_BUILD_DIR)/apps/$(call get_app, $@)/template.yaml \ - --output-template-file $@ \ - --region $(call get_region, $@) \ - --resolve-s3 \ - --s3-prefix aws-sam-apps/$(VERSION) \ - --no-progressbar \ - --config-file $(SAM_CONFIG_FILE) \ - --config-env $(SAM_CONFIG_ENV) + --template-file $(SAM_BUILD_DIR)/apps/$(call get_app, $@)/template.yaml \ + --output-template-file $@ \ + --region $(call get_region, $@) \ + --resolve-s3 \ + --s3-prefix aws-sam-apps/$(VERSION) \ + --no-progressbar \ + --config-file $(SAM_CONFIG_FILE) \ + --config-env $(SAM_CONFIG_ENV) else sam package \ - --template-file apps/$(call get_app, $@)/template.yaml \ - --output-template-file $@ \ - --region $(call get_region, $@) \ - --s3-bucket "$(S3_BUCKET_PREFIX)$(call get_region, $@)" \ - --s3-prefix aws-sam-apps/$(VERSION) \ - --no-progressbar \ - --config-file $(SAM_CONFIG_FILE) \ - --config-env $(SAM_CONFIG_ENV) + --template-file $(SAM_BUILD_DIR)/apps/$(call get_app, $@)/template.yaml \ + --output-template-file $@ \ + --region $(call get_region, $@) \ + --s3-bucket "$(S3_BUCKET_PREFIX)$(call get_region, $@)" \ + --s3-prefix aws-sam-apps/$(VERSION) \ + --no-progressbar \ + --config-file $(SAM_CONFIG_FILE) \ + --config-env $(SAM_CONFIG_ENV) endif +SAM_PULL_REGION_TARGETS = $(foreach region,$(AWS_REGIONS),sam-pull-$(region)) + +.PHONY: $(SAM_PULL_REGION_TARGETS) +$(SAM_PULL_REGION_TARGETS): require_bucket_prefix + # force ourselves to use the public URLs, verifying ACLs are correctly set + mkdir -p $(SAM_BUILD_DIR)/regions/$(subst sam-pull-,,$@) && \ + cd $(SAM_BUILD_DIR)/regions/$(subst sam-pull-,,$@) && \ + for app in $(APPS); do \ + curl -fs \ + -O https://$(S3_BUCKET_PREFIX)$(subst sam-pull-,,$@).s3.amazonaws.com/aws-sam-apps/$(VERSION)/$${app}.yaml \ + -w "Pulled %{url_effective} status=%{http_code} size=%{size_download}\n" || exit 1; \ + done + +SAM_PUSH_REGION_TARGETS = $(foreach region,$(AWS_REGIONS),sam-push-$(region)) + +$(foreach target,$(SAM_PUSH_REGION_TARGETS),$(eval \ + $(target): $(foreach app,$(APPS), $(SAM_BUILD_DIR)/regions/$(subst sam-push-,,$(target))/$(app).yaml) \ +)) + +require_bucket_prefix: + $(call check_var,S3_BUCKET_PREFIX) + +.PHONY: $(SAM_PUSH_REGION_TARGETS) +$(SAM_PUSH_REGION_TARGETS): require_bucket_prefix + # ensure all previously pushed assets are public + aws s3 cp \ + --acl public-read \ + --recursive \ + s3://$(S3_BUCKET_PREFIX)$(subst sam-push-,,$@)/aws-sam-apps/$(VERSION)/ s3://$(S3_BUCKET_PREFIX)$(subst sam-push-,,$@)/aws-sam-apps/$(VERSION)/ + # push base manifests + aws s3 cp \ + --acl public-read \ + --recursive \ + $(SAM_BUILD_DIR)/regions/$(subst sam-push-,,$@)/ s3://$(S3_BUCKET_PREFIX)$(subst sam-push-,,$@)/aws-sam-apps/$(VERSION)/ + SAM_VALIDATE_TARGETS = $(foreach app,$(APPS),sam-validate-$(app)) .PHONY: $(SAM_VALIDATE_TARGETS) @@ -138,38 +244,79 @@ test-init: .PHONY: $(TEST_INTEGRATION_TARGETS) $(TEST_INTEGRATION_TARGETS): test-init if [ "$(TF_TEST_DEBUG)" = "1" ]; then \ - CHECK_DEBUG_FILE=debug.sh terraform -chdir=integration test -filter=tests/$(lastword $(subst -, ,$@)).tftest.hcl $(TF_TEST_ARGS); \ + CHECK_DEBUG_FILE=debug.sh terraform -chdir=integration test -filter=tests/$(lastword $(subst -, ,$@)).tftest.hcl $(TF_TEST_ARGS); \ else \ - terraform -chdir=integration test -filter=tests/$(lastword $(subst -, ,$@)).tftest.hcl $(TF_TEST_ARGS); \ + terraform -chdir=integration test -filter=tests/$(lastword $(subst -, ,$@)).tftest.hcl $(TF_TEST_ARGS); \ fi -.PHONY: release -release: $(SAM_PACKAGE_TEMPLATES) + +TAG_REGION_TARGETS = $(foreach region,$(AWS_REGIONS),tag-$(region)) + +$(foreach target,$(TAG_REGION_TARGETS),$(eval \ + $(target): sam-pull-$(subst tag-,,$(target)) \ +)) + +$(TAG_REGION_TARGETS): + $(call check_var,RELEASE_TAG) + aws s3 sync \ + --acl public-read \ + --delete \ + $(SAM_BUILD_DIR)/regions/$(subst tag-,,$@)/ s3://$(S3_BUCKET_PREFIX)$(subst tag-,,$@)/aws-sam-apps/$(RELEASE_TAG)/ .PHONY: sam-package -sam-package: # @HELP package all SAM templates. For specific app append name (e.g sam-package-forwarder) +sam-package: # @HELP package all SAM templates. sam-package: $(SAM_PACKAGE_TARGETS) +sam-package-%: # @HELP package specific SAM app (e.g sam-package-forwarder). + +.PHONY: sam-pull +sam-pull: # @HELP pull SAM app manifests from remote URI to local build directory. +sam-pull: $(SAM_PULL_TARGETS) + +sam-pull-%: # @HELP puall SAM app manifests for specific region (e.g sam-pull-us-west-2). + +.PHONY: sam-push +sam-push: # @HELP package and push SAM assets to S3 to all regions. +sam-push: $(SAM_PUSH_REGION_TARGETS) + +sam-push-%: # @HELP push all SAM apps to specific region (e.g sam-push-us-west-2) + .PHONY: sam-validate -sam-validate: # @HELP validate all SAM templates. For specific app append name (e.g sam-validate-logwriter) +sam-validate: # @HELP validate all SAM templates. sam-validate: $(SAM_VALIDATE_TARGETS) +sam-validate-%: # @HELP validate specific SAM app (e.g. sam-validate-logwriter). + +.PHONY: tag +tag: # @HELP pull SAM manifests for RELEASE_VERSION, and publish as RELEASE_TAG. +tag: $(TAG_REGION_TARGETS) + +tag-%: # @HELP tag for specific region (e.g tag-us-west-2). + + .PHONY: test-integration -test-integration: # @HELP run all integration tests. For specific test append name (e.g test-integration-stack) +test-integration: # @HELP run all integration tests. test-integration: $(TEST_INTEGRATION_TARGETS) +test-integration-%: # @HELP run specific integration test (e.g. test-integration-stack). + +.PHONY: version +version: # @HELP display version +version: + echo "$(VERSION)" -help: # @HELP displays this message +help: # @HELP displays this message. help: echo "VARIABLES:" - echo " APPS = $(APPS)" - echo " AWS_REGION = $(AWS_REGION)" - echo " BINS = $(BINS)" - echo " TF_TESTS = $(TF_TESTS)" - echo " VERSION = $(VERSION)" + echo " APPS = $(APPS)" + echo " AWS_REGION = $(AWS_REGION)" + echo " GO_BINS = $(GO_BINS)" + echo " GO_BUILD_DIRS = $(GO_BUILD_DIRS)" + echo " TF_TESTS = $(TF_TESTS)" + echo " VERSION = $(VERSION)" echo echo "TARGETS:" - grep -E '^.*: *# *@HELP' $(MAKEFILE_LIST) \ + grep -E '^.*: *# *@HELP' $(MAKEFILE_LIST) | cut -d':' -f2- \ | awk ' \ BEGIN {FS = ": *# *@HELP"}; \ { printf " %-30s %s\n", $$1, $$2 }; \ diff --git a/apps/forwarder/template.yaml b/apps/forwarder/template.yaml index cda06be3..61187173 100644 --- a/apps/forwarder/template.yaml +++ b/apps/forwarder/template.yaml @@ -388,7 +388,7 @@ Resources: - !Ref AWS::StackName - !Ref NameOverride Role: !GetAtt Role.Arn - CodeUri: ../../bin/ + CodeUri: ../../bin/linux_arm64 Handler: bootstrap Runtime: provided.al2 MemorySize: !If diff --git a/apps/logwriter/template.yaml b/apps/logwriter/template.yaml index cecf578b..9dc2e306 100644 --- a/apps/logwriter/template.yaml +++ b/apps/logwriter/template.yaml @@ -422,7 +422,7 @@ Resources: - !Ref AWS::StackName - !Ref NameOverride Role: !GetAtt SubscriberRole.Arn - CodeUri: ../../bin/ + CodeUri: ../../bin/linux_arm64 Handler: bootstrap Runtime: provided.al2 MemorySize: !If diff --git a/golang.mk b/golang.mk deleted file mode 100644 index 95f71cea..00000000 --- a/golang.mk +++ /dev/null @@ -1,112 +0,0 @@ -BINS := forwarder subscriber - -GO_BUILD_IMAGE ?= golang:1.22-alpine - -OS := $(if $(GOOS),$(GOOS),linux) -ARCH := $(if $(GOARCH),$(GOARCH),arm64) - -# Which Go modules mode to use ("mod" or "vendor") -MOD ?= vendor - -# Satisfy --warn-undefined-variables. -GOFLAGS ?= - -# The following structure defeats Go's (intentional) behavior to always touch -# result files, even if they have not changed. This will still run `go` but -# will not trigger further work if nothing has actually changed. -GO_OUTBINS = $(foreach bin,$(BINS),bin/$(OS)_$(ARCH)/$(bin)) - -go-build: $(GO_OUTBINS) - echo - -# Directories that we need created to build/test. -GO_BUILD_DIRS := bin/$(OS)_$(ARCH) \ - .go/bin/$(OS)_$(ARCH) \ - .go/cache \ - .go/pkg - -$(GO_BUILD_DIRS): - mkdir -p $@ - -# Each outbin target is just a facade for the respective stampfile target. -# This `eval` establishes the dependencies for each. -$(foreach outbin,$(GO_OUTBINS),$(eval \ - $(outbin): .go/$(outbin).stamp \ -)) - -# This is the target definition for all outbins. -$(GO_OUTBINS): - true - -# Each stampfile target can reference an $(OUTBIN) variable. -$(foreach outbin,$(GO_OUTBINS),$(eval $(strip \ - .go/$(outbin).stamp: OUTBIN = $(outbin) \ -))) - -# This is the target definition for all stampfiles. -# This will build the binary under ./.go and update the real binary iff needed. -STAMPS = $(foreach outbin,$(GO_OUTBINS),.go/$(outbin).stamp) -.PHONY: $(STAMPS) -$(STAMPS): go-build-bin - echo -ne "binary: $(OUTBIN) " - if ! cmp -s .go/$(OUTBIN) $(OUTBIN); then \ - mv .go/$(OUTBIN) $(OUTBIN); \ - date >$@; \ - echo; \ - else \ - echo "(cached)"; \ - fi - -# This runs the actual `go build` which updates all binaries. -go-build-bin: | $(GO_BUILD_DIRS) - echo "# building $(VERSION) for $(OS)/$(ARCH)" - docker run \ - -i \ - --rm \ - -u $$(id -u):$$(id -g) \ - -v $$(pwd):/src \ - -w /src \ - -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH) \ - -v $$(pwd)/.go/cache:/.cache \ - -v $$(pwd)/.go/pkg:/go/pkg \ - --env GOARCH=$(ARCH) \ - --env GOOS=$(OS) \ - $(GO_BUILD_IMAGE) \ - /bin/sh -c " \ - go install \ - -installsuffix \"static\" \ - -ldflags \"-X $$(go list -m)/pkg/version.Version=${VERSION}\" \ - ./... \ - " - -bin/Makefile: $(GO_BUILD_DIRS) - cp lambda.mk $@ - -bin-clean: - test -d .go && chmod -R u+w .go || true - rm -rf .go bin - -go-test: | $(GO_BUILD_DIRS) - docker run \ - -i \ - --rm \ - -u $$(id -u):$$(id -g) \ - -v $$(pwd):/src \ - -w /src \ - -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin \ - -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH) \ - -v $$(pwd)/.go/cache:/.cache \ - -v $$(pwd)/.go/pkg:/go/pkg \ - $(GO_BUILD_IMAGE) \ - /bin/sh -c " \ - ARCH=$(ARCH) \ - OS=$(OS) \ - VERSION=$(VERSION) \ - MOD=$(MOD) \ - GOFLAGS=$(GOFLAGS) \ - go test -installsuffix "static" ./... \ - " - -go-lint: # @HELP lint golang workspace -go-lint: - docker run --rm -v "$$(pwd):/workspace:cached" -w "/workspace/." golangci/golangci-lint:latest golangci-lint run --timeout 3m && echo "lint passed" diff --git a/lambda.mk b/lambda.mk index e844f176..8c57be83 100644 --- a/lambda.mk +++ b/lambda.mk @@ -1,5 +1,17 @@ +# This makefile is called by `sam package` when it is assembling template assets +# We do not compile the binaries in this Makefile. Instead, we rely on our +# Makefile to have already compiled everything prior to reaching this stage. +# +# This Makefile is copied into the Go build directory where it can be invoked by SAM. +# We do this because SAM will copy over the entire directory referenced in the +# CodeUri field. Relying on our root Makefile incurs a lot of delay due to the +# large number of files that would be copied over. +# +# SAM invokes this makefile with the target `build-`. We name our +# lambda function resources according to the binary name, but capitalize them +# for consistency with the remainder of the SAM template. We must therefore +# make the copy case insensitive. strip_and_lowercase = $(shell echo $(1) | sed 's/^build-//' | tr '[:upper:]' '[:lower:]') build-%: - cp linux_arm64/$(call strip_and_lowercase,$@) $(ARTIFACTS_DIR)/bootstrap - + cp $(call strip_and_lowercase,$@) $(ARTIFACTS_DIR)/bootstrap diff --git a/variables.mk b/variables.mk new file mode 100644 index 00000000..1f58e889 --- /dev/null +++ b/variables.mk @@ -0,0 +1,61 @@ +# Each directory under apps/* must contain a validate SAM template +APPS := $(shell find apps/* -type d -maxdepth 0 -exec basename {} \;) + +# This is our default region when not provided. +AWS_REGION ?= us-west-2 + +# List of regions supported by `make sam-push-*`. +AWS_REGIONS := us-west-2 \ + us-west-1 \ + us-east-2 \ + us-east-1 \ + sa-east-1 \ + eu-west-3 \ + eu-west-2 \ + eu-west-1 \ + eu-north-1 \ + eu-central-1 \ + ca-central-1 \ + ap-southeast-2 \ + ap-southeast-1 \ + ap-south-1 \ + ap-northeast-3 \ + ap-northeast-2 \ + ap-northeast-1 \ + +# Assume lambda functions are linux/arm64 +# These variables must be defined before GO_BUILD_DIRS +OS := $(if $(GOOS),$(GOOS),linux) +ARCH := $(if $(GOARCH),$(GOARCH),arm64) + +# Names of binaries to compile as lambda functions +GO_BINS := forwarder subscriber +# Directories that we need created to build/test. +GO_BUILD_DIRS := bin/$(OS)_$(ARCH) \ + .go/bin/$(OS)_$(ARCH) \ + .go/cache \ + .go/pkg +# Build image to use for building lambda functions +GO_BUILD_IMAGE ?= golang:1.22-alpine +# Which Go modules mode to use ("mod" or "vendor") +GO_MOD ?= vendor +GOFLAGS ?= + +# Bucket prefix used when running `sam-push-*`. This can be omitted for +# development purposes, in which case the `sam package` command will provision +# a bucket. +S3_BUCKET_PREFIX ?= +SAM_BUILD_DIR ?= .aws-sam/build +SAM_CONFIG_FILE ?= $(shell pwd)/samconfig.yaml +SAM_CONFIG_ENV ?= default + +# List of tftests supported by `make test-integration-*` +TF_TESTS ?= $(shell ls integration/tests | awk -F. '{print $$1}') +# Setting this flag to 1 will enable verbose logging and allow debugging of checks. +TF_TEST_DEBUG ?= 0 +TF_TEST_ARGS ?= + +RELEASE_TAG ?= latest + +# Version should only be overridden in CI. Cannot be empty. +VERSION := $(if $(RELEASE_VERSION),$(RELEASE_VERSION),$(shell git describe --tags --always --dirty))