diff --git a/.fastly/config.toml b/.fastly/config.toml index 150708133..7a18a7a05 100644 --- a/.fastly/config.toml +++ b/.fastly/config.toml @@ -14,5 +14,3 @@ api_endpoint = "https://api.fastly.com" [viceroy] ttl = "24h" - -# starter kits will be appended by devhub build process diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml index 82e8dd6f8..b933f7def 100644 --- a/.github/workflows/pr_test.yml +++ b/.github/workflows/pr_test.yml @@ -7,25 +7,45 @@ concurrency: cancel-in-progress: true jobs: + config: + runs-on: ubuntu-latest + steps: + - name: "Checkout code" + uses: actions/checkout@v3 + - name: "Install Rust" + uses: actions-rs/toolchain@v1 + with: + toolchain: stable # to install tomlq via `make config` + - name: "Generate static app config" + run: make config + - name: "Config Artifact" + uses: actions/upload-artifact@v3 + with: + name: config-artifact-${{ github.sha }} + path: pkg/config/config.toml lint: runs-on: ubuntu-latest steps: - name: "Checkout code" - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: "Install Rust" + uses: actions-rs/toolchain@v1 + with: + toolchain: stable # to install tomlq via `make vet` pre-requisite - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.18.x - name: "Restore golang bin cache" id: go-bin-cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/bin key: ${{ runner.os }}-go-bin-${{ hashFiles('~/go/bin') }}-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-bin- - name: "Restore golang mod cache" - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/Library/Caches/go-build @@ -37,13 +57,13 @@ jobs: if: steps.go-bin-cache.outputs.cache-hit != 'true' run: make dependencies shell: bash - - name: "Download latest app config" - run: | - make config - name: "Run go mod tidy" run: make tidy - name: "Run go fmt" run: make fmt + # NOTE: We don't download the config artifact in this job. + # This is because we know Linux is able to generate the configuration file. + # Which is triggered by the `make vet` pre-requisite target `config`. - name: "Run go vet" run: make vet shell: bash @@ -57,60 +77,70 @@ jobs: run: make gosec shell: bash test: + needs: config strategy: matrix: - tinygo-version: [0.24.0] + tinygo-version: [0.26.0] go-version: [1.18.x] node-version: [18] rust-toolchain: [stable] platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: - - name: "Checkout code" - uses: actions/checkout@v2 - - name: "Install Go" - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - uses: Integralist/setup-tinygo@v1.0.0 - with: - tinygo-version: ${{ matrix.tinygo-version }} - - name: "Restore golang bin cache" - uses: actions/cache@v2 - with: - path: ~/go/bin - key: ${{ runner.os }}-go-bin-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-bin- - - name: "Restore golang mod cache" - uses: actions/cache@v2 - with: - path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-mod- - - name: "Install Rust" - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.rust-toolchain }} - - name: "Add wasm32-wasi Rust target" - run: rustup target add wasm32-wasi --toolchain ${{ matrix.rust-toolchain }} - - name: "Validate Rust toolchain" - run: rustup show && rustup target list --installed --toolchain stable - shell: bash - - name: "Install Node" - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - name: "Download latest app config" - run: | - make config - - name: "Test suite" - run: make test - shell: bash - env: - TEST_COMPUTE_INIT: true - TEST_COMPUTE_BUILD: true - TEST_COMPUTE_DEPLOY: true + - name: "Checkout code" + uses: actions/checkout@v3 + - name: "Install Go" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - uses: Integralist/setup-tinygo@v1.0.0 + with: + tinygo-version: ${{ matrix.tinygo-version }} + - name: "Restore golang bin cache" + uses: actions/cache@v3 + with: + path: ~/go/bin + key: ${{ runner.os }}-go-bin-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-bin- + - name: "Restore golang mod cache" + uses: actions/cache@v3 + with: + path: | + ~/Library/Caches/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-mod- + - name: "Install Rust" + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust-toolchain }} + - name: "Add wasm32-wasi Rust target" + run: rustup target add wasm32-wasi --toolchain ${{ matrix.rust-toolchain }} + - name: "Validate Rust toolchain" + run: rustup show && rustup target list --installed --toolchain stable + shell: bash + - name: "Install Node" + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: "Config Artifact" + uses: actions/download-artifact@v3 + with: + name: config-artifact-${{ github.sha }} + - name: "Move Config" + run: mv config.toml pkg/config/config.toml + # NOTE: Windows should fail quietly for 'test' pre-requisite target. + # On Windows, executing `make config` works fine. + # But via GitHub Actions the ../../scripts/config.sh isn't run. + # This is because you can't nest PowerShell instances. + # Each GitHub Action 'run' step is a PowerShell instance. + # And each instance is run as: powershell.exe -command ". '...'" + - name: "Test suite" + run: make test + shell: bash + env: + TEST_COMPUTE_INIT: true + TEST_COMPUTE_BUILD: true + TEST_COMPUTE_DEPLOY: true diff --git a/.github/workflows/tag_release.yml b/.github/workflows/tag_release.yml index 70418cb94..06782a663 100644 --- a/.github/workflows/tag_release.yml +++ b/.github/workflows/tag_release.yml @@ -9,22 +9,21 @@ jobs: runs-on: ubuntu-latest steps: - name: "Checkout code" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Fetch unshallow repo" run: git fetch --prune --unshallow - name: "Install Go" - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: '1.18.x' - name: "Set GOHOSTOS and GOHOSTARCH" run: echo "GOHOSTOS=$(go env GOHOSTOS)" >> $GITHUB_ENV && echo "GOHOSTARCH=$(go env GOHOSTARCH)" >> $GITHUB_ENV - - name: "Download latest app config" - run: | - make config - - name: "Display app config" - run: cat pkg/config/config.toml - - name: "Validate app config" - run: cat pkg/config/config.toml | grep 'remote_config = "https://developer.fastly.com/api/internal/cli-config"' + - name: "Install Rust" + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: "Generate static app config" + run: make config # Passing the raw SSH private key causes an error: # Load key "/tmp/id_*": invalid format # @@ -38,11 +37,11 @@ jobs: - name: "Store AUR_KEY in local file" run: echo '${{ secrets.AUR_KEY }}' > '${{ github.workspace }}/aur_key' && chmod 600 '${{ github.workspace }}/aur_key' - name: "Run GoReleaser" - uses: goreleaser/goreleaser-action@v2 + uses: goreleaser/goreleaser-action@v3 with: # goreleaser version (NOT goreleaser-action version) # update inline with the Makefile - version: v1.9.2 + version: latest args: release --rm-dist env: AUR_KEY: '${{ github.workspace }}/aur_key' diff --git a/.gitignore b/.gitignore index cf53929c3..39e2f82fc 100644 --- a/.gitignore +++ b/.gitignore @@ -63,5 +63,5 @@ vendor/ aur_key # Ignore static config that is embedded into the CLI -# All Makefile targets use the 'config' as a prerequisite (which downloads the config) +# All Makefile targets use the 'config' as a prerequisite (which generates the config) pkg/config/config.toml diff --git a/DEVELOP.md b/DEVELOP.md index 9d274af17..a68bae272 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -55,18 +55,12 @@ CLI_CATEGORY=logging CLI_CATEGORY_COMMAND=logging CLI_PACKAGE=foobar CLI_COMMAND > **NOTE**: Within the generated files, keep an eye out for any `<...>` references that need to be manually updated. -### config.toml +### `.fastly/config.toml` -When compiling the CLI for a new release, it will pull the required configuration data from the following API endpoint: +The CLI dynamically generates the `./pkg/config/config.toml` within the CI release process so it can be embedded into the CLI binary. -``` -https://developer.fastly.com/api/internal/cli-config -``` - -The config served from that endpoint is the result of an internal Fastly build process that uses the `.fastly/config.toml` file in the CLI repo as a template, and then dynamically adds in any available Starter Kits. +The file is added to `.gitignore` to avoid it being added to the git repository. -The configuration is then saved to disk (at the following location) and embedded into the CLI when compiled. +When compiling the CLI for a new release, it will execute [`./scripts/config.sh`](./scripts/config.sh). The script uses [`./.fastly/config.toml`](./.fastly/config.toml) as a template file to then dynamically inject a list of starter kits (pulling their data from their public repositories). -``` -./pkg/config/config.toml -``` +The resulting configuration is then saved to disk at `./pkg/config/config.toml` and embedded into the CLI when compiled. diff --git a/Makefile b/Makefile index 6f58d2769..592d8fdaf 100644 --- a/Makefile +++ b/Makefile @@ -27,8 +27,12 @@ ifeq ($(OS), Windows_NT) .SHELLFLAGS = /c GO_FILES = $(shell where /r pkg *.go) GO_FILES += $(shell where /r cmd *.go) + CONFIG_SCRIPT = scripts\config.sh + CONFIG_FILE = pkg\config\config.toml else GO_FILES = $(shell find cmd pkg -type f -name '*.go') + CONFIG_SCRIPT = ./scripts/config.sh + CONFIG_FILE = pkg/config/config.toml endif # You can pass flags to goreleaser via GORELEASER_ARGS @@ -40,14 +44,19 @@ endif fastly: dependencies $(GO_FILES) @GOHOSTOS="${GOHOSTOS}" GOHOSTARCH="${GOHOSTARCH}" goreleaser build ${GORELEASER_ARGS} -# useful for attaching a debugger such as https://github.com/go-delve/delve -debug: config +# Useful for attaching a debugger such as https://github.com/go-delve/delve +debug: @$(GO_BIN) build -gcflags="all=-N -l" $(GO_ARGS) -o "fastly" ./cmd/fastly +.PHONY: config +config: + @$(CONFIG_SCRIPT) + @cat $(CONFIG_FILE) + .PHONY: all -all: dependencies config tidy fmt vet staticcheck gosec test build install +all: config dependencies tidy fmt vet staticcheck gosec test build install -# update goreleaser inline with the release GHA workflow +# Update CI tools used by ./.github/workflows/pr_test.yml .PHONY: dependencies dependencies: $(GO_BIN) install github.com/securego/gosec/v2/cmd/gosec@latest @@ -55,54 +64,63 @@ dependencies: $(GO_BIN) install github.com/mgechev/revive@latest $(GO_BIN) install github.com/goreleaser/goreleaser@v1.9.2 +# Clean up Go modules file. .PHONY: tidy tidy: $(GO_BIN) mod tidy +# Run formatter. .PHONY: fmt fmt: @echo gofmt -l ./{cmd,pkg} @eval "bash -c 'F=\$$(gofmt -l ./{cmd,pkg}) ; if [[ \$$F ]] ; then echo \$$F ; exit 1 ; fi'" +# Run static analysis. .PHONY: vet -vet: +vet: config $(GO_BIN) vet ./{cmd,pkg}/... +# Run linter. .PHONY: revive revive: revive ./... +# Run security vulnerability checker. .PHONY: gosec gosec: gosec -quiet -exclude=G104 ./{cmd,pkg}/... +# Run third-party static analysis. .PHONY: staticcheck staticcheck: staticcheck ./{cmd,pkg}/... +# Run tests .PHONY: test test: config @$(TEST_COMMAND) -race $(TEST_ARGS) -# GO_ARGS allows for passing additional args to the build e.g. -# make build GO_ARGS='--ldflags "-s -w"' +# Compile program. +# +# GO_ARGS allows for passing additional arguments. +# e.g. make build GO_ARGS='--ldflags "-s -w"' .PHONY: build build: config $(GO_BIN) build $(GO_ARGS) ./cmd/fastly -# GO_ARGS allows for passing additional args to the build e.g. +# Compile and install program. +# +# GO_ARGS allows for passing additional arguments. .PHONY: install install: config $(GO_BIN) install $(GO_ARGS) ./cmd/fastly -.PHONY: config -config: - @curl -so pkg/config/config.toml https://developer.fastly.com/api/internal/cli-config - +# Scaffold a new CLI command from template files. .PHONY: scaffold scaffold: @$(shell pwd)/scripts/scaffold.sh $(CLI_PACKAGE) $(CLI_COMMAND) $(CLI_API) +# Scaffold a new CLI 'category' command from template files. .PHONY: scaffold-category scaffold-category: @$(shell pwd)/scripts/scaffold-category.sh $(CLI_CATEGORY) $(CLI_CATEGORY_COMMAND) $(CLI_PACKAGE) $(CLI_COMMAND) $(CLI_API) diff --git a/pkg/config/config.go b/pkg/config/config.go index d7b550727..21f8cf4ec 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -305,7 +305,7 @@ func (f *File) SetNonInteractive(v bool) { // NOTE: Static 👇 is public for the sake of the test suite. -// Static is the embedded configuration file used by the CLI.:was +// Static is the embedded configuration file used by the CLI. // //go:embed config.toml var Static []byte diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1181c1ab0..cfda44313 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -198,7 +198,10 @@ func TestUseStatic(t *testing.T) { // Validate that legacy configuration can be migrated to the static one // embedded in the CLI binary. f := config.File{} - f.Read(legacyUserConfigPath, strings.NewReader(""), &out, fsterr.MockLog{}, false) + err = f.Read(legacyUserConfigPath, strings.NewReader(""), &out, fsterr.MockLog{}, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } if f.CLI.Version == "" { t.Fatalf("expected CLI.Version to be set: %+v", f) diff --git a/scripts/config.sh b/scripts/config.sh new file mode 100755 index 000000000..8bcb79e4f --- /dev/null +++ b/scripts/config.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +cp ".fastly/config.toml" "pkg/config/config.toml" + +if ! command -v tomlq &> /dev/null +then + cargo install tomlq +fi + +kits=( + compute-starter-kit-rust-default + compute-starter-kit-rust-empty + compute-starter-kit-rust-static-content + compute-starter-kit-rust-websockets + compute-starter-kit-javascript-default + compute-starter-kit-javascript-empty + compute-starter-kit-assemblyscript-default + compute-starter-kit-go-default +) + +function parse() { + tomlq -f "$k.toml" $1 +} + +function append() { + echo $1 >> pkg/config/config.toml +} + +for k in ${kits[@]}; do + curl -s "https://raw.githubusercontent.com/fastly/$k/main/fastly.toml" -o "$k.toml" + + append "[[starter-kits.$(parse language)]]" + append "description = \"$(parse description)\"" + append "name = \"$(parse name)\"" + append "path = \"https://github.com/fastly/$k\"" + append '' + + rm "$k.toml" +done diff --git a/scripts/release-commits.sh b/scripts/release-commits.sh index d537d477b..bfcf35db1 100755 --- a/scripts/release-commits.sh +++ b/scripts/release-commits.sh @@ -34,8 +34,8 @@ arr+="]" # # so we do that using sed... -echo "::set-output name=commits::$(echo $arr | sed 's/},]/}]/')" +echo "commits=$(echo $arr | sed 's/},]/}]/')" >> $GITHUB_OUTPUT # example output: # -# ::set-output name=commits::[{"id":"afe0311974601664933a37391a9f78e8e2ee320b"},{"id":"2fc2c62f9a4851a7a54af9c6fe1946f8df0a77a7"},{"id":"151c6edfff93560bcaab28acc0757a01e1eb916f"},] +# commits=[{"id":"afe0311974601664933a37391a9f78e8e2ee320b"},{"id":"2fc2c62f9a4851a7a54af9c6fe1946f8df0a77a7"},{"id":"151c6edfff93560bcaab28acc0757a01e1eb916f"},]