From c6273e0bb25c39905bf4cadee82b143de53596e1 Mon Sep 17 00:00:00 2001 From: Moritz Wiesinger Date: Thu, 19 Dec 2024 10:00:18 +0100 Subject: [PATCH] test: run linux package tests nightly against latest contrib main [new] (#762) * previous commits * release pipeline fixes * fixes after rebasing Signed-off-by: Moritz Wiesinger * trigger build * downgrade ubuntu version back to 22.04 Signed-off-by: Moritz Wiesinger * try to restore state before rebase Signed-off-by: Moritz Wiesinger * run tests to ensure that everything works Signed-off-by: Moritz Wiesinger * revert after successful testing Signed-off-by: Moritz Wiesinger --------- Signed-off-by: Moritz Wiesinger --- .github/workflows/base-ci-goreleaser.yaml | 73 ++++++++++- ...age-tests.yaml => base-package-tests.yaml} | 0 .github/workflows/base-release.yaml | 30 +++++ .github/workflows/ci-goreleaser-contrib.yaml | 2 +- .github/workflows/ci-goreleaser-core.yaml | 2 +- .github/workflows/package-test.yaml | 24 ++++ cmd/goreleaser/internal/configure.go | 119 +++++++++++------- cmd/goreleaser/main.go | 10 +- .../otelcol-contrib/.goreleaser-build.yaml | 43 +++++++ .../otelcol-contrib/.goreleaser.yaml | 10 +- scripts/build.sh | 2 +- scripts/generate-goreleaser.sh | 4 + 12 files changed, 262 insertions(+), 57 deletions(-) rename .github/workflows/{package-tests.yaml => base-package-tests.yaml} (100%) create mode 100644 .github/workflows/package-test.yaml create mode 100644 distributions/otelcol-contrib/.goreleaser-build.yaml diff --git a/.github/workflows/base-ci-goreleaser.yaml b/.github/workflows/base-ci-goreleaser.yaml index bade00b9..6cc0723e 100644 --- a/.github/workflows/base-ci-goreleaser.yaml +++ b/.github/workflows/base-ci-goreleaser.yaml @@ -12,6 +12,11 @@ on: goarch: required: true type: string + nightly: + required: false + type: boolean + default: false + description: "Set to true to fetch latest otelcol-contrib main branch version instead of building the version in this repo" env: # renovate: datasource=github-tags depName=goreleaser-pro packageName=goreleaser/goreleaser-pro @@ -64,9 +69,75 @@ jobs: go-version: '1.23' check-latest: true - - name: Generate the sources + - name: Create artifacts directory to store build artifacts + if: inputs.distribution == 'otelcol-contrib' + run: mkdir -p distributions/otelcol-contrib/artifacts + + - name: "[Nightly] Get latest finished run ID from contrib repo build-and-test" + id: get-run-id + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux' + env: + GH_TOKEN: ${{ github.token }} + run: | + run_id=$(gh run list \ + --branch main \ + --workflow build-and-test \ + --repo open-telemetry/opentelemetry-collector-contrib \ + --limit 1 \ + --status success \ + --json databaseId \ + --jq '.[0].databaseId' \ + ) + echo "Found run ID: $run_id" + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + + - name: "[Nightly] Create sub-directory for otelcol-contrib nightly build" + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux' + run: mkdir -p distributions/otelcol-contrib/artifacts/otelcol-contrib_linux_amd64_v1 + + - name: "[Nightly] Download built otelcol-contrib artifact from contrib repo" + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux' + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: collector-binaries-linux-amd64 + repository: open-telemetry/opentelemetry-collector-contrib + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ steps.get-run-id.outputs.run_id }} + + - name: "[Nightly] Move downloaded artifact" + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux' + run: mv otelcontribcol_linux_amd64 distributions/otelcol-contrib/artifacts/otelcol-contrib_linux_amd64_v1/otelcol-contrib + + - name: Generate the sources for ${{ inputs.distribution }} + if: inputs.nightly != true + env: + DISTRIBUTIONS: ${{ inputs.distribution }} run: make generate-sources + # otelcol-contrib is built in a separate stage + - name: Build ${{ inputs.distribution }} + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true + uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0 + with: + distribution: goreleaser-pro + version: ${{ env.GORELEASER_PRO_VERSION }} + workdir: distributions/otelcol-contrib + args: --snapshot --clean --timeout 2h --split --config .goreleaser-build.yaml + env: + GOOS: ${{ matrix.GOOS }} + GOARCH: ${{ matrix.GOARCH }} + GOARM: 7 # Default is 6 + GOAMD64: v1 + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + + - name: Move built artifacts + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true + run: mv distributions/otelcol-contrib/dist/**/* distributions/otelcol-contrib/artifacts/ + + - name: Show built or downloaded content + if: inputs.distribution == 'otelcol-contrib' + run: ls -laR distributions/otelcol-contrib/artifacts + - name: Run GoReleaser for ${{ inputs.distribution }} uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0 with: diff --git a/.github/workflows/package-tests.yaml b/.github/workflows/base-package-tests.yaml similarity index 100% rename from .github/workflows/package-tests.yaml rename to .github/workflows/base-package-tests.yaml diff --git a/.github/workflows/base-release.yaml b/.github/workflows/base-release.yaml index 6969d1d9..3e63bebb 100644 --- a/.github/workflows/base-release.yaml +++ b/.github/workflows/base-release.yaml @@ -84,6 +84,36 @@ jobs: run: | echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + - name: Create artifacts directory to store build artifacts + if: inputs.distribution == 'otelcol-contrib' + run: mkdir -p distributions/otelcol-contrib/artifacts + + # otelcol-contrib is built in a separate stage + - name: Build ${{ inputs.distribution }} + if: inputs.distribution == 'otelcol-contrib' + uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0 + with: + distribution: goreleaser-pro + version: ${{ env.GORELEASER_PRO_VERSION }} + workdir: distributions/${{ inputs.distribution }} + args: release --clean --split --timeout 2h --config .goreleaser-build.yaml --release-header-tmpl=../../.github/release-template.md + env: + GOOS: ${{ matrix.GOOS }} + GOARCH: ${{ matrix.GOARCH }} + GOARM: 7 # Default is 6 + GOAMD64: v1 + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + COSIGN_YES: true + + - name: Move built artifacts + if: inputs.distribution == 'otelcol-contrib' + run: mv distributions/otelcol-contrib/dist/**/* distributions/otelcol-contrib/artifacts/ + + - name: Show built or downloaded content + if: inputs.distribution == 'otelcol-contrib' + run: ls -laR distributions/otelcol-contrib/artifacts + - uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0 with: distribution: goreleaser-pro diff --git a/.github/workflows/ci-goreleaser-contrib.yaml b/.github/workflows/ci-goreleaser-contrib.yaml index 71f731b4..9458c3e9 100644 --- a/.github/workflows/ci-goreleaser-contrib.yaml +++ b/.github/workflows/ci-goreleaser-contrib.yaml @@ -36,7 +36,7 @@ jobs: package-tests: name: Package tests needs: check-goreleaser - uses: ./.github/workflows/package-tests.yaml + uses: ./.github/workflows/base-package-tests.yaml with: distribution: otelcol-contrib type: '[ "deb", "rpm" ]' diff --git a/.github/workflows/ci-goreleaser-core.yaml b/.github/workflows/ci-goreleaser-core.yaml index 4e761948..6b232699 100644 --- a/.github/workflows/ci-goreleaser-core.yaml +++ b/.github/workflows/ci-goreleaser-core.yaml @@ -36,7 +36,7 @@ jobs: package-tests: name: Package tests needs: check-goreleaser - uses: ./.github/workflows/package-tests.yaml + uses: ./.github/workflows/base-package-tests.yaml with: distribution: otelcol type: '[ "deb", "rpm" ]' diff --git a/.github/workflows/package-test.yaml b/.github/workflows/package-test.yaml new file mode 100644 index 00000000..b0ab6cfc --- /dev/null +++ b/.github/workflows/package-test.yaml @@ -0,0 +1,24 @@ +name: Package Tests - Contrib + +on: + schedule: + - cron: "0 2 * * *" # every day at 2am UTC + +jobs: + check-goreleaser: + name: Build - Contrib - GoReleaser + uses: ./.github/workflows/base-ci-goreleaser.yaml + with: + distribution: otelcol-contrib + goos: '[ "linux" ]' + goarch: '[ "amd64" ]' + nightly: true + secrets: inherit + + package-tests: + name: Linux Package tests + needs: check-goreleaser + uses: ./.github/workflows/base-package-tests.yaml + with: + distribution: otelcol-contrib + type: '[ "deb", "rpm" ]' diff --git a/cmd/goreleaser/internal/configure.go b/cmd/goreleaser/internal/configure.go index 32741f16..75281bdb 100644 --- a/cmd/goreleaser/internal/configure.go +++ b/cmd/goreleaser/internal/configure.go @@ -42,7 +42,6 @@ const ( var ( ImagePrefixes = []string{DockerHub, GHCR} Architectures = []string{"386", "amd64", "arm", "arm64", "ppc64le", "s390x"} - ArmVersions = []string{"7"} DefaultConfigDists = map[string]bool{CoreDistro: true, ContribDistro: true} MSIWindowsDists = map[string]bool{CoreDistro: true, ContribDistro: true, OTLPDistro: true} K8sDockerSkipArchs = map[string]bool{"arm": true, "386": true} @@ -50,14 +49,25 @@ var ( K8sArchs = []string{"amd64", "arm64", "ppc64le", "s390x"} ) -func Generate(dist string) config.Project { +func GenerateContribBuildOnly(dist string, buildOrRest bool) config.Project { + return config.Project{ + ProjectName: "opentelemetry-collector-releases", + Builds: Builds(dist, buildOrRest), + Version: 2, + Monorepo: config.Monorepo{ + TagPrefix: "v", + }, + } +} + +func Generate(dist string, buildOrRest bool) config.Project { return config.Project{ ProjectName: "opentelemetry-collector-releases", Checksum: config.Checksum{ NameTemplate: fmt.Sprintf("{{ .ProjectName }}_%v_checksums.txt", dist), }, Env: []string{"COSIGN_YES=true"}, - Builds: Builds(dist), + Builds: Builds(dist, buildOrRest), Archives: Archives(dist), MSI: WinPackages(dist), NFPMs: Packages(dist), @@ -73,37 +83,41 @@ func Generate(dist string) config.Project { } } -func Builds(dist string) []config.Build { +func Builds(dist string, buildOrRest bool) []config.Build { return []config.Build{ - Build(dist), + Build(dist, buildOrRest), } } // Build configures a goreleaser build. // https://goreleaser.com/customization/build/ -func Build(dist string) config.Build { - var goos []string - var archs []string - var ignore []config.IgnoredBuild - var armVersions []string +func Build(dist string, buildOrRest bool) config.Build { + goos := []string{"darwin", "linux", "windows"} + archs := Architectures + + if dist == ContribDistro && !buildOrRest { + // only return build config for contrib build file + return config.Build{ + ID: dist, + Builder: "prebuilt", + PreBuilt: config.PreBuiltOptions{ + Path: "artifacts/otelcol-contrib_{{ .Target }}" + + "/otelcol-contrib{{- if eq .Os \"windows\" }}.exe{{ end }}", + }, + Goos: goos, + Goarch: archs, + Goarm: ArmVersions(dist), + Dir: "_build", + Binary: dist, + Ignore: IgnoreBuildCombinations(dist), + } + } + if dist == K8sDistro { goos = K8sGoos archs = K8sArchs - ignore = make([]config.IgnoredBuild, 0) - armVersions = make([]string, 0) - } else { - goos = []string{"darwin", "linux", "windows"} - archs = Architectures - ignore = []config.IgnoredBuild{ - {Goos: "darwin", Goarch: "386"}, - {Goos: "darwin", Goarch: "arm"}, - {Goos: "darwin", Goarch: "s390x"}, - {Goos: "windows", Goarch: "arm"}, - {Goos: "windows", Goarch: "arm64"}, - {Goos: "windows", Goarch: "s390x"}, - } - armVersions = ArmVersions } + return config.Build{ ID: dist, Dir: "_build", @@ -115,12 +129,33 @@ func Build(dist string) config.Build { }, Goos: goos, Goarch: archs, - Goarm: armVersions, - Ignore: ignore, + Goarm: ArmVersions(dist), + Ignore: IgnoreBuildCombinations(dist), } } -func Archives(dist string) (r []config.Archive) { +func IgnoreBuildCombinations(dist string) []config.IgnoredBuild { + if dist == K8sDistro { + return nil + } + return []config.IgnoredBuild{ + {Goos: "darwin", Goarch: "386"}, + {Goos: "darwin", Goarch: "arm"}, + {Goos: "darwin", Goarch: "s390x"}, + {Goos: "windows", Goarch: "arm"}, + {Goos: "windows", Goarch: "arm64"}, + {Goos: "windows", Goarch: "s390x"}, + } +} + +func ArmVersions(dist string) []string { + if dist == K8sDistro { + return nil + } + return []string{"7"} +} + +func Archives(dist string) []config.Archive { return []config.Archive{ Archive(dist), } @@ -138,7 +173,7 @@ func Archive(dist string) config.Archive { func WinPackages(dist string) []config.MSI { if _, ok := MSIWindowsDists[dist]; !ok { - return []config.MSI{} + return nil } return []config.MSI{ WinPackage(dist), @@ -160,9 +195,9 @@ func WinPackage(dist string) config.MSI { } } -func Packages(dist string) (r []config.NFPM) { +func Packages(dist string) []config.NFPM { if dist == K8sDistro { - return []config.NFPM{} + return nil } return []config.NFPM{ Package(dist), @@ -191,21 +226,17 @@ func Package(dist string) config.NFPM { }) } return config.NFPM{ - ID: dist, - Builds: []string{dist}, - Formats: []string{"deb", "rpm"}, - + ID: dist, + Builds: []string{dist}, + Formats: []string{"deb", "rpm"}, License: "Apache 2.0", Description: fmt.Sprintf("OpenTelemetry Collector - %s", dist), Maintainer: "The OpenTelemetry Collector maintainers ", Overrides: map[string]config.NFPMOverridables{ "rpm": { - Dependencies: []string{ - "/bin/sh", - }, + Dependencies: []string{"/bin/sh"}, }, }, - NFPMOverridables: config.NFPMOverridables{ PackageName: dist, Scripts: config.NFPMScripts{ @@ -219,16 +250,14 @@ func Package(dist string) config.NFPM { } func DockerImages(dist string) []config.Docker { - r := make([]config.Docker, 0) + var r []config.Docker for _, arch := range Architectures { - if dist == K8sDistro { - if _, ok := K8sDockerSkipArchs[arch]; ok { - continue - } + if dist == K8sDistro && K8sDockerSkipArchs[arch] { + continue } switch arch { case ArmArch: - for _, vers := range ArmVersions { + for _, vers := range ArmVersions(dist) { r = append(r, DockerImage(dist, arch, vers)) } default: @@ -302,7 +331,7 @@ func DockerManifest(prefix, version, dist string) config.DockerManifest { } switch arch { case ArmArch: - for _, armVers := range ArmVersions { + for _, armVers := range ArmVersions(dist) { dockerArchTag := strings.ReplaceAll(archName(arch, armVers), "/", "") imageTemplates = append( imageTemplates, diff --git a/cmd/goreleaser/main.go b/cmd/goreleaser/main.go index b466e06d..ae9ffcaa 100644 --- a/cmd/goreleaser/main.go +++ b/cmd/goreleaser/main.go @@ -19,12 +19,14 @@ import ( "log" "os" + "github.com/goreleaser/goreleaser-pro/v2/pkg/config" "gopkg.in/yaml.v3" "github.com/open-telemetry/opentelemetry-collector-releases/cmd/goreleaser/internal" ) var distFlag = flag.String("d", "", "Collector distributions to build") +var contribBuildOrRestFlag = flag.Bool("generate-build-step", false, "Collector Contrib distribution only - switch between build and package config file - set to true to generate build step, false to generate package step") func main() { flag.Parse() @@ -32,8 +34,14 @@ func main() { if len(*distFlag) == 0 { log.Fatal("no distribution to build") } + var project config.Project - project := internal.Generate(*distFlag) + if *distFlag == internal.ContribDistro && *contribBuildOrRestFlag { + // Special care needs to be taken for otelcol-contrib since it has a split setup + project = internal.GenerateContribBuildOnly(*distFlag, *contribBuildOrRestFlag) + } else { + project = internal.Generate(*distFlag, *contribBuildOrRestFlag) + } partial := map[string]any{ "partial": map[string]any{ diff --git a/distributions/otelcol-contrib/.goreleaser-build.yaml b/distributions/otelcol-contrib/.goreleaser-build.yaml new file mode 100644 index 00000000..b5f60c0d --- /dev/null +++ b/distributions/otelcol-contrib/.goreleaser-build.yaml @@ -0,0 +1,43 @@ +partial: + by: target +version: 2 +project_name: opentelemetry-collector-releases +builds: + - id: otelcol-contrib + goos: + - darwin + - linux + - windows + goarch: + - "386" + - amd64 + - arm + - arm64 + - ppc64le + - s390x + goarm: + - "7" + ignore: + - goos: darwin + goarch: "386" + - goos: darwin + goarch: arm + - goos: darwin + goarch: s390x + - goos: windows + goarch: arm + - goos: windows + goarch: arm64 + - goos: windows + goarch: s390x + dir: _build + binary: otelcol-contrib + ldflags: + - -s + - -w + flags: + - -trimpath + env: + - CGO_ENABLED=0 +monorepo: + tag_prefix: v diff --git a/distributions/otelcol-contrib/.goreleaser.yaml b/distributions/otelcol-contrib/.goreleaser.yaml index 593efbbc..6323ebea 100644 --- a/distributions/otelcol-contrib/.goreleaser.yaml +++ b/distributions/otelcol-contrib/.goreleaser.yaml @@ -41,13 +41,9 @@ builds: goarch: s390x dir: _build binary: otelcol-contrib - ldflags: - - -s - - -w - flags: - - -trimpath - env: - - CGO_ENABLED=0 + builder: prebuilt + prebuilt: + path: artifacts/otelcol-contrib_{{ .Target }}/otelcol-contrib{{- if eq .Os "windows" }}.exe{{ end }} archives: - id: otelcol-contrib builds: diff --git a/scripts/build.sh b/scripts/build.sh index bcf810b3..e6b325f4 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,7 +6,7 @@ BUILDER='' # default values skipcompilation=false -while getopts d:s:b:g: flag +while getopts d:s:b: flag do case "${flag}" in d) distributions=${OPTARG};; diff --git a/scripts/generate-goreleaser.sh b/scripts/generate-goreleaser.sh index 7b6c7ae7..e3262bd6 100755 --- a/scripts/generate-goreleaser.sh +++ b/scripts/generate-goreleaser.sh @@ -23,5 +23,9 @@ echo "Distributions to generate: $distributions"; for distribution in $(echo "$distributions" | tr "," "\n") do + if [[ "$distribution" == "otelcol-contrib" ]]; then + ${GO} run cmd/goreleaser/main.go -d "${distribution}" --generate-build-step > "./distributions/${distribution}/.goreleaser-build.yaml" + fi + ${GO} run cmd/goreleaser/main.go -d "${distribution}" > "./distributions/${distribution}/.goreleaser.yaml" done