diff --git a/Dockerfile b/Dockerfile index 4d30312d2..0c5f645d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG GO_VERSION=1.18 ARG XX_VERSION=1.1.2 ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only -ARG LIBGIT2_TAG=v0.1.4 +ARG LIBGIT2_TAG=v0.2.0 FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs @@ -37,22 +37,6 @@ COPY go.sum go.sum # Cache modules RUN go mod download -# The musl-tool-chain layer is an adhoc solution -# for the problem in which xx gets confused during compilation -# and a) looks for gold linker and then b) cannot find musl's dynamic linker. -FROM --platform=$BUILDPLATFORM alpine as musl-tool-chain - -COPY --from=xx / / - -RUN apk add bash curl tar - -WORKDIR /workspace -COPY hack/download-musl.sh . - -ARG TARGETPLATFORM -ARG TARGETARCH -RUN ROOT_DIR="$(pwd)" TARGET_ARCH="$(xx-info alpine-arch)" ENV_FILE=true \ - ./download-musl.sh # Build stage install per target platform # dependency and effectively cross compile the application. @@ -64,7 +48,7 @@ COPY --from=libgit2-libs /usr/local/ /usr/local/ # Some dependencies have to installed # for the target platform: https://github.com/tonistiigi/xx#go--cgo -RUN xx-apk add musl-dev gcc lld +RUN xx-apk add musl-dev gcc clang lld WORKDIR /workspace @@ -74,17 +58,14 @@ COPY controllers/ controllers/ COPY pkg/ pkg/ COPY internal/ internal/ -COPY --from=musl-tool-chain /workspace/build /workspace/build - ARG TARGETPLATFORM ARG TARGETARCH ENV CGO_ENABLED=1 # Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain. -RUN export $(cat build/musl/$(xx-info alpine-arch).env | xargs) && \ - export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \ - export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static" && \ - GOARCH=$TARGETARCH go build \ +RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \ + export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \ + xx-go build \ -ldflags "-s -w" \ -tags 'netgo,osusergo,static_build' \ -o /source-controller -trimpath main.go; diff --git a/Makefile b/Makefile index f783d6b2b..ed634cb81 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ TAG ?= latest # Base image used to build the Go binary LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only -LIBGIT2_TAG ?= v0.1.4 +LIBGIT2_TAG ?= v0.2.0 # Allows for defining additional Go test args, e.g. '-tags integration'. GO_TEST_ARGS ?= -race @@ -33,24 +33,12 @@ ENVTEST_BIN_VERSION ?= 1.19.2 LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG) LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a -MUSL-CC = export CGO_ENABLED=1 export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null) GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))' -ifeq ($(shell uname -s),Linux) -ifneq ($(shell uname -m),x86_64) - MUSL-PREFIX=$(BUILD_DIR)/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl - MUSL-CC=$(MUSL-PREFIX)-gcc - export CC=$(MUSL-PREFIX)-gcc - export CXX=$(MUSL-PREFIX)-g++ - export AR=$(MUSL-PREFIX)-ar - GO_STATIC_FLAGS=-ldflags "-s -w -extldflags \"-static\"" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))' -endif -endif - # API (doc) generation utilities CONTROLLER_GEN_VERSION ?= v0.7.0 GEN_API_REF_DOCS_VERSION ?= v0.3.0 @@ -179,15 +167,11 @@ install-envtest: setup-envtest ## Download envtest binaries locally. libgit2: $(LIBGIT2) ## Detect or download libgit2 library COSIGN = $(GOBIN)/cosign -$(LIBGIT2): $(MUSL-CC) +$(LIBGIT2): $(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest) IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh -$(MUSL-CC): -ifneq ($(shell uname -s),Darwin) - ./hack/download-musl.sh -endif .PHONY: help help: ## Display this help menu diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 5ab8c339e..011b5de7b 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -45,6 +45,7 @@ import ( dockerRegistry "github.com/distribution/distribution/v3/registry" _ "github.com/distribution/distribution/v3/registry/auth/htpasswd" _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" + git2go "github.com/libgit2/git2go/v33" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/fluxcd/source-controller/internal/cache" @@ -179,6 +180,8 @@ func setupRegistryServer(ctx context.Context) (*registryClientTestServer, error) } func TestMain(m *testing.M) { + mustHaveNoThreadSupport() + initTestTLS() utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme)) @@ -333,3 +336,22 @@ func randStringRunes(n int) string { func int64p(i int64) *int64 { return &i } + +// This provides a regression assurance for image-automation-controller/#339. +// Validates that: +// - libgit2 was built with no support for threads. +// - git2go accepts libgit2 built with no support for threads. +// +// The logic below does the validation of the former, whilst +// referring to git2go forces its init() execution, which is +// where any validation to that effect resides. +// +// git2go does not support threadless libgit2 by default, +// hence a fork is being used which disables such validation. +// +// TODO: extract logic into pkg. +func mustHaveNoThreadSupport() { + if git2go.Features()&git2go.FeatureThreads != 0 { + panic("libgit2 must not be build with thread support") + } +} diff --git a/go.mod b/go.mod index b430aa710..7e295c58d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,16 @@ go 1.18 replace github.com/fluxcd/source-controller/api => ./api +// A temporary fork of git2go was created to enable use +// of libgit2 without thread support to fix: +// fluxcd/image-automation-controller/#339. +// +// This can be removed once libgit2/git2go#918 is merged. +// +// The fork automatically releases new patches based on upstream: +// https://github.com/pjbgf/git2go/commit/d72e39cdc20f7fe014ba73072b01ba7b569e9253 +replace github.com/libgit2/git2go/v33 => github.com/pjbgf/git2go/v33 v33.0.9-nothread-check + require ( cloud.google.com/go/storage v1.23.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.22.0 diff --git a/go.sum b/go.sum index 9f02f316a..62b64234a 100644 --- a/go.sum +++ b/go.sum @@ -593,8 +593,6 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libgit2/git2go/v33 v33.0.9 h1:4ch2DJed6IhJO28BEohkUoGvxLsRzUjxljoNFJ6/O78= -github.com/libgit2/git2go/v33 v33.0.9/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= @@ -719,6 +717,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pjbgf/git2go/v33 v33.0.9-nothread-check h1:gSK7FaLECIM3VSuBOAsVZQtWd+51iTB5lv9RyxhOYMk= +github.com/pjbgf/git2go/v33 v33.0.9-nothread-check/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/hack/download-musl.sh b/hack/download-musl.sh deleted file mode 100755 index 3f5b527d2..000000000 --- a/hack/download-musl.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -set -eoux pipefail - -MUSL_X86_64_FILENAME=x86_64-linux-musl-native.tgz -MUSL_X86_64_SHA512=44d441ad9aa11a06feddf3daa4c9f53ad7d9ca37af1f5a61379aca07793703d179410cea723c1b7fca94c4de19a321228bdb3656bc5cbdb5e3bea8e2d6dac6c7 -MUSL_AARCH64_FILENAME=aarch64-linux-musl-native.tgz -MUSL_AARCH64_SHA512=16d544e09845c9dbba50f29e0cb04dd661e17eb63c56acad6a67fd2a78aa7596b792477c7177d3cd56d408a27dc291a90507df882f2b099c0f25511ce08fd3b5 -MUSL_XX86_64_FILENAME=x86_64-linux-musl-cross.tgz -MUSL_XX86_64_SHA512=52abd1a56e670952116e35d1a62e048a9b6160471d988e16fa0e1611923dd108a581d2e00874af5eb04e4968b1ba32e0eb449a1f15c3e4d5240ebe09caf5a9f3 -MUSL_XAARCH64_FILENAME=aarch64-linux-musl-cross.tgz -MUSL_XAARCH64_SHA512=8695ff86979cdf30fbbcd33061711f5b1ebc3c48a87822b9ca56cde6d3a22abd4dab30fdcd1789ac27c6febbaeb9e5bde59d79d66552fae53d54cc1377a19272 -MUSL_XARMV7_FILENAME=armv7l-linux-musleabihf-cross.tgz -MUSL_XARMV7_SHA512=1bb399a61da425faac521df9b8d303e60ad101f6c7827469e0b4bc685ce1f3dedc606ac7b1e8e34d79f762a3bfe3e8ab479a97e97d9f36fbd9fc5dc9d7ed6fd1 - -TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}" -ENV_FILE="${ENV_FILE:-false}" - -MUSL_FILENAME="" -MUSL_SHA512="" - -ROOT_DIR="${ROOT_DIR:-$(git rev-parse --show-toplevel)}" -MUSL_DIR="${ROOT_DIR}/build/musl" - - -if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then - MUSL_FILENAME="${MUSL_X86_64_FILENAME}" - MUSL_SHA512="${MUSL_X86_64_SHA512}" - MUSL_PREFIX="${TARGET_ARCH}-linux-musl-native/bin/${TARGET_ARCH}-linux-musl" - if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then - MUSL_FILENAME="${MUSL_AARCH64_FILENAME}" - MUSL_SHA512="${MUSL_AARCH64_SHA512}" - fi -else - MUSL_FILENAME="${MUSL_XX86_64_FILENAME}" - MUSL_SHA512="${MUSL_XX86_64_SHA512}" - MUSL_PREFIX="${TARGET_ARCH}-linux-musl-cross/bin/${TARGET_ARCH}-linux-musl" - if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then - MUSL_FILENAME="${MUSL_XAARCH64_FILENAME}" - MUSL_SHA512="${MUSL_XAARCH64_SHA512}" - elif [ "${TARGET_ARCH}" = "arm" ] || [ "${TARGET_ARCH}" = "armv7" ]; then - MUSL_FILENAME="${MUSL_XARMV7_FILENAME}" - MUSL_SHA512="${MUSL_XARMV7_SHA512}" - MUSL_PREFIX=armv7l-linux-musleabihf-cross/bin/armv7l-linux-musleabihf - fi -fi - -mkdir -p "${MUSL_DIR}" - -if "${ENV_FILE}"; then - cat< "${MUSL_DIR}/${TARGET_ARCH}.env" -CC="$(pwd)/build/musl/${MUSL_PREFIX}-gcc" -CXX="$(pwd)/build/musl/${MUSL_PREFIX}-g++" -AR="$(pwd)/build/musl/${MUSL_PREFIX}-ar" -EOF -fi - -MUSL_AARCH64_URL="https://more.musl.cc/11.2.1/x86_64-linux-musl/${MUSL_FILENAME}" - -if [ ! -f "${MUSL_DIR}/bin" ]; then - TARGET_FILE="${MUSL_DIR}/${MUSL_FILENAME}" - curl -o "${TARGET_FILE}" -LO "${MUSL_AARCH64_URL}" - if ! echo "${MUSL_SHA512} ${TARGET_FILE}" | sha512sum; then - echo "Checksum failed for ${MUSL_FILENAME}." - rm -rf "${MUSL_DIR}" - exit 1 - fi - - tar xzf "${TARGET_FILE}" -C "${MUSL_DIR}" - rm "${TARGET_FILE}" -fi diff --git a/hack/install-libraries.sh b/hack/install-libraries.sh index f39f60385..afec8bc97 100755 --- a/hack/install-libraries.sh +++ b/hack/install-libraries.sh @@ -79,7 +79,7 @@ fix_pkgconfigs(){ # Update the prefix paths included in the .pc files. if [[ $OSTYPE == 'darwin'* ]]; then # https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L158 - INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/libgit2-darwin-amd64" + INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/darwin-libgit2-only" # This will make it easier to update to the location in which they will be used. # sed has a sight different behaviour in MacOS @@ -137,7 +137,7 @@ install_libraries(){ fi fi - FILE_NAME="linux-x86_64-libgit2-only.tar.gz" + FILE_NAME="linux-$(uname -m)-libgit2-only.tar.gz" DIR="linux-libgit2-only" if [[ $OSTYPE == 'darwin'* ]]; then FILE_NAME="darwin-libgit2-only.tar.gz" diff --git a/tests/fuzz/oss_fuzz_build.sh b/tests/fuzz/oss_fuzz_build.sh index 645946ef8..2284cf579 100755 --- a/tests/fuzz/oss_fuzz_build.sh +++ b/tests/fuzz/oss_fuzz_build.sh @@ -16,7 +16,7 @@ set -euxo pipefail -LIBGIT2_TAG="${LIBGIT2_TAG:-v0.1.4}" +LIBGIT2_TAG="${LIBGIT2_TAG:-v0.2.0}" GOPATH="${GOPATH:-/root/go}" GO_SRC="${GOPATH}/src" PROJECT_PATH="github.com/fluxcd/source-controller"