diff --git a/Dockerfile b/Dockerfile index 6cfbc0c5..b2d47835 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ ARG BASE_VARIANT=alpine ARG GO_VERSION=1.18 -ARG XX_VERSION=1.1.0 +ARG XX_VERSION=1.1.2 -ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-all -ARG LIBGIT2_TAG=v0.1.2 +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only +ARG LIBGIT2_TAG=v0.2.0 FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs @@ -17,7 +17,7 @@ FROM gostable AS go-linux # These will be used at current arch to yield execute the cross compilations. FROM go-${TARGETOS} AS build-base -RUN apk add --no-cache clang lld pkgconfig +RUN apk add clang lld pkgconfig COPY --from=xx / / @@ -37,23 +37,6 @@ COPY go.sum go.sum # Cache modules RUN go mod download -# The musl-tool-chain layer is an adhoc solution -# for the problem in which xx gets confused during compilation -# and a) looks for gold linker and then b) cannot find musl's dynamic linker. -FROM --platform=$BUILDPLATFORM alpine as musl-tool-chain - -COPY --from=xx / / - -RUN apk add bash curl tar - -WORKDIR /workspace -COPY hack/download-musl.sh . - -ARG TARGETPLATFORM -ARG TARGETARCH -RUN ROOT_DIR="$(pwd)" TARGET_ARCH="$(xx-info alpine-arch)" ENV_FILE=true \ - ./download-musl.sh - # Build stage install per target platform # dependency and effectively cross compile the application. FROM build-go-mod as build @@ -64,7 +47,7 @@ COPY --from=libgit2-libs /usr/local/ /usr/local/ # Some dependencies have to installed # for the target platform: https://github.com/tonistiigi/xx#go--cgo -RUN xx-apk add musl-dev gcc lld +RUN xx-apk add musl-dev gcc clang lld WORKDIR /workspace @@ -74,20 +57,14 @@ COPY controllers/ controllers/ COPY pkg/ pkg/ COPY internal/ internal/ -COPY --from=musl-tool-chain /workspace/build /workspace/build - ARG TARGETPLATFORM ARG TARGETARCH ENV CGO_ENABLED=1 -# Performance related changes: -# - Use read-only bind instead of copying go source files. -# - Cache go packages. -RUN export $(cat build/musl/$(xx-info alpine-arch).env | xargs) && \ - export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \ - export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \ - export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2) -static" && \ - GOARCH=$TARGETARCH go build \ +RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)" && \ + export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \ + export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \ + xx-go build \ -ldflags "-s -w" \ -tags 'netgo,osusergo,static_build' \ -o /image-automation-controller -trimpath main.go; diff --git a/Makefile b/Makefile index b3461e6e..b0386cce 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,8 @@ TAG ?= latest CRD_OPTIONS ?= crd:crdVersions=v1 # Base image used to build the Go binary -LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-all -LIBGIT2_TAG ?= v0.1.2 +LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only +LIBGIT2_TAG ?= v0.2.0 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. @@ -42,37 +42,20 @@ LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG) LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib LIBGIT2_LIB64_PATH := $(LIBGIT2_PATH)/lib64 LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a -MUSL-CC = export CGO_ENABLED=1 export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig export LIBRARY_PATH=$(LIBGIT2_LIB_PATH) export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include -I$(LIBGIT2_PATH)/include/openssl - +export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null) # The pkg-config command will yield warning messages until libgit2 is downloaded. ifeq ($(shell uname -s),Darwin) -export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2 2>/dev/null) GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build' -else -export PKG_CONFIG_PATH:=$(PKG_CONFIG_PATH):$(LIBGIT2_LIB64_PATH)/pkgconfig -export LIBRARY_PATH:=$(LIBRARY_PATH):$(LIBGIT2_LIB64_PATH) -export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2 2>/dev/null) endif ifeq ($(shell uname -s),Linux) -ifeq ($(shell uname -m),x86_64) -# Linux x86_64 seem to be able to cope with the static libraries -# by having only musl-dev installed, without the need of using musl toolchain. GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build' -else - MUSL-PREFIX=$(BUILD_DIR)/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl - MUSL-CC=$(MUSL-PREFIX)-gcc - export CC=$(MUSL-PREFIX)-gcc - export CXX=$(MUSL-PREFIX)-g++ - export AR=$(MUSL-PREFIX)-ar - GO_STATIC_FLAGS=-ldflags "-s -w -extldflags \"-static\"" -tags 'netgo,osusergo,static_build' -endif endif # API (doc) generation utilities @@ -213,16 +196,11 @@ controller-gen: ## Download controller-gen locally if necessary. libgit2: $(LIBGIT2) ## Detect or download libgit2 library COSIGN = $(GOBIN)/cosign -$(LIBGIT2): $(MUSL-CC) +$(LIBGIT2): $(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest) IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh -$(MUSL-CC): -ifneq ($(shell uname -s),Darwin) - ./hack/download-musl.sh -endif - # Find or download gen-crd-api-reference-docs GEN_CRD_API_REFERENCE_DOCS = $(GOBIN)/gen-crd-api-reference-docs .PHONY: gen-crd-api-reference-docs diff --git a/controllers/suite_test.go b/controllers/suite_test.go index f28695e8..aaf00f2d 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + git2go "github.com/libgit2/git2go/v33" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -53,6 +54,8 @@ func init() { } func TestMain(m *testing.M) { + mustHaveNoThreadSupport() + utilruntime.Must(imagev1_reflect.AddToScheme(scheme.Scheme)) utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme)) utilruntime.Must(imagev1.AddToScheme(scheme.Scheme)) @@ -90,3 +93,22 @@ func TestMain(m *testing.M) { os.Exit(code) } + +// This provides a regression assurance for image-automation-controller/#339. +// Validates that: +// - libgit2 was built with no support for threads. +// - git2go accepts libgit2 built with no support for threads. +// +// The logic below does the validation of the former, whilst +// referring to git2go forces its init() execution, which is +// where any validation to that effect resides. +// +// git2go does not support threadless libgit2 by default, +// hence a fork is being used which disables such validation. +// +// TODO: extract logic into pkg. +func mustHaveNoThreadSupport() { + if git2go.Features()&git2go.FeatureThreads != 0 { + panic("libgit2 must not be build with thread support") + } +} diff --git a/go.mod b/go.mod index 6a9f0ad6..24bb4865 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,16 @@ go 1.18 replace github.com/fluxcd/image-automation-controller/api => ./api +// A temporary fork of git2go was created to enable use +// of libgit2 without thread support to fix: +// fluxcd/image-automation-controller/#339. +// +// This can be removed once libgit2/git2go#918 is merged. +// +// The fork automatically releases new patches based on upstream: +// https://github.com/pjbgf/git2go/commit/d72e39cdc20f7fe014ba73072b01ba7b569e9253 +replace github.com/libgit2/git2go/v33 => github.com/pjbgf/git2go/v33 v33.0.9-nothread-check + require ( github.com/Masterminds/sprig/v3 v3.2.2 github.com/ProtonMail/go-crypto v0.0.0-20220714114130-e85cedf506cd diff --git a/go.sum b/go.sum index 0dc9dbad..246d6670 100644 --- a/go.sum +++ b/go.sum @@ -449,8 +449,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/libgit2/git2go/v33 v33.0.9 h1:4ch2DJed6IhJO28BEohkUoGvxLsRzUjxljoNFJ6/O78= -github.com/libgit2/git2go/v33 v33.0.9/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -536,6 +534,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pjbgf/git2go/v33 v33.0.9-nothread-check h1:gSK7FaLECIM3VSuBOAsVZQtWd+51iTB5lv9RyxhOYMk= +github.com/pjbgf/git2go/v33 v33.0.9-nothread-check/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/hack/download-musl.sh b/hack/download-musl.sh deleted file mode 100755 index 3f5b527d..00000000 --- a/hack/download-musl.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -set -eoux pipefail - -MUSL_X86_64_FILENAME=x86_64-linux-musl-native.tgz -MUSL_X86_64_SHA512=44d441ad9aa11a06feddf3daa4c9f53ad7d9ca37af1f5a61379aca07793703d179410cea723c1b7fca94c4de19a321228bdb3656bc5cbdb5e3bea8e2d6dac6c7 -MUSL_AARCH64_FILENAME=aarch64-linux-musl-native.tgz -MUSL_AARCH64_SHA512=16d544e09845c9dbba50f29e0cb04dd661e17eb63c56acad6a67fd2a78aa7596b792477c7177d3cd56d408a27dc291a90507df882f2b099c0f25511ce08fd3b5 -MUSL_XX86_64_FILENAME=x86_64-linux-musl-cross.tgz -MUSL_XX86_64_SHA512=52abd1a56e670952116e35d1a62e048a9b6160471d988e16fa0e1611923dd108a581d2e00874af5eb04e4968b1ba32e0eb449a1f15c3e4d5240ebe09caf5a9f3 -MUSL_XAARCH64_FILENAME=aarch64-linux-musl-cross.tgz -MUSL_XAARCH64_SHA512=8695ff86979cdf30fbbcd33061711f5b1ebc3c48a87822b9ca56cde6d3a22abd4dab30fdcd1789ac27c6febbaeb9e5bde59d79d66552fae53d54cc1377a19272 -MUSL_XARMV7_FILENAME=armv7l-linux-musleabihf-cross.tgz -MUSL_XARMV7_SHA512=1bb399a61da425faac521df9b8d303e60ad101f6c7827469e0b4bc685ce1f3dedc606ac7b1e8e34d79f762a3bfe3e8ab479a97e97d9f36fbd9fc5dc9d7ed6fd1 - -TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}" -ENV_FILE="${ENV_FILE:-false}" - -MUSL_FILENAME="" -MUSL_SHA512="" - -ROOT_DIR="${ROOT_DIR:-$(git rev-parse --show-toplevel)}" -MUSL_DIR="${ROOT_DIR}/build/musl" - - -if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then - MUSL_FILENAME="${MUSL_X86_64_FILENAME}" - MUSL_SHA512="${MUSL_X86_64_SHA512}" - MUSL_PREFIX="${TARGET_ARCH}-linux-musl-native/bin/${TARGET_ARCH}-linux-musl" - if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then - MUSL_FILENAME="${MUSL_AARCH64_FILENAME}" - MUSL_SHA512="${MUSL_AARCH64_SHA512}" - fi -else - MUSL_FILENAME="${MUSL_XX86_64_FILENAME}" - MUSL_SHA512="${MUSL_XX86_64_SHA512}" - MUSL_PREFIX="${TARGET_ARCH}-linux-musl-cross/bin/${TARGET_ARCH}-linux-musl" - if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then - MUSL_FILENAME="${MUSL_XAARCH64_FILENAME}" - MUSL_SHA512="${MUSL_XAARCH64_SHA512}" - elif [ "${TARGET_ARCH}" = "arm" ] || [ "${TARGET_ARCH}" = "armv7" ]; then - MUSL_FILENAME="${MUSL_XARMV7_FILENAME}" - MUSL_SHA512="${MUSL_XARMV7_SHA512}" - MUSL_PREFIX=armv7l-linux-musleabihf-cross/bin/armv7l-linux-musleabihf - fi -fi - -mkdir -p "${MUSL_DIR}" - -if "${ENV_FILE}"; then - cat< "${MUSL_DIR}/${TARGET_ARCH}.env" -CC="$(pwd)/build/musl/${MUSL_PREFIX}-gcc" -CXX="$(pwd)/build/musl/${MUSL_PREFIX}-g++" -AR="$(pwd)/build/musl/${MUSL_PREFIX}-ar" -EOF -fi - -MUSL_AARCH64_URL="https://more.musl.cc/11.2.1/x86_64-linux-musl/${MUSL_FILENAME}" - -if [ ! -f "${MUSL_DIR}/bin" ]; then - TARGET_FILE="${MUSL_DIR}/${MUSL_FILENAME}" - curl -o "${TARGET_FILE}" -LO "${MUSL_AARCH64_URL}" - if ! echo "${MUSL_SHA512} ${TARGET_FILE}" | sha512sum; then - echo "Checksum failed for ${MUSL_FILENAME}." - rm -rf "${MUSL_DIR}" - exit 1 - fi - - tar xzf "${TARGET_FILE}" -C "${MUSL_DIR}" - rm "${TARGET_FILE}" -fi diff --git a/hack/install-libraries.sh b/hack/install-libraries.sh index be04af3b..2399c483 100755 --- a/hack/install-libraries.sh +++ b/hack/install-libraries.sh @@ -136,11 +136,11 @@ install_libraries(){ fi fi - FILE_NAME="linux-$(uname -m)-all-libs.tar.gz" - DIR="libgit2-linux-all-libs" + FILE_NAME="linux-$(uname -m)-libgit2-only.tar.gz" + DIR="linux-libgit2-only" if [[ $OSTYPE == 'darwin'* ]]; then - FILE_NAME="darwin-all-libs.tar.gz" - DIR="darwin-all-libs" + FILE_NAME="darwin-libgit2-only.tar.gz" + DIR="darwin-libgit2-only" fi download_files "${FILE_NAME}" diff --git a/tests/fuzz/oss_fuzz_build.sh b/tests/fuzz/oss_fuzz_build.sh index 5a9a62cf..f7ff272a 100755 --- a/tests/fuzz/oss_fuzz_build.sh +++ b/tests/fuzz/oss_fuzz_build.sh @@ -16,7 +16,7 @@ set -euxo pipefail -LIBGIT2_TAG="${LIBGIT2_TAG:-v0.1.2}" +LIBGIT2_TAG="${LIBGIT2_TAG:-v0.2.0}" GOPATH="${GOPATH:-/root/go}" GO_SRC="${GOPATH}/src" PROJECT_PATH="github.com/fluxcd/image-automation-controller" @@ -28,9 +28,9 @@ export TARGET_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}" # For most cases, libgit2 will already be present. # The exception being at the oss-fuzz integration. if [ ! -d "${TARGET_DIR}" ]; then - curl -o output.tar.gz -LO "https://github.com/fluxcd/golang-with-libgit2/releases/download/${LIBGIT2_TAG}/linux-$(uname -m)-all-libs.tar.gz" + curl -o output.tar.gz -LO "https://github.com/fluxcd/golang-with-libgit2/releases/download/${LIBGIT2_TAG}/linux-$(uname -m)-libgit2-only.tar.gz" - DIR=libgit2-linux-all-libs + DIR=linux-libgit2-only NEW_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}" INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}"