From 536bacbea3cdfa598a534233b9f6206eb6051b45 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Thu, 21 Jul 2022 23:30:28 +0530 Subject: [PATCH 1/2] libgit2: upgrade to golang-with-libgit2 v0.1.4 Upgrade to golang-with-libgit2 v0.1.4 and use libgit2 which is compiled without being linked to libssh2 and openssl. Signed-off-by: Sanskar Jaiswal --- ATTRIBUTIONS.md | 90 ------------------------------------ Dockerfile | 9 ++-- Makefile | 25 ++-------- hack/install-libraries.sh | 17 +++---- hack/update-attributions.sh | 48 ------------------- tests/fuzz/oss_fuzz_build.sh | 19 ++++---- 6 files changed, 25 insertions(+), 183 deletions(-) diff --git a/ATTRIBUTIONS.md b/ATTRIBUTIONS.md index 5719c992d..696ab9fa4 100644 --- a/ATTRIBUTIONS.md +++ b/ATTRIBUTIONS.md @@ -5,61 +5,6 @@ code of their open source projects along with license information below. We acknowledge and are grateful to these developers for their contributions to open source. -## libssh2 - -Libssh2 was obtained in source-code form from its github repository: -https://github.com/libssh2/libssh2/ - -No changes were made to its original source code. - -Copyright notice (https://raw.githubusercontent.com/libssh2/libssh2/master/COPYING): - -/* Copyright (c) 2004-2007 Sara Golemon - * Copyright (c) 2005,2006 Mikhail Gusarov - * Copyright (c) 2006-2007 The Written Word, Inc. - * Copyright (c) 2007 Eli Fant - * Copyright (c) 2009-2021 Daniel Stenberg - * Copyright (C) 2008, 2009 Simon Josefsson - * Copyright (c) 2000 Markus Friedl - * Copyright (c) 2015 Microsoft Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted provided - * that the following conditions are met: - * - * Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * Neither the name of the copyright holder nor the names - * of any other contributors may be used to endorse or - * promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE - * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY - * OF SUCH DAMAGE. - */ - -*** - ## libgit2 Libgit2 was obtained in source-code form from its github repository: @@ -1254,38 +1199,3 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*** - -## zlib - -Zlib was obtained in binary form via official distribution channels. -No changes were made to its original source code. - -Copyright notice (https://zlib.net/zlib_license.html): - - /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.11, January 15th, 2017 - - Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - -*/ diff --git a/Dockerfile b/Dockerfile index e136ab476..d680009dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ ARG BASE_VARIANT=alpine ARG GO_VERSION=1.17 ARG XX_VERSION=1.1.0 -ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-all -ARG LIBGIT2_TAG=v0.1.2 +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only +ARG LIBGIT2_TAG=v0.1.4 FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs @@ -82,9 +82,8 @@ ENV CGO_ENABLED=1 # Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain. RUN export $(cat build/musl/$(xx-info alpine-arch).env | xargs) && \ - export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \ - export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \ - export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2) -static" && \ + export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \ + export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static" && \ GOARCH=$TARGETARCH go build \ -ldflags "-s -w" \ -tags 'netgo,osusergo,static_build' \ diff --git a/Makefile b/Makefile index e78406f32..e87344235 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ IMG ?= fluxcd/source-controller TAG ?= latest # Base image used to build the Go binary -LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-all -LIBGIT2_TAG ?= v0.1.2 +LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only +LIBGIT2_TAG ?= v0.1.4 # Allows for defining additional Go test args, e.g. '-tags integration'. GO_TEST_ARGS ?= -race @@ -32,33 +32,16 @@ ENVTEST_BIN_VERSION ?= 1.19.2 # Caches libgit2 versions per tag, "forcing" rebuild only when needed. LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG) LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib -LIBGIT2_LIB64_PATH := $(LIBGIT2_PATH)/lib64 LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a MUSL-CC = export CGO_ENABLED=1 export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig -export LIBRARY_PATH=$(LIBGIT2_LIB_PATH) -export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include -I$(LIBGIT2_PATH)/include/openssl - - -# The pkg-config command will yield warning messages until libgit2 is downloaded. -ifeq ($(shell uname -s),Darwin) -export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2 2>/dev/null) +export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null) GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))' -else -export PKG_CONFIG_PATH:=$(PKG_CONFIG_PATH):$(LIBGIT2_LIB64_PATH)/pkgconfig -export LIBRARY_PATH:=$(LIBRARY_PATH):$(LIBGIT2_LIB64_PATH) -export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2 2>/dev/null) -endif - ifeq ($(shell uname -s),Linux) -ifeq ($(shell uname -m),x86_64) -# Linux x86_64 seem to be able to cope with the static libraries -# by having only musl-dev installed, without the need of using musl toolchain. - GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))' -else +ifneq ($(shell uname -m),x86_64) MUSL-PREFIX=$(BUILD_DIR)/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl MUSL-CC=$(MUSL-PREFIX)-gcc export CC=$(MUSL-PREFIX)-gcc diff --git a/hack/install-libraries.sh b/hack/install-libraries.sh index be04af3b2..f39f60385 100755 --- a/hack/install-libraries.sh +++ b/hack/install-libraries.sh @@ -74,12 +74,12 @@ extract_libraries(){ } fix_pkgconfigs(){ - DIR="$1" NEW_DIR="$(/bin/pwd)/build/libgit2/${TAG}" # Update the prefix paths included in the .pc files. if [[ $OSTYPE == 'darwin'* ]]; then - INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}" + # https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L158 + INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/libgit2-darwin-amd64" # This will make it easier to update to the location in which they will be used. # sed has a sight different behaviour in MacOS @@ -90,7 +90,8 @@ fix_pkgconfigs(){ find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "" "s;${INSTALLED_DIR};${NEW_DIR};g" {} fi else - INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}" + # https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L52 + INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/build_libgit2_only" find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {} fi @@ -136,17 +137,17 @@ install_libraries(){ fi fi - FILE_NAME="linux-$(uname -m)-all-libs.tar.gz" - DIR="libgit2-linux-all-libs" + FILE_NAME="linux-x86_64-libgit2-only.tar.gz" + DIR="linux-libgit2-only" if [[ $OSTYPE == 'darwin'* ]]; then - FILE_NAME="darwin-all-libs.tar.gz" - DIR="darwin-all-libs" + FILE_NAME="darwin-libgit2-only.tar.gz" + DIR="darwin-libgit2-only" fi download_files "${FILE_NAME}" assure_provenance "${FILE_NAME}" extract_libraries "${FILE_NAME}" "${DIR}" - fix_pkgconfigs "${DIR}" + fix_pkgconfigs } install_libraries diff --git a/hack/update-attributions.sh b/hack/update-attributions.sh index 77c1df618..41c2d45ae 100755 --- a/hack/update-attributions.sh +++ b/hack/update-attributions.sh @@ -12,19 +12,6 @@ code of their open source projects along with license information below. We acknowledge and are grateful to these developers for their contributions to open source. -## libssh2 - -Libssh2 was obtained in source-code form from its github repository: -https://github.com/libssh2/libssh2/ - -No changes were made to its original source code. - -Copyright notice (https://raw.githubusercontent.com/libssh2/libssh2/master/COPYING): - -$(curl --max-time 5 -L https://raw.githubusercontent.com/libssh2/libssh2/master/COPYING) - -${SPLIT} - ## libgit2 Libgit2 was obtained in source-code form from its github repository: @@ -35,39 +22,4 @@ No changes were made to its original source code. Copyright notice (https://raw.githubusercontent.com/libgit2/libgit2/main/COPYING): $(curl --max-time 5 -L https://raw.githubusercontent.com/libgit2/libgit2/main/COPYING) - -${SPLIT} - -## zlib - -Zlib was obtained in binary form via official distribution channels. -No changes were made to its original source code. - -Copyright notice (https://zlib.net/zlib_license.html): - - /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.11, January 15th, 2017 - - Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - -*/ EOF diff --git a/tests/fuzz/oss_fuzz_build.sh b/tests/fuzz/oss_fuzz_build.sh index b58b94d4f..645946ef8 100755 --- a/tests/fuzz/oss_fuzz_build.sh +++ b/tests/fuzz/oss_fuzz_build.sh @@ -16,7 +16,7 @@ set -euxo pipefail -LIBGIT2_TAG="${LIBGIT2_TAG:-v0.1.2}" +LIBGIT2_TAG="${LIBGIT2_TAG:-v0.1.4}" GOPATH="${GOPATH:-/root/go}" GO_SRC="${GOPATH}/src" PROJECT_PATH="github.com/fluxcd/source-controller" @@ -28,9 +28,9 @@ export TARGET_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}" # For most cases, libgit2 will already be present. # The exception being at the oss-fuzz integration. if [ ! -d "${TARGET_DIR}" ]; then - curl -o output.tar.gz -LO "https://github.com/fluxcd/golang-with-libgit2/releases/download/${LIBGIT2_TAG}/linux-$(uname -m)-all-libs.tar.gz" + curl -o output.tar.gz -LO "https://github.com/fluxcd/golang-with-libgit2/releases/download/${LIBGIT2_TAG}/linux-x86_64-libgit2-only.tar.gz" - DIR=libgit2-linux-all-libs + DIR=linux-libgit2-only NEW_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}" INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}" @@ -49,10 +49,10 @@ fi apt-get update && apt-get install -y pkg-config export CGO_ENABLED=1 -export LIBRARY_PATH="${TARGET_DIR}/lib:${TARGET_DIR}/lib64" -export PKG_CONFIG_PATH="${TARGET_DIR}/lib/pkgconfig:${TARGET_DIR}/lib64/pkgconfig" -export CGO_CFLAGS="-I${TARGET_DIR}/include -I${TARGET_DIR}/include/openssl" -export CGO_LDFLAGS="$(pkg-config --libs --static --cflags libssh2 openssl libgit2)" +export PKG_CONFIG_PATH="${TARGET_DIR}/lib/pkgconfig" +export CGO_LDFLAGS="$(pkg-config --libs --static --cflags libgit2)" +export LIBRARY_PATH="${TARGET_DIR}/lib" +export CGO_CFLAGS="-I${TARGET_DIR}/include" go get -d github.com/AdaLogics/go-fuzz-headers @@ -92,10 +92,7 @@ function go_compile(){ else go-fuzz -tags gofuzz -func="${function}" -o "${fuzzer}.a" . ${CXX} ${CXXFLAGS} ${LIB_FUZZING_ENGINE} -o "${OUT}/${fuzzer}" \ - "${fuzzer}.a" \ - "${TARGET_DIR}/lib/libgit2.a" "${TARGET_DIR}/lib/libssh2.a" \ - "${TARGET_DIR}/lib/libz.a" "${TARGET_DIR}/lib64/libssl.a" \ - "${TARGET_DIR}/lib64/libcrypto.a" \ + "${fuzzer}.a" "${TARGET_DIR}/lib/libgit2.a" \ -fsanitize="${SANITIZER}" fi } From bd37bc31615eaa1641ff910ea575a12fc2c6e562 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Tue, 26 Jul 2022 15:10:44 +0530 Subject: [PATCH 2/2] build: bump tonistiigi/xx version Signed-off-by: Sanskar Jaiswal --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d680009dd..068938a20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_VARIANT=alpine ARG GO_VERSION=1.17 -ARG XX_VERSION=1.1.0 +ARG XX_VERSION=1.1.2 ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only ARG LIBGIT2_TAG=v0.1.4