diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c2b7d63..732e4f4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,14 +4,99 @@ on: push: branches: - main + - testdarwin2 tags: - '**' - paths-ignore: - - README.md - pull_request: - paths-ignore: - - README.md + # paths-ignore: + # - README.md + # pull_request: + # paths-ignore: + # - README.md + +permissions: + contents: write # needed to write releases + packages: write # needed for ghcr access + jobs: + + mac-build: + # This job builds and releases "universal libraries" that are + # supported by both darwin-amd64 and darwin-arm64. + # + # First builds in amd64, then cross-compile in arm64. Later combining + # both outcomes onto a single binary for each static library. + # + # `macos-11` has been picked as support for arm64 was only added on Xcode 12. + # Although some minor versions of Catalina 10.15 can support it, at the time + # of testing, GitHub's macos-10.15 did not seem to. + # Cross-compiling to arm64 on that runner consistently failed. + runs-on: macos-11 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build universal static libraries for Darwin + run: | + TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64 \ + BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/amd \ + ./hack/static.sh all + + TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64 \ + BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/arm \ + TARGET_ARCH=arm64 \ + CMAKE_APPLE_SILICON_PROCESSOR=arm64 \ + ./hack/static.sh all + + mkdir -p ./libgit2-darwin/lib + mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/include ./libgit2-darwin/ + mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/share ./libgit2-darwin/ + mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/cmake ./libgit2-darwin/lib/ + mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/engines-3 ./libgit2-darwin/lib/ + mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/ossl-modules ./libgit2-darwin/lib/ + mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/pkgconfig ./libgit2-darwin/lib/ + + libtool -static -o ./libgit2-darwin/lib/libcrypto.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libcrypto.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libcrypto.a + libtool -static -o ./libgit2-darwin/lib/libgit2.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libgit2.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libgit2.a + libtool -static -o ./libgit2-darwin/lib/libssh2.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssh2.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssh2.a + libtool -static -o ./libgit2-darwin/lib/libssl.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssl.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssl.a + libtool -static -o ./libgit2-darwin/lib/libz.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libz.a \ + ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libz.a + + tar -zcvf darwin-libs.tar.gz ./libgit2-darwin + env: + MACOSX_DEPLOYMENT_TARGET: 10.15 + + - name: Create Release + if: github.event_name != 'pull_request' + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + release_name: ${{ github.ref }} + tag_name: ${{ github.ref }} + draft: false + prerelease: true + - name: Upload Release Asset + if: github.event_name != 'pull_request' + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./darwin-libs.tar.gz + asset_name: darwin-libs.tar.gz + asset_content_type: application/gzip + build: runs-on: ubuntu-latest env: @@ -55,7 +140,7 @@ jobs: key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-ghcache- - - run: cat ./hack/Makefile + - run: cat ./hack/static.sh - name: Build candidate image id: build_candidate uses: docker/build-push-action@v2 diff --git a/Dockerfile b/Dockerfile index 11bd33b..82304d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # This Dockerfile tests the hack/Makefile output against git2go. ARG BASE_VARIANT=alpine -ARG GO_VERSION=1.17.6 +ARG GO_VERSION=1.17 ARG XX_VERSION=1.1.0 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx diff --git a/Dockerfile.test b/Dockerfile.test index 8c92d87..29e862d 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,13 +1,87 @@ +# This Dockerfile tests the hack/Makefile output against git2go. ARG BASE_VARIANT=alpine ARG GO_VERSION=1.17 ARG XX_VERSION=1.1.0 -ARG LIBGIT2_IMG -ARG LIBGIT2_TAG +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx -FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS build-deps +FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-base + +RUN apk add --no-cache \ + bash \ + curl \ + build-base \ + linux-headers \ + perl \ + cmake \ + pkgconfig \ + gcc \ + musl-dev \ + clang \ + lld -FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx +COPY --from=xx / / + +FROM build-base AS build-cross + +ARG TARGETPLATFORM + +RUN xx-apk add --no-cache \ + build-base \ + pkgconfig \ + gcc \ + musl-dev \ + clang \ + lld \ + llvm \ + linux-headers + +WORKDIR /build +COPY hack/static.sh . + +ENV CC=xx-clang +ENV CXX=xx-clang++ + +RUN CHOST=$(xx-clang --print-target-triple) \ + ./static.sh build_libz + +RUN CHOST=$(xx-clang --print-target-triple) \ + ./static.sh build_openssl + +RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \ + export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \ + export OPENSSL_ROOT_DIR="/usr/local/$(xx-info triple)" && \ + export OPENSSL_CRYPTO_LIBRARY="/usr/local/$(xx-info triple)/lib64" && \ + export OPENSSL_INCLUDE_DIR="/usr/local/$(xx-info triple)/include/openssl" + +RUN ./static.sh build_libssh2 +RUN ./static.sh build_libgit2 + + +# trimmed removes all non necessary files (i.e. openssl binary). +FROM build-cross AS trimmed + +ARG TARGETPLATFORM +RUN mkdir -p /trimmed/usr/local/$(xx-info triple)/ && \ + mkdir -p /trimmed/usr/local/$(xx-info triple)/share + +RUN cp -r /usr/local/$(xx-info triple)/lib/ /trimmed/usr/local/$(xx-info triple)/ && \ + cp -r /usr/local/$(xx-info triple)/lib64/ /trimmed/usr/local/$(xx-info triple)/ | true && \ + cp -r /usr/local/$(xx-info triple)/include/ /trimmed/usr/local/$(xx-info triple)/ && \ + cp -r /usr/local/$(xx-info triple)/share/doc/ /trimmed/usr/local/$(xx-info triple)/share/ + +FROM scratch as libs-arm64 +COPY --from=trimmed /trimmed/ / + +FROM scratch as libs-amd64 +COPY --from=trimmed /trimmed/ / + +FROM scratch as libs-armv7 +COPY --from=trimmed /trimmed/ / + +FROM libs-$TARGETARCH$TARGETVARIANT as libs + +# Everything above this line is a copy from Dockefile. FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable @@ -15,14 +89,14 @@ FROM gostable AS go-linux # Build-base consists of build platform dependencies and xx. # These will be used at current arch to yield execute the cross compilations. -FROM go-${TARGETOS} AS build-base +FROM go-${TARGETOS} AS go-base RUN apk add clang lld pkgconfig COPY --from=xx / / # build-go-mod can still be cached at build platform architecture. -FROM build-base as build-go-mod +FROM go-base as build-go-mod WORKDIR /root/smoketest COPY tests/smoketest/go.mod . @@ -42,7 +116,7 @@ RUN xx-apk add musl-dev gcc clang lld WORKDIR /root/smoketest COPY tests/smoketest/main.go . -COPY --from=build-deps /usr/local/ /usr/local/ +COPY --from=libs /usr/local/ /usr/local/ ENV CGO_ENABLED=1 RUN export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \ diff --git a/README.md b/README.md index ed368d9..62b1066 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # golang-with-libgit2 -This repository contains a `Dockerfile` with two files: `Makefile` and `static.sh`. +This repository contains a `Dockerfile` with the statically built libgit2 and its dependency chain. + +The `hack` directory contains two main files: `Makefile` and `static.sh`. Both of which can be used to build the [libgit2][] dependency chain for **AMD64, ARM64 and ARMv7** binaries of Go projects that depend on [git2go][]. @@ -8,6 +10,8 @@ The `Makefile` is useful for development environments and will leverage OS speci The `static.sh` will build all `libgit2` dependencies from source using `musl` toolchain. This enables for a full static binary with the freedom of configuring each of the dependencies in chain. +Alternatively, the statically built libraries can be pulling from the produced images for Linux or from the github release artifacts for MacOS. + ### :warning: **Public usage discouraged** The set of dependencies was handpicked for the Flux project, based on the issue list documented below. While this setup @@ -47,17 +51,14 @@ while testing these against the git2go code before releasing the image. - [ ] [libgit2/git2go#836](https://github.com/libgit2/git2go/issues/836) - [ ] [libgit2/git2go#837](https://github.com/libgit2/git2go/issues/837) ---- -**NOTE** -The issues above do not affect libgit2 built with `static.sh` as all its +> **NOTE:** The issues above do not affect libgit2 built with `static.sh` as all its dependencies have been configured to be optimal for its use, as the first supported version of libgit2 is `1.3.0`. ---- ## Usage -The [Dockerfile.test](./Dockerfile.test) file provides a working example on how to statically build a golang application that has a dependency to libgit2 and git2go. +The [Dockerfile.test](./Dockerfile.test) file provides a working example on how to statically build a golang application that has a dependency on libgit2 and git2go. The example will statically build all dependencies based on the versions specified on `static.sh`. Then statically build the golang application and deploy it into an image based off `gcr.io/distroless/static`. diff --git a/hack/static.sh b/hack/static.sh index 119c814..6348569 100755 --- a/hack/static.sh +++ b/hack/static.sh @@ -14,7 +14,7 @@ TARGET_DIR="${TARGET_DIR:-/usr/local/$(xx-info triple)}" BUILD_ROOT_DIR="${BUILD_ROOT_DIR:-/build}" SRC_DIR="${BUILD_ROOT_DIR}/src" -TARGET_ARCH="$(uname -m)" +TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}" if command -v xx-info; then TARGET_ARCH="$(xx-info march)" fi @@ -59,21 +59,34 @@ function build_openssl(){ export OPENSSL_ROOT_DIR="${TARGET_DIR}" export OPENSSL_LIBRARIES="${TARGET_DIR}/lib" + export KERNEL_BITS=64 target_arch="" - if [ "${TARGET_ARCH}" = "armv7l" ]; then - # openssl does not have a specific armv7l - # using generic32 instead. - target_arch="linux-generic32" - elif [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then - target_arch="linux-aarch64" - elif [ "${TARGET_ARCH}" = "x86_64" ]; then - target_arch="linux-x86_64" + if [[ ! $OSTYPE == darwin* ]]; then + if [ "${TARGET_ARCH}" = "armv7l" ]; then + # openssl does not have a specific armv7l + # using generic32 instead. + target_arch="linux-generic32" + export KERNEL_BITS=32 + elif [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then + target_arch="linux-aarch64" + elif [ "${TARGET_ARCH}" = "x86_64" ]; then + target_arch="linux-x86_64" + fi else - echo "Architecture currently not supported: ${TARGET_ARCH}" - exit 1 + SUFFIX="" + if [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then + SUFFIX="-cc" + fi + + if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then + target_arch="darwin64-arm64${SUFFIX}" + elif [ "${TARGET_ARCH}" = "x86_64" ]; then + target_arch="darwin64-x86_64${SUFFIX}" + fi + # if none of the above, let openssl figure it out. fi - ./Configure "${target_arch}" threads no-shared zlib -fPIC -DOPENSSL_PIC \ + ./Configure "${target_arch}" threads no-shared no-tests zlib -fPIC -DOPENSSL_PIC \ --prefix="${TARGET_DIR}" \ --with-zlib-include="${TARGET_DIR}/include" \ --with-zlib-lib="${TARGET_DIR}/lib" \ @@ -94,20 +107,28 @@ function build_libssh2(){ pushd build OPENSSL_LIBRARIES="${TARGET_DIR}/lib" - if [ "${TARGET_ARCH}" = "x86_64" ]; then + if [ "${TARGET_ARCH}" = "x86_64" ] && [[ ! $OSTYPE == darwin* ]]; then OPENSSL_LIBRARIES="${TARGET_DIR}/lib64" fi + # Set osx arch only when cross compiling on darwin + if [[ $OSTYPE == darwin* ]] && [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then + CMAKE_PARAMS=-DCMAKE_OSX_ARCHITECTURES="${TARGET_ARCH}" + fi + # Building examples allow for validating against missing symbols at compilation time. cmake "${CMAKE_PARAMS}" \ -DCMAKE_C_COMPILER="${C_COMPILER}" \ -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \ - -DBUILD_SHARED_LIBS=OFF \ - -DLINT=OFF \ + -DBUILD_SHARED_LIBS:BOOL=OFF \ + -DLINT:BOOL=OFF \ + -DBUILD_EXAMPLES:BOOL=ON \ + -DBUILD_TESTING:BOOL=OFF \ -DCMAKE_C_FLAGS=-fPIC \ -DCRYPTO_BACKEND=OpenSSL \ - -DENABLE_ZLIB_COMPRESSION=ON \ + -DENABLE_ZLIB_COMPRESSION:BOOL=ON \ -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ + -DZLIB_LIBRARY="${TARGET_DIR}/lib/libz.a" \ -DOPENSSL_CRYPTO_LIBRARY="${OPENSSL_LIBRARIES}/libcrypto.a" \ -DOPENSSL_SSL_LIBRARY="${OPENSSL_LIBRARIES}/libssl.a" \ .. @@ -129,17 +150,21 @@ function build_libgit2(){ SSL_LIBRARY="${TARGET_DIR}/lib/libssl.a" CRYPTO_LIBRARY="${TARGET_DIR}/lib/libcrypto.a" - if [ "${TARGET_ARCH}" = "x86_64" ]; then + if [[ ! $OSTYPE == darwin* ]] && [ "${TARGET_ARCH}" = "x86_64" ]; then SSL_LIBRARY="${TARGET_DIR}/lib64/libssl.a" CRYPTO_LIBRARY="${TARGET_DIR}/lib64/libcrypto.a" fi + # Set osx arch only when cross compiling on darwin + if [[ $OSTYPE == darwin* ]] && [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then + CMAKE_PARAMS=-DCMAKE_OSX_ARCHITECTURES="${TARGET_ARCH}" + fi + cmake "${CMAKE_PARAMS}" \ -DCMAKE_C_COMPILER="${C_COMPILER}" \ -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \ -DTHREADSAFE:BOOL=ON \ -DBUILD_CLAR:BOOL=OFF \ - -DBUILD_TESTS:BOOL=OFF \ -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \ -DCMAKE_C_FLAGS=-fPIC \ @@ -151,9 +176,9 @@ function build_libgit2(){ -DREGEX_BACKEND:STRING=builtin \ -DOPENSSL_SSL_LIBRARY="${SSL_LIBRARY}" \ -DOPENSSL_CRYPTO_LIBRARY="${CRYPTO_LIBRARY}" \ - -DZLIB_LIBRARY="${TARGET_DIR}/lib/libz.a" \ -DCMAKE_INCLUDE_PATH="${TARGET_DIR}/include" \ -DCMAKE_LIBRARY_PATH="${TARGET_DIR}/lib" \ + -DCMAKE_PREFIX_PATH="${TARGET_DIR}" \ -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ .. diff --git a/tests/smoketest/main.go b/tests/smoketest/main.go index fa5bca8..44bd78d 100644 --- a/tests/smoketest/main.go +++ b/tests/smoketest/main.go @@ -76,10 +76,6 @@ func main() { if err != nil { panic(fmt.Errorf("generating rsa key: %w", err)) } - privateKey := filepath.Join(testsDir, "id_rsa") - os.WriteFile(privateKey, rsa.PrivateKey, 0o644) - pubKey := filepath.Join(testsDir, "id_rsa.pub") - os.WriteFile(pubKey, rsa.PrivateKey, 0o644) test("SSH clone with rsa key", filepath.Join(testsDir, "/ssh-clone-rsa"), @@ -89,8 +85,7 @@ func main() { FetchOptions: &git2go.FetchOptions{ RemoteCallbacks: git2go.RemoteCallbacks{ CredentialsCallback: func(url string, username string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { - return git2go.NewCredentialSSHKey("git", - pubKey, privateKey, "") + return git2go.NewCredentialSSHKeyFromMemory("git", string(rsa.PublicKey), string(rsa.PrivateKey), "") }, CertificateCheckCallback: knownHostsCallback(u.Host, knownHosts), }, @@ -101,11 +96,6 @@ func main() { if err != nil { panic(fmt.Errorf("generating ed25519 key: %w", err)) } - privateKey = filepath.Join(testsDir, "id_ed25519") - os.WriteFile(privateKey, ed25519.PrivateKey, 0o644) - pubKey = filepath.Join(testsDir, "id_ed25519.pub") - os.WriteFile(pubKey, ed25519.PrivateKey, 0o644) - test("SSH clone with ed25519 key", filepath.Join(testsDir, "/ssh-clone-ed25519"), sshRepoURL, @@ -114,8 +104,7 @@ func main() { FetchOptions: &git2go.FetchOptions{ RemoteCallbacks: git2go.RemoteCallbacks{ CredentialsCallback: func(url string, username string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { - return git2go.NewCredentialSSHKey("git", - pubKey, privateKey, "") + return git2go.NewCredentialSSHKeyFromMemory("git", string(ed25519.PublicKey), string(ed25519.PrivateKey), "") }, CertificateCheckCallback: knownHostsCallback(u.Host, knownHosts), },