Merge pull request #14 from pjbgf/testdarwin

Generate MacOS universal static libraries

.github/workflows/build.yaml

@@ -4,14 +4,99 @@ on:
   push:
     branches:
       - main
+      - testdarwin2
     tags:
       - '**'
-    paths-ignore:
-      - README.md
-  pull_request:
-    paths-ignore:
-      - README.md
+  #   paths-ignore:
+  #     - README.md
+  # pull_request:
+  #   paths-ignore:
+  #     - README.md
+
+permissions:
+  contents: write # needed to write releases
+  packages: write # needed for ghcr access
+
 jobs:
+
+  mac-build:
+    # This job builds and releases "universal libraries" that are
+    # supported by both darwin-amd64 and darwin-arm64.
+    #
+    # First builds in amd64, then cross-compile in arm64. Later combining
+    # both outcomes onto a single binary for each static library.
+    #
+    # `macos-11` has been picked as support for arm64 was only added on Xcode 12.
+    # Although some minor versions of Catalina 10.15 can support it, at the time 
+    # of testing, GitHub's macos-10.15 did not seem to. 
+    # Cross-compiling to arm64 on that runner consistently failed.
+    runs-on: macos-11
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Build universal static libraries for Darwin
+        run: |
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/amd \
+          ./hack/static.sh all
+          
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/arm \
+          TARGET_ARCH=arm64 \
+          CMAKE_APPLE_SILICON_PROCESSOR=arm64 \
+          ./hack/static.sh all
+
+          mkdir -p ./libgit2-darwin/lib
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/include ./libgit2-darwin/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/share ./libgit2-darwin/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/cmake ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/engines-3 ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/ossl-modules ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/pkgconfig ./libgit2-darwin/lib/
+
+          libtool -static -o ./libgit2-darwin/lib/libcrypto.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libcrypto.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libcrypto.a 
+          libtool -static -o ./libgit2-darwin/lib/libgit2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libgit2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libgit2.a 
+          libtool -static -o ./libgit2-darwin/lib/libssh2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssh2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssh2.a 
+          libtool -static -o ./libgit2-darwin/lib/libssl.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssl.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssl.a  
+          libtool -static -o ./libgit2-darwin/lib/libz.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libz.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libz.a 
+
+          tar -zcvf darwin-libs.tar.gz ./libgit2-darwin
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 10.15
+
+      - name: Create Release
+        if: github.event_name != 'pull_request'
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          release_name: ${{ github.ref }}
+          tag_name: ${{ github.ref }}
+          draft: false
+          prerelease: true
+      - name: Upload Release Asset
+        if: github.event_name != 'pull_request'
+        id: upload-release-asset 
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./darwin-libs.tar.gz
+          asset_name: darwin-libs.tar.gz
+          asset_content_type: application/gzip
+
   build:
     runs-on: ubuntu-latest
     env:
@@ -55,7 +140,7 @@ jobs:
           key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-ghcache-
-      - run: cat ./hack/Makefile
+      - run: cat ./hack/static.sh
       - name: Build candidate image
         id: build_candidate
         uses: docker/build-push-action@v2

Dockerfile

@@ -1,6 +1,6 @@
 # This Dockerfile tests the hack/Makefile output against git2go.
 ARG BASE_VARIANT=alpine
-ARG GO_VERSION=1.17.6
+ARG GO_VERSION=1.17
 ARG XX_VERSION=1.1.0
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx

Dockerfile.test

@@ -1,28 +1,102 @@
+# This Dockerfile tests the hack/Makefile output against git2go.
 ARG BASE_VARIANT=alpine
 ARG GO_VERSION=1.17
 ARG XX_VERSION=1.1.0
 
-ARG LIBGIT2_IMG
-ARG LIBGIT2_TAG
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
-FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS build-deps
+FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-base
+
+RUN apk add --no-cache \
+        bash \
+        curl \
+        build-base \
+        linux-headers \
+        perl \
+        cmake \
+        pkgconfig \
+        gcc \
+        musl-dev \
+        clang \
+        lld
 
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
+COPY --from=xx / /
+
+FROM build-base AS build-cross
+
+ARG TARGETPLATFORM
+
+RUN xx-apk add --no-cache \
+        build-base \
+        pkgconfig \
+        gcc \
+        musl-dev \
+        clang \
+        lld \
+        llvm \
+        linux-headers
+
+WORKDIR /build
+COPY hack/static.sh .
+
+ENV CC=xx-clang
+ENV CXX=xx-clang++
+
+RUN CHOST=$(xx-clang --print-target-triple) \
+    ./static.sh build_libz
+
+RUN CHOST=$(xx-clang --print-target-triple) \
+    ./static.sh build_openssl
+
+RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \
+    export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \
+    export OPENSSL_ROOT_DIR="/usr/local/$(xx-info triple)" && \
+    export OPENSSL_CRYPTO_LIBRARY="/usr/local/$(xx-info triple)/lib64" && \
+    export OPENSSL_INCLUDE_DIR="/usr/local/$(xx-info triple)/include/openssl"
+
+RUN ./static.sh build_libssh2
+RUN ./static.sh build_libgit2
+
+
+# trimmed removes all non necessary files (i.e. openssl binary).
+FROM build-cross AS trimmed
+
+ARG TARGETPLATFORM
+RUN mkdir -p /trimmed/usr/local/$(xx-info triple)/ && \
+        mkdir -p /trimmed/usr/local/$(xx-info triple)/share
+
+RUN cp -r /usr/local/$(xx-info triple)/lib/ /trimmed/usr/local/$(xx-info triple)/ && \
+        cp -r /usr/local/$(xx-info triple)/lib64/ /trimmed/usr/local/$(xx-info triple)/ | true && \
+        cp -r /usr/local/$(xx-info triple)/include/ /trimmed/usr/local/$(xx-info triple)/ && \
+        cp -r /usr/local/$(xx-info triple)/share/doc/ /trimmed/usr/local/$(xx-info triple)/share/
+
+FROM scratch as libs-arm64
+COPY --from=trimmed /trimmed/ /
+
+FROM scratch as libs-amd64
+COPY --from=trimmed /trimmed/ /
+
+FROM scratch as libs-armv7
+COPY --from=trimmed /trimmed/ /
+
+FROM libs-$TARGETARCH$TARGETVARIANT as libs
+
+# Everything above this line is a copy from Dockefile.
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
 
 FROM gostable AS go-linux
 
 # Build-base consists of build platform dependencies and xx.
 # These will be used at current arch to yield execute the cross compilations.
-FROM go-${TARGETOS} AS build-base
+FROM go-${TARGETOS} AS go-base
 
 RUN apk add clang lld pkgconfig
 
 COPY --from=xx / /
 
 # build-go-mod can still be cached at build platform architecture.
-FROM build-base as build-go-mod
+FROM go-base as build-go-mod
 
 WORKDIR /root/smoketest
 COPY tests/smoketest/go.mod .
@@ -42,7 +116,7 @@ RUN xx-apk add musl-dev gcc clang lld
 WORKDIR /root/smoketest
 
 COPY tests/smoketest/main.go .
-COPY --from=build-deps /usr/local/ /usr/local/
+COPY --from=libs /usr/local/ /usr/local/
 
 ENV CGO_ENABLED=1
 RUN export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \

README.md

@@ -1,13 +1,17 @@
 # golang-with-libgit2
 
-This repository contains a `Dockerfile` with two files: `Makefile` and `static.sh`. 
+This repository contains a `Dockerfile` with the statically built libgit2 and its dependency chain.
+
+The `hack` directory contains two main files: `Makefile` and `static.sh`.
 Both of which can be used to build the [libgit2][] dependency chain for **AMD64, ARM64 and ARMv7** binaries 
 of Go projects that depend on [git2go][]. 
 
 The `Makefile` is useful for development environments and will leverage OS specific packages to build `libgit2`.
 The `static.sh` will build all `libgit2` dependencies from source using `musl` toolchain. This enables for a full
 static binary with the freedom of configuring each of the dependencies in chain.
 
+Alternatively, the statically built libraries can be pulling from the produced images for Linux or from the github release artifacts for MacOS.
+
 ### :warning: **Public usage discouraged**
 
 The set of dependencies was handpicked for the Flux project, based on the issue list documented below. While this setup
@@ -47,17 +51,14 @@ while testing these against the git2go code before releasing the image.
   - [ ] [libgit2/git2go#836](https://github.com/libgit2/git2go/issues/836)
   - [ ] [libgit2/git2go#837](https://github.com/libgit2/git2go/issues/837)
 
----
-**NOTE**
 
-The issues above do not affect libgit2 built with `static.sh` as all its
+> **NOTE:** The issues above do not affect libgit2 built with `static.sh` as all its
 dependencies have been configured to be optimal for its use, as the first supported version of libgit2 is `1.3.0`.
 
----
 
 ## Usage
 
-The [Dockerfile.test](./Dockerfile.test) file provides a working example on how to statically build a golang application that has a dependency to libgit2 and git2go.
+The [Dockerfile.test](./Dockerfile.test) file provides a working example on how to statically build a golang application that has a dependency on libgit2 and git2go.
 
 The example will statically build all dependencies based on the versions specified on `static.sh`.
 Then statically build the golang application and deploy it into an image based off `gcr.io/distroless/static`.