feat: osxcross (AztecProtocol#6099)

At present we only learn if mac builds fail when we run the bb-publish workflow. Ultimately it would be nice to build bb for mac as part of the rest of our build workflow. We can do this with cross-compilation from linux to mac. Developers should read and be aware of [Apples License](https://www.apple.com/legal/sla/docs/xcode.pdf) terms before using this functionality. - Added support for Apple arm64 and amd64 cross-compilation to CmakePresets. `darwin-arm64` and `darwin-amd64`. - Updated `serialize.hpp` so we can still build on Apple. (it's a kludge, we shouldn't be serializing `size_t`). - Updated `build-image` to include osxcross. - `osxcross` and `foundry` are both published so they shouldn't need to be rebuilt. However this does mean if they need updating we have a manual process to follow e.g. `make foundry` on both an x86 and arm machine. - Add new target to bb Earthfile that can cross-compile `bb`.
madztheo · May 2, 2024 · 6cc924d · 6cc924d
1 parent ef2589a
commit 6cc924d
Show file tree

Hide file tree

Showing 5 changed files with 142 additions and 11 deletions.
diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json
@@ -31,6 +31,31 @@
         "CXX": "$env{BREW_PREFIX}/opt/llvm/bin/clang++"
       }
     },
+    {
+      "name": "darwin-arm64",
+      "displayName": "Apple arm64 Cross-compile",
+      "inherits": "default",
+      "binaryDir": "build-darwin-arm64",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_TOOLCHAIN_FILE": "/opt/osxcross/toolchain.cmake"
+      },
+      "environment": {
+        "OSXCROSS_HOST": "arm64-apple-darwin23",
+        "OSXCROSS_TARGET_DIR": "/opt/osxcross",
+        "OSXCROSS_TARGET": "darwin23",
+        "OSXCROSS_SDK": "/opt/osxcross/SDK/MacOSX14.0.sdk"
+      }
+    },
+    {
+      "name": "darwin-amd64",
+      "displayName": "Apple amd64 Cross-compile",
+      "inherits": "darwin-arm64",
+      "binaryDir": "build-darwin-amd64",
+      "environment": {
+        "OSXCROSS_HOST": "x86_64-apple-darwin23"
+      }
+    },
     {
       "name": "clang16",
       "displayName": "Build with Clang-16",
@@ -358,6 +383,16 @@
       "inherits": "default",
       "configurePreset": "op-count"
     },
+    {
+      "name": "darwin-arm64",
+      "inherits": "default",
+      "configurePreset": "darwin-arm64"
+    },
+    {
+      "name": "darwin-amd64",
+      "inherits": "default",
+      "configurePreset": "darwin-amd64"
+    },
     {
       "name": "clang16-dbg",
       "inherits": "default",
@@ -456,18 +491,14 @@
       "configurePreset": "wasm-dbg",
       "inheritConfigureEnvironment": true,
       "jobs": 0,
-      "targets": [
-        "barretenberg.wasm"
-      ]
+      "targets": ["barretenberg.wasm"]
     },
     {
       "name": "wasm-threads",
       "configurePreset": "wasm-threads",
       "inheritConfigureEnvironment": true,
       "jobs": 0,
-      "targets": [
-        "barretenberg.wasm"
-      ]
+      "targets": ["barretenberg.wasm"]
     },
     {
       "name": "xray",

diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile
@@ -16,6 +16,14 @@ source:
     # for debugging rebuilds
     RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash
 
+preset-darwin-arm64:
+    FROM +source
+    LET OSX_SDK="MacOSX14.0.sdk"
+    LET OSX_SDK_URL="https://github.com/joseluisq/macosx-sdks/releases/download/14.0/${OSX_SDK}.tar.xz"
+    RUN curl -sSL "$OSX_SDK_URL" | tar -xJ -C /opt/osxcross/SDK && rm -rf /opt/osxcross/SDK/$OSX_SDK/System
+    RUN cmake --preset darwin-arm64 -Bbuild && cmake --build build --target bb
+    SAVE ARTIFACT build/bin AS LOCAL build-darwin-arm64/bin
+
 preset-release:
     FROM +source
     RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb

diff --git a/barretenberg/cpp/src/barretenberg/common/serialize.hpp b/barretenberg/cpp/src/barretenberg/common/serialize.hpp
@@ -121,6 +121,20 @@ inline void write(uint8_t*& it, uint64_t value)
     it += 8;
 }
 
+#ifdef __APPLE__
+inline void read(uint8_t const*& it, unsigned long& value)
+{
+    value = ntohll(*reinterpret_cast<unsigned long const*>(it));
+    it += 8;
+}
+
+inline void write(uint8_t*& it, unsigned long value)
+{
+    *reinterpret_cast<unsigned long*>(it) = htonll(value);
+    it += 8;
+}
+#endif
+
 #ifndef __i386__
 inline void read(uint8_t const*& it, uint128_t& value)
 {

diff --git a/build-images/Dockerfile b/build-images/Dockerfile
@@ -22,19 +22,67 @@ RUN mv build/install/opt/wasi-sdk /opt/wasi-sdk
 FROM ubuntu:noble AS wasi-sdk
 COPY --from=wasi-sdk-build /opt/wasi-sdk /opt/wasi-sdk
 
+########################################################################################################################
+# Build osxcross.
+FROM ubuntu:noble AS osxcross-build
+RUN export DEBIAN_FRONTEND="noninteractive" \
+    && apt-get update \
+    && apt-get install --no-install-recommends -y \
+        bash \
+        binutils-multiarch-dev \
+        build-essential \
+        ca-certificates \
+        clang \
+        git \
+        libbz2-dev \
+        libmpc-dev \
+        libmpfr-dev \
+        libgmp-dev \
+        liblzma-dev \
+        libpsi3-dev \
+        libssl-dev \
+        libxml2-dev \
+        libz-dev \
+        lzma-dev \
+        make \
+        patch \
+        python3 \
+        uuid-dev \
+        wget \
+        xz-utils \
+        zlib1g-dev \
+        cmake \
+        curl \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+WORKDIR /usr/src/osxcross
+ARG OSX_CROSS_COMMIT="ff8d100f3f026b4ffbe4ce96d8aac4ce06f1278b"
+RUN git clone https://github.com/tpoechtrager/osxcross.git . && git reset --hard $OSX_CROSS_COMMIT
+ARG OSX_SDK="MacOSX14.0.sdk"
+ARG OSX_SDK_URL="https://github.com/joseluisq/macosx-sdks/releases/download/14.0/${OSX_SDK}.tar.xz"
+RUN curl -sSL "$OSX_SDK_URL" -o "./tarballs/$OSX_SDK.tar.xz" \
+    && OSX_VERSION_MIN=14.0 UNATTENDED=1 ENABLE_COMPILER_RT_INSTALL=1 TARGET_DIR=/opt/osxcross ./build.sh \
+    && rm -rf ./tarballs/$OSX_SDK.tar.xz /opt/osxcross/SDK/$OSX_SDK
+FROM scratch AS osxcross
+COPY --from=osxcross-build /opt/osxcross /opt/osxcross
+
 ########################################################################################################################
 # Build foundry.
-FROM ubuntu:noble AS foundry
+FROM ubuntu:noble AS foundry-build
 RUN apt update && apt install -y git cargo
+ARG TAG
 RUN ulimit -n 65535 && \
-    git clone --depth 1 --branch nightly-de33b6af53005037b463318d2628b5cfcaf39916 \
+    git clone --depth 1 --branch nightly-$TAG \
         https://github.com/foundry-rs/foundry.git && \
     cd foundry && cargo build --profile local && \
     mkdir -p /opt/foundry/bin && \
     for t in forge cast anvil chisel; do \
         mv ./target/local/$t /opt/foundry/bin/$t; \
         strip /opt/foundry/bin/$t; \
     done
+FROM scratch AS foundry
+COPY --from=foundry-build /opt/foundry /opt/foundry
 
 ########################################################################################################################
 # This image contains *just* what's needed to perform a full build of the aztec project.
@@ -73,8 +121,13 @@ RUN apt update && \
 # Install wasi-sdk.
 COPY --from=aztecprotocol/wasi-sdk:22.0 /opt/wasi-sdk /opt/wasi-sdk
 
+# Install osxcross. Requires developer to mount SDK from their mac host.
+COPY --from=aztecprotocol/osxcross:14.0 /opt/osxcross /opt/osxcross
+ENV PATH="/opt/osxcross/bin:$PATH"
+ENV LD_LIBRARY_PATH="/opt/osxcross/lib:$LD_LIBRARY_PATH"
+
 # Install foundry.
-COPY --from=foundry /opt/foundry /opt/foundry
+COPY --from=aztecprotocol/foundry:de33b6af53005037b463318d2628b5cfcaf39916 /opt/foundry /opt/foundry
 ENV PATH="/opt/foundry/bin:$PATH"
 
 # Install rust and cross-compilers. Noir specifically uses 1.74.1.
@@ -87,7 +140,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --de
     chmod -R a+w /opt/rust
 
 # Install yq
-RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_$(dpkg --print-architecture) \
+RUN curl -sL https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_$(dpkg --print-architecture) \
         -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq
 
 # Install yarn
@@ -180,6 +233,14 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
 # - It provides docker via the hosts docker instance, mounted at /var/lib/docker.sock.
 # - It uses an entrypoint script at runtime to perform uid/gid alignment with the host and drop into user account.
 FROM basebox as devbox
+
+# Install docker client. Will use mounted host docker socket.
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
+        | tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get update && apt-get install -y docker-ce-cli
+ADD https://raw.githubusercontent.com/docker/docker-ce/master/components/cli/contrib/completion/bash/docker /etc/bash_completion.d/docker.sh
+
 RUN apt install -y gosu
 ENV TERM=xterm-256color
 # Detect if the host machine is Mac, if so set an env var, and disable prompts vcs info for performance.

diff --git a/build-images/Makefile b/build-images/Makefile
@@ -1,10 +1,27 @@
+ARCH := $(shell uname -m | sed 's/aarch64/arm64/')
+
 wasi-sdk:
-	docker build -t aztecprotocol/wasi-sdk:$$(uname -m | sed 's/aarch64/arm64/')-22.0 --target wasi-sdk --push .
+	docker build -t aztecprotocol/wasi-sdk:$(ARCH)-22.0 --target wasi-sdk --push .
 	docker manifest create aztecprotocol/wasi-sdk:22.0 \
 		--amend aztecprotocol/wasi-sdk:x86_64-22.0 \
 		--amend aztecprotocol/wasi-sdk:arm64-22.0
 	docker manifest push aztecprotocol/wasi-sdk:22.0
 
+FOUNDRY_TAG := de33b6af53005037b463318d2628b5cfcaf39916
+foundry:
+	docker build -t aztecprotocol/foundry:$(ARCH)-$(FOUNDRY_TAG) --build-arg TAG=$(FOUNDRY_TAG) --target foundry --push .
+	docker manifest create aztecprotocol/foundry:$(FOUNDRY_TAG) \
+		--amend aztecprotocol/foundry:x86_64-$(FOUNDRY_TAG) \
+		--amend aztecprotocol/foundry:arm64-$(FOUNDRY_TAG)
+	docker manifest push aztecprotocol/foundry:$(FOUNDRY_TAG)
+
+osxcross:
+	docker build -t aztecprotocol/osxcross:$(ARCH)-14.0 --target osxcross --push .
+	docker manifest create aztecprotocol/osxcross:14.0 \
+		--amend aztecprotocol/osxcross:x86_64-14.0 \
+		--amend aztecprotocol/osxcross:arm64-14.0
+	docker manifest push aztecprotocol/osxcross:14.0
+
 build:
 	docker build -t aztecprotocol/build --target build .