Merge pull request #3102 from autonomys/cross-compilation

Convenient cross-compilation, CUDA in aarch64 container image

.github/workflows/snapshot-build.yml

@@ -18,7 +18,9 @@ env:
 
 jobs:
   container-linux:
-    runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
+    # TODO: Restore once we have self-hosted runners with Ubuntu 22.04
+    #runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
+    runs-on: ubuntu-22.04
     permissions:
       contents: write
       packages: write
@@ -28,28 +30,17 @@ jobs:
           - farmer
           - node
           - bootstrap-node
-        platform:
-          - arch: linux/amd64
-            dockerfile-suffix: ""
-            suffix: ubuntu-x86_64-${{ github.ref_name }}
-            image-suffix: ""
-            rustflags: "-C target-cpu=skylake"
-          # We build AArch64
-          - arch: linux/amd64
-            dockerfile-suffix: ".aarch64"
-            suffix: ubuntu-aarch64-${{ github.ref_name }}
-            image-suffix: "-aarch64"
       fail-fast: false
 
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
 
       - name: Log into registry
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -67,19 +58,18 @@ jobs:
             type=sha,format=long
           flavor: |
             latest=false
-            suffix=${{ matrix.platform.image-suffix }}
+            suffix=
 
       - name: Build and push ${{ matrix.image }} image
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
         with:
-          file: Dockerfile-${{ matrix.image }}${{ matrix.platform.dockerfile-suffix }}
-          platforms: ${{ matrix.platform.arch }}
+          file: Dockerfile-${{ matrix.image }}
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }}
-            RUSTFLAGS=${{ matrix.platform.rustflags }}
 
       - name: Trigger trivy-security-scan Workflow
         uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # @v3.0.0

Dockerfile-bootstrap-node

@@ -1,12 +1,15 @@
-FROM ubuntu:20.04
+# This Dockerfile supports both native building and cross-compilation to aarch64 on x86-64
+FROM --platform=$BUILDPLATFORM ubuntu:22.04
 
 ARG RUSTC_VERSION=nightly-2024-09-26
 ARG PROFILE=production
 ARG RUSTFLAGS
-# Workaround for https://github.com/rust-lang/cargo/issues/10583
-ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 # Incremental compilation here isn't helpful
 ENV CARGO_INCREMENTAL=0
+ENV PKG_CONFIG_ALLOW_CROSS=true
+
+ARG BUILDARCH
+ARG TARGETARCH
 
 WORKDIR /code
 
@@ -22,10 +25,19 @@ RUN \
         automake \
         libtool \
         pkg-config \
-        make && \
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
+        make
+
+RUN \
+    if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \
+      DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+          g++-aarch64-linux-gnu \
+          gcc-aarch64-linux-gnu \
+          libc6-dev-arm64-cross \
+    ; fi
 
-RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown
+RUN \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION && \
+    /root/.cargo/bin/rustup target add wasm32-unknown-unknown
 
 COPY Cargo.lock /code/Cargo.lock
 COPY Cargo.toml /code/Cargo.toml
@@ -39,16 +51,23 @@ COPY test /code/test
 # Up until this line all Rust images in this repo should be the same to share the same layers
 
 RUN \
+    if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \
+      export RUSTFLAGS="$RUSTFLAGS -C linker=aarch64-linux-gnu-gcc" \
+    ; fi && \
+    if [ $TARGETARCH = "amd64" ] && [ $RUSTFLAGS = ""]; then \
+      export RUSTFLAGS="-C target-cpu=skylake" \
+    ; fi && \
+    RUSTC_TARGET_ARCH=$(echo $TARGETARCH | sed "s/amd64/x86_64/g" | sed "s/arm64/aarch64/g") && \
     /root/.cargo/bin/cargo -Zgitoxide -Zgit build \
         --locked \
         -Z build-std \
         --profile $PROFILE \
         --bin subspace-bootstrap-node \
-        --target $(uname -p)-unknown-linux-gnu && \
+        --target $RUSTC_TARGET_ARCH-unknown-linux-gnu && \
     mv target/*/*/subspace-bootstrap-node subspace-bootstrap-node && \
     rm -rf target
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 COPY --from=0 /code/subspace-bootstrap-node /subspace-bootstrap-node
 

Dockerfile-farmer

@@ -1,12 +1,15 @@
-FROM ubuntu:20.04
+# This Dockerfile supports both native building and cross-compilation to aarch64 on x86-64
+FROM --platform=$BUILDPLATFORM ubuntu:22.04
 
 ARG RUSTC_VERSION=nightly-2024-09-26
 ARG PROFILE=production
 ARG RUSTFLAGS
-# Workaround for https://github.com/rust-lang/cargo/issues/10583
-ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 # Incremental compilation here isn't helpful
 ENV CARGO_INCREMENTAL=0
+ENV PKG_CONFIG_ALLOW_CROSS=true
+
+ARG BUILDARCH
+ARG TARGETARCH
 
 WORKDIR /code
 
@@ -22,10 +25,19 @@ RUN \
         automake \
         libtool \
         pkg-config \
-        make && \
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
+        make
 
-RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown
+RUN \
+    if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \
+      DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+          g++-aarch64-linux-gnu \
+          gcc-aarch64-linux-gnu \
+          libc6-dev-arm64-cross \
+    ; fi
+
+RUN \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION && \
+    /root/.cargo/bin/rustup target add wasm32-unknown-unknown
 
 COPY Cargo.lock /code/Cargo.lock
 COPY Cargo.toml /code/Cargo.toml
@@ -38,26 +50,39 @@ COPY test /code/test
 
 # Up until this line all Rust images in this repo should be the same to share the same layers
 
+# CUDA toolchain, including support for cross-compilation to aarch64 on x86-64
 RUN \
-    CUDA_ARCH=$(uname -p | sed "s/aarch64/arm64/g") && \
-    curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/$CUDA_ARCH/cuda-keyring_1.1-1_all.deb && \
-    dpkg -i cuda-keyring_1.1-1_all.deb && \
-    echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/$CUDA_ARCH/ /" | tee /etc/apt/sources.list.d/cuda-ubuntu2004-$CUDA_ARCH.list && \
-    curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/$CUDA_ARCH/cuda-ubuntu2004.pin && \
-    mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cuda-minimal-build-12-4 && \
-    echo "/usr/local/cuda/lib64" > /etc/ld.so.conf.d/cuda.conf && \
-    ldconfig
+    if [ $BUILDARCH = "amd64" ] || [ $BUILDARCH = "arm64" ]; then \
+      if [ $TARGETARCH = "amd64" ] || [ $TARGETARCH = "arm64" ]; then \
+        CUDA_ARCH=$(echo $BUILDARCH | sed "s/amd64/x86_64/g") && \
+        curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/cuda-keyring_1.1-1_all.deb && \
+        dpkg -i cuda-keyring_1.1-1_all.deb && \
+        echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/ /" | tee /etc/apt/sources.list.d/cuda-ubuntu2204-$CUDA_ARCH.list && \
+        curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/cuda-ubuntu2204.pin && \
+        mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
+        apt-get update && \
+        DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cuda-minimal-build-12-4 && \
+        echo "/usr/local/cuda/lib64" > /etc/ld.so.conf.d/cuda.conf && \
+        if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \
+          CUDA_ARCH="cross-linux-aarch64" && \
+          echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/ /" | tee /etc/apt/sources.list.d/cuda-ubuntu2204-$CUDA_ARCH.list && \
+          curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/cuda-ubuntu2204.pin && \
+          mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600-cross-aarch64 && \
+          apt-get update && \
+          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cuda-cross-aarch64-12-4 \
+        ; fi && \
+        ldconfig \
+      ; fi \
+    ; fi
 
 # ROCm is only used on x86-64 since they don't have other packages
 ARG ROCM_VERSION=6.2.2
 RUN \
-    if [ $(uname -p) = "x86_64" ]; then \
+    if [ $BUILDARCH = "amd64" ] && [ $TARGETARCH = "amd64" ]; then \
       DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gpg && \
       mkdir -p --mode=0755 /etc/apt/keyrings && \
       curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/rocm.gpg && \
-      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" > /etc/apt/sources.list.d/rocm.list && \
+      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION jammy main" > /etc/apt/sources.list.d/rocm.list && \
       echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" > /etc/apt/preferences.d/rocm-pin-600 && \
       apt-get update && \
       DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends rocm-hip-runtime-dev && \
@@ -68,15 +93,22 @@ RUN \
 # TODO: Remove `NVCC=off` hack once `sppark` has proper features for CUDA and ROCm
 # ROCm is only used on x86-64 since they don't have other packages
 RUN \
+    if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \
+      export RUSTFLAGS="$RUSTFLAGS -C linker=aarch64-linux-gnu-gcc" \
+    ; fi && \
+    if [ $TARGETARCH = "amd64" ] && [ $RUSTFLAGS = ""]; then \
+      export RUSTFLAGS="-C target-cpu=skylake" \
+    ; fi && \
     export PATH=/usr/local/cuda/bin${PATH:+:${PATH}} && \
-    if [ $(uname -p) = "x86_64" ]; then \
+    RUSTC_TARGET_ARCH=$(echo $TARGETARCH | sed "s/amd64/x86_64/g" | sed "s/arm64/aarch64/g") && \
+    if [ $BUILDARCH = "amd64" ] && [ $TARGETARCH = "amd64" ]; then \
       NVCC=off /root/.cargo/bin/cargo -Zgitoxide -Zgit build \
           --locked \
           -Z build-std \
           --profile $PROFILE \
           --bin subspace-farmer \
           --features rocm \
-          --target $(uname -p)-unknown-linux-gnu && \
+          --target $RUSTC_TARGET_ARCH-unknown-linux-gnu && \
       mv target/*/*/subspace-farmer subspace-farmer-rocm \
     ; fi && \
     /root/.cargo/bin/cargo -Zgitoxide -Zgit build \
@@ -85,22 +117,22 @@ RUN \
         --profile $PROFILE \
         --bin subspace-farmer \
         --features cuda \
-        --target $(uname -p)-unknown-linux-gnu && \
+        --target $RUSTC_TARGET_ARCH-unknown-linux-gnu && \
     mv target/*/*/subspace-farmer subspace-farmer && \
     rm -rf target
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # Next block is for ROCm support
 # ROCm is only used on x86-64 since they don't have other packages
 ARG ROCM_VERSION=6.2.2
 RUN \
-    if [ $(uname -p) = "x86_64" ]; then \
+    if [ $BUILDARCH = "amd64" ] && [ $TARGETARCH = "amd64" ]; then \
       apt-get update && \
       DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates gpg && \
       mkdir -p --mode=0755 /etc/apt/keyrings && \
       curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/rocm.gpg && \
-      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" > /etc/apt/sources.list.d/rocm.list && \
+      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION jammy main" > /etc/apt/sources.list.d/rocm.list && \
       echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" > /etc/apt/preferences.d/rocm-pin-600 && \
       apt-get update && \
       DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends hip-runtime-amd && \