chore(cross): Split images

.github/buildkitd.toml

@@ -0,0 +1,2 @@
+[worker.oci]
+  max-parallelism = 1

.github/workflows/push-cross-images.yml → .github/workflows/cross-images.yml

@@ -3,13 +3,13 @@ name: Cross Images
 on:
   push:
     paths:
-      - '.github/workflows/push-cross-images.yml'
+      - '.github/workflows/cross-images.yml'
       - 'rust/cubestore/cross/**'
     branches:
       - 'master'
   pull_request:
     paths:
-      - '.github/workflows/push-cross-images.yml'
+      - '.github/workflows/cross-images.yml'
       - 'rust/cubestore/cross/**'
 
 jobs:
@@ -33,17 +33,13 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Load .cross file
-        uses: xom9ikk/dotenv@v2.3.0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
         with:
-          path: rust/cubestore/cross/
-      - name: Push to Docker Hub
-        uses: docker/build-push-action@v6
+          buildkitd-config: .github/buildkitd.toml
+      - name: Build and push
+        uses: docker/bake-action@v5
         with:
-          context: ./
-          file: ./rust/cubestore/cross/${{ matrix.target }}.Dockerfile
-          platforms: linux/amd64
+          workdir: ./rust/cubestore/cross
+          targets: ${{ matrix.target }}-python
           push: ${{ github.ref == 'refs/heads/master' }}
-          tags: cubejs/rust-cross:${{ matrix.target }},cubejs/rust-cross:${{ matrix.target }}-${{ env.CROSS_VERSION }}

rust/cubestore/cross/README.md

@@ -19,18 +19,13 @@ Keep in mind:
 
 ```sh
 # dmY
-export $(cat .env | xargs)
+docker buildx bake x86_64-unknown-linux-gnu-python --push
+docker buildx bake aarch64-unknown-linux-gnu-python --push
+docker buildx bake x86_64-unknown-linux-musl-python --push
 
-docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION -f x86_64-unknown-linux-gnu.Dockerfile .
-docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-musl-$CROSS_VERSION -f x86_64-unknown-linux-musl.Dockerfile .
-docker buildx build --platform linux/amd64 -t cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION -f aarch64-unknown-linux-gnu.Dockerfile .
-
-docker push cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION
-docker push cubejs/rust-cross:x86_64-unknown-linux-musl-$CROSS_VERSION
-docker push cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION
+export CROSS_VERSION=15082024
 
 # Verify versions
 docker run --platform linux/amd64 --rm -it cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION cc --version
-docker run --platform linux/amd64 --rm -it cubejs/rust-cross:x86_64-unknown-linux-gnu-buster-$CROSS_VERSION cc --version
 docker run --platform linux/amd64 --rm -it cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION cc --version
 ```

rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile

@@ -0,0 +1,27 @@
+FROM base
+
+ARG PYTHON_VERSION
+ARG PYTHON_RELEASE
+
+# python  is required for cross compiling python :D
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${PYTHON_RELEASE} \
+    && rm -rf /var/lib/apt/lists/*;
+
+RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \
+    && cd Python-${PYTHON_VERSION} \
+    && touch config.site-aarch64 \
+    && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \
+    && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \
+    && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \
+    && CONFIG_SITE=config.site-aarch64 ./configure  \
+      --enable-shared \
+      --enable-optimizations \
+      --disable-ipv6 \
+      --prefix=/usr/aarch64-linux-gnu \
+      --build=aarch64-unknown-linux-gnu \
+      --host=x86_64-linux-gnu \
+      --with-build-python=/usr/bin/python${PYTHON_RELEASE} \
+    && make -j $(nproc) \
+    && make install \
+    && cd .. && rm -rf Python-${PYTHON_VERSION} \
+    && rm -rf /var/lib/apt/lists/*; \

rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile

@@ -3,28 +3,30 @@
 # https://github.com/rust-embedded/cross/blob/master/docker/Dockerfile.aarch64-unknown-linux-gnu
 FROM cubejs/cross-aarch64-unknown-linux-gnu:31122022
 
+ARG LLVM_VERSION=18
+
 RUN apt-get update \
     && apt-get -y upgrade \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget curl apt-transport-https ca-certificates \
     && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
-    && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-18 main"  \
+    && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-$LLVM_VERSION main"  \
     && add-apt-repository -y ppa:deadsnakes/ppa \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y libffi-dev binutils-multiarch binutils-aarch64-linux-gnu gcc-multilib g++-multilib \
     # llvm14-dev will install python 3.8 as bin/python3
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-18 lld-18 clang-18 libclang-18-dev clang-18 \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-$LLVM_VERSION lld-$LLVM_VERSION clang-$LLVM_VERSION libclang-$LLVM_VERSION-dev clang-$LLVM_VERSION \
         make cmake libsasl2-dev \
         libc6 libc6-dev libc6-arm64-cross libc6-dev-arm64-cross \
         gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*;
 
-RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 \
-    && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 \
-    && update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-18 100 \
-    && update-alternatives --install /usr/bin/cc cc /usr/bin/clang-18 100 \
-    && update-alternatives --install /usr/bin/lld clang-cpp /usr/bin/lld-18 100 \
-    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-18 100;
+RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION 100 \
+    && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$LLVM_VERSION 100 \
+    && update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-$LLVM_VERSION 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/clang-$LLVM_VERSION 100 \
+    && update-alternatives --install /usr/bin/lld clang-cpp /usr/bin/lld-$LLVM_VERSION 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-$LLVM_VERSION 100;
 
 ENV ARCH=arm \
     MACHINE=armv8 \
@@ -63,102 +65,6 @@ ENV PYO3_CROSS_PYTHON_VERSION=3.11 \
     OPENSSL_LIB_DIR=/usr/aarch64-linux-gnu/lib \
     OPENSSL_LIBRARIES=/usr/aarch64-linux-gnu/lib
 
-ENV PYTHON_VERSION=3.12.4
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \
-    # python  is required for cross compiling python :D
-    && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.12 \
-    && cd Python-${PYTHON_VERSION} \
-    && touch config.site-aarch64 \
-    && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \
-    && CONFIG_SITE=config.site-aarch64 ./configure  \
-      --enable-shared \
-      --enable-optimizations \
-      --disable-ipv6 \
-      --prefix=/usr/aarch64-linux-gnu \
-      --build=aarch64-unknown-linux-gnu \
-      --host=x86_64-linux-gnu \
-      --with-build-python=/usr/bin/python3.12 \
-    && make -j $(nproc) \
-    && make install \
-    && cd .. && rm -rf Python-${PYTHON_VERSION} \
-    && apt-get remove -y python3.12 \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*;
-
-ENV PYTHON_VERSION=3.11.3
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \
-    # python  is required for cross compiling python :D
-    && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.11 \
-    && cd Python-${PYTHON_VERSION} \
-    && touch config.site-aarch64 \
-    && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \
-    && CONFIG_SITE=config.site-aarch64 ./configure  \
-      --enable-shared \
-      --enable-optimizations \
-      --disable-ipv6 \
-      --prefix=/usr/aarch64-linux-gnu \
-      --build=aarch64-unknown-linux-gnu \
-      --host=x86_64-linux-gnu \
-      --with-build-python=/usr/bin/python3.11 \
-    && make -j $(nproc) \
-    && make install \
-    && cd .. && rm -rf Python-${PYTHON_VERSION} \
-    && apt-get remove -y python3.11 \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*;
-
-ENV PYTHON_VERSION=3.10.11
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \
-    # python  is required for cross compiling python :D
-    && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.10 \
-    && cd Python-${PYTHON_VERSION} \
-    && touch config.site-aarch64 \
-    && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \
-    && CONFIG_SITE=config.site-aarch64 ./configure  \
-      --enable-shared \
-      --enable-optimizations \
-      --disable-ipv6 \
-      --prefix=/usr/aarch64-linux-gnu \
-      --build=aarch64-unknown-linux-gnu \
-      --host=x86_64-linux-gnu \
-      --with-build-python=/usr/bin/python3.10 \
-    && make -j $(nproc) \
-    && make install \
-    && cd .. && rm -rf Python-${PYTHON_VERSION} \
-    && apt-get remove -y python3.10 \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*;
-
-ENV PYTHON_VERSION=3.9.18
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \
-    # python  is required for cross compiling python :D
-    && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.9 \
-    && cd Python-${PYTHON_VERSION} \
-    && touch config.site-aarch64 \
-    && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \
-    && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \
-    && CONFIG_SITE=config.site-aarch64 ./configure  \
-      --enable-shared \
-      --enable-optimizations \
-      --disable-ipv6 \
-      --prefix=/usr/aarch64-linux-gnu \
-      --build=aarch64-unknown-linux-gnu \
-      --host=x86_64-linux-gnu \
-      --with-build-python=/usr/bin/python3.9 \
-    && make -j $(nproc) \
-    && make install \
-    && cd .. && rm -rf Python-${PYTHON_VERSION} \
-    && apt-get remove -y python3.9 \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*;
-
 ENV PKG_CONFIG_ALLOW_CROSS=true \
     PKG_CONFIG_ALL_STATIC=true \
     RUSTFLAGS="-C target-feature=-crt-static" \

rust/cubestore/cross/docker-bake.hcl

@@ -0,0 +1,110 @@
+variable "CROSS_VERSION" {
+  default = "15082024"
+}
+
+variable "LLVM_VERSION" {
+  default = "18"
+}
+
+target "aarch64-unknown-linux-gnu" {
+  context = "."
+  dockerfile = "aarch64-unknown-linux-gnu.Dockerfile"
+  args = {
+    LLVM_VERSION = LLVM_VERSION
+  }
+  tags = ["cubejs/cross:aarch64-unknown-linux-gnu-${CROSS_VERSION}"]
+  platforms = ["linux/amd64"]
+}
+
+target "aarch64-unknown-linux-gnu-python" {
+  inherits = ["aarch64-unknown-linux-gnu"]
+  contexts = {
+    base = "target:aarch64-unknown-linux-gnu"
+  }
+  dockerfile = "aarch64-unknown-linux-gnu-python.Dockerfile"
+  name = "aarch64-unknown-linux-gnu-python-${replace(item.python_release, ".", "-")}"
+  matrix = {
+    item = [
+      {
+        python_version = "3.12.4"
+        python_release = "3.12"
+      },
+      {
+        python_version = "3.11.3"
+        python_release = "3.11"
+      },
+      {
+        python_version = "3.10.11"
+        python_release = "3.10"
+      },
+      {
+        python_version = "3.9.18"
+        python_release = "3.9"
+      }
+    ]
+  }
+  args = {
+    CROSS_VERSION = CROSS_VERSION
+    PYTHON_VERSION = item.python_version
+    PYTHON_RELEASE = item.python_release
+  }
+  tags = ["cubejs/rust-cross:aarch64-unknown-linux-gnu-${CROSS_VERSION}-python-${item.python_release}"]
+  platforms = ["linux/amd64"]
+}
+
+# We use -python prefix for ci
+target "x86_64-unknown-linux-musl-python" {
+  context = "."
+  dockerfile = "x86_64-unknown-linux-musl.Dockerfile"
+  args = {
+    LLVM_VERSION = LLVM_VERSION
+  }
+  tags = ["cubejs/rust-cross:x86_64-unknown-linux-musl-${CROSS_VERSION}"]
+  platforms = ["linux/amd64"]
+}
+
+target "x86_64-unknown-linux-gnu" {
+  context = "."
+  dockerfile = "x86_64-unknown-linux-gnu.Dockerfile"
+  args = {
+    LLVM_VERSION = LLVM_VERSION
+  }
+  tags = ["cubejs/rust-cross:x86_64-unknown-linux-gnu-${CROSS_VERSION}"]
+  platforms = ["linux/amd64"]
+}
+
+target "x86_64-unknown-linux-gnu-python" {
+  inherits = ["x86_64-unknown-linux-gnu"]
+  contexts = {
+    base = "target:x86_64-unknown-linux-gnu"
+  }
+  dockerfile = "x86_64-unknown-linux-gnu-python.Dockerfile"
+  name = "x86_64-unknown-linux-gnu-python-${replace(item.python_release, ".", "-")}"
+  matrix = {
+    item = [
+      {
+        python_version = "3.12.4"
+        python_release = "3.12"
+      },
+      {
+        python_version = "3.11.3"
+        python_release = "3.11"
+      },
+      {
+        python_version = "3.10.11"
+        python_release = "3.10"
+      },
+      {
+        python_version = "3.9.18"
+        python_release = "3.9"
+      }
+    ]
+  }
+  args = {
+    CROSS_VERSION = CROSS_VERSION
+    PYTHON_VERSION = item.python_version
+    PYTHON_RELEASE = item.python_release
+  }
+  tags = ["cubejs/rust-cross:x86_64-unknown-linux-gnu-${CROSS_VERSION}-python-${item.python_release}"]
+  platforms = ["linux/amd64"]
+}

rust/cubestore/cross/x86_64-unknown-linux-gnu-python.Dockerfile

@@ -0,0 +1,22 @@
+FROM base
+
+ARG PYTHON_VERSION
+ARG PYTHON_RELEASE
+
+RUN cd tmp && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \
+    && cd Python-${PYTHON_VERSION} && \
+    ./configure  \
+      --enable-shared \
+      --with-openssl=/openssl \
+      --enable-optimizations \
+      --disable-ipv6 \
+      --prefix=/usr \
+    && make -j $(nproc) \
+    && make install \
+    && ln -f -s /usr/bin/python${PYTHON_RELEASE} /usr/bin/python3 \
+    && cd .. && rm -rf Python-${PYTHON_VERSION};
+
+# pyo3 uses python3 to detect version, but there is a bug and it uses python3.9 (system), this force it to use a new python
+ENV PYO3_PYTHON=python${PYTHON_RELEASE}
+
+ENV PATH="/cargo/bin:$PATH"