ci: split CI into separate moldy jobs (#1192)

* use [`mold`](https://github.com/rui314/mold) linker for building Rust code * Splits into several parallel jobs * disables one test that was taking ~7 minutes with miri, cutting the miri time down considerably * Add new check for generate fbs/proto files being up to date Shaves ~40% off of CI times
spiraldb · Nov 4, 2024 · 7846c36 · 7846c36
1 parent 615ad5e
commit 7846c36
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 28 deletions.
diff --git a/.github/actions/cleanup/action.yml b/.github/actions/cleanup/action.yml
@@ -15,8 +15,6 @@ runs:
           /opt/ghc \
           /usr/lib/mono \
           /usr/local/julia* \
-          /usr/local/lib/android \
-          /usr/local/lib/node_modules \
           /usr/local/share/chromium \
           /usr/local/share/powershell \
           /usr/share/dotnet \

diff --git a/.github/actions/setup-flatc/action.yml b/.github/actions/setup-flatc/action.yml
@@ -0,0 +1,12 @@
+name: "Setup flatc"
+description: "Download and install flatc binary"
+runs:
+  using: "composite"
+  steps:
+    - name: Download flatc
+      id: download-flatc
+      shell: bash
+      run: |
+        wget -O /tmp/flatc.zip https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.clang++-15.zip
+        unzip /tmp/flatc.zip flatc
+        mv flatc /usr/local/bin/
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ "develop" ]
-  pull_request: { }
-  workflow_dispatch: { }
+    branches: ["develop"]
+  pull_request: {}
+  workflow_dispatch: {}
 
 permissions:
   actions: read
@@ -15,35 +15,24 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  build:
-    name: 'build'
+  python-lint:
+    name: "Python (lint)"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
-
-      - uses: ./.github/actions/setup-rust
       - uses: spiraldb/actions/.github/actions/setup-uv@0.2.0
 
       - name: Python Lint - Format
         run: uv run ruff format --check .
       - name: Python Lint - Ruff
         run: uv run ruff check .
 
-      - name: Rust Lint - Format
-        run: cargo fmt --all --check
-      - name: Rust Lint - Clippy
-        run: cargo clippy --all-features --all-targets
-      - name: Docs
-        run: cargo doc --no-deps
-      - name: Rust Test
-        run: cargo test --workspace --all-features
-
-      - name: Rust Build (Default features)
-        run: cargo build --all-targets
-
-      - name: Rust Build (All Features)
-        run: cargo build --all-features --all-targets
+  python-test:
+    name: "Python (test)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: spiraldb/actions/.github/actions/setup-uv@0.2.0
 
       - name: Pytest - PyVortex
         run: |
@@ -62,6 +51,57 @@ jobs:
           (cd pyvortex && uv run maturin develop)
 
           (cd docs && uv run make html)
+
+  rust-docs:
+    name: "Rust (docs)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-rust
+      - name: Docs
+        run: cargo doc --no-deps
+
+  build-default:
+    name: "Build (default)"
+    runs-on: ubuntu-latest
+    steps:
+      # setup mold linker
+      - uses: rui314/setup-mold@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/cleanup
+      - uses: ./.github/actions/setup-rust
+      - name: Rust Build (Default features)
+        run: cargo build --all-targets
+
+  build-all:
+    name: "Build (all-features)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: rui314/setup-mold@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/cleanup
+      - uses: ./.github/actions/setup-rust
+      - name: Rust Build (All Features)
+        run: cargo build --all-features --all-targets
+
+  rust-test:
+    name: "Rust (tests)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: rui314/setup-mold@v1
+      - uses: actions/checkout@v4
+      - name: Rust Lint - Format
+        run: cargo fmt --all --check
+      - name: Rust Lint - Clippy
+        run: cargo clippy --all-features --all-targets
+      - name: Rust Test
+        run: cargo test --workspace --all-features
+
+  license-check:
+    name: License check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
       - name: License Check
         run: cargo install --locked cargo-deny && cargo deny check
       - uses: rustsec/audit-check@v2.0.0
@@ -70,24 +110,24 @@ jobs:
           ignore: "RUSTSEC-2023-0086"
 
   miri:
-    name: 'miri'
+    name: "miri"
     runs-on: ubuntu-latest
     env:
       MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-backtrace=full
     steps:
+      - uses: rui314/setup-mold@v1
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
       - uses: ./.github/actions/setup-rust
       - name: Run tests with Miri
         run: cargo miri test
 
   bench-test:
-    name: 'bench test'
+    name: "bench test"
     runs-on: ubuntu-latest
     steps:
+      - uses: rui314/setup-mold@v1
       - uses: actions/checkout@v4
       - uses: ./.github/actions/cleanup
-
       - uses: ./.github/actions/setup-rust
       - uses: spiraldb/actions/.github/actions/setup-uv@0.2.0
       # Required to run benchmarks
@@ -97,3 +137,23 @@ jobs:
           version: v1.0.0
       - name: Rust Bench as test
         run: cargo bench --bench '*[!noci]' -- --test
+
+  generated-files:
+    name: "Check generated proto/fbs files are up to date"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: rui314/setup-mold@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/cleanup
+      - uses: ./.github/actions/setup-rust
+      - uses: ./.github/actions/setup-flatc
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+      - name: "regenerate all .fbs/.proto Rust code"
+        run: |
+          cargo xtask generate-fbs
+          cargo xtask generate-proto
+      - name: "Make sure no files changed after regenerating"
+        run: |
+          test -z "$(git status --porcelain)"
+
diff --git a/encodings/fsst/src/compute.rs b/encodings/fsst/src/compute.rs
@@ -170,6 +170,7 @@ mod tests {
     use crate::{fsst_compress, fsst_train_compressor};
 
     #[test]
+    #[cfg_attr(miri, ignore)]
     fn test_compare_fsst() {
         let lhs = VarBinArray::from_iter(
             [