ICICLE V3 (#526)

Co-authored-by: Shanie Winitz <123639968+ShanieWinitz@users.noreply.github.com> Co-authored-by: Koren-Brand <152154782+Koren-Brand@users.noreply.github.com> Co-authored-by: emirsoyturk <emir@ingonyama.com> Co-authored-by: Miki <100796045+mickeyasa@users.noreply.github.com> Co-authored-by: ChickenLover <Romangg81@gmail.com> Co-authored-by: Jeremy Felder <jeremy.felder1@gmail.com> Co-authored-by: hadaringonyama <hadar@ingonyama.com> Co-authored-by: nonam3e <timur@ingonyama.com> Co-authored-by: Otsar <122266060+Otsar-Raikou@users.noreply.github.com>
ingonyama-zk · Sep 3, 2024 · 5ef9ed1 · 5ef9ed1
1 parent aacec3f
commit 5ef9ed1
Show file tree

Hide file tree

Showing 811 changed files with 24,887 additions and 96,413 deletions.
diff --git a/.github/changed-files.yml b/.github/changed-files.yml
@@ -16,10 +16,8 @@ cpp:
   - icicle/**/*.c
   - icicle/**/*.h
   - icicle/CMakeLists.txt
-  - .github/workflows/cpp_cuda.yml  
-  - icicle/cmake/Common.cmake
-  - icicle/cmake/CurvesCommon.cmake
-  - icicle/cmake/FieldsCommon.cmake
+  - icicle/cmake/**/*.cmake
+  - .github/workflows/cpp.yml
 examples:
   - examples/**/*
   - .github/workflows/examples.yml
diff --git a/.github/workflows/check-changed-files.yml b/.github/workflows/check-changed-files.yml
@@ -9,9 +9,9 @@ on:
       rust:
         description: "Flag for if Rust files changed"
         value: ${{ jobs.check-changed-files.outputs.rust }}
-      cpp_cuda:
-        description: "Flag for if C++/CUDA files changed"
-        value: ${{ jobs.check-changed-files.outputs.cpp_cuda }}
+      cpp:
+        description: "Flag for if C++ files changed"
+        value: ${{ jobs.check-changed-files.outputs.cpp }}
       examples:
         description: "Flag for if example files changed"
         value: ${{ jobs.check-changed-files.outputs.examples }}
@@ -23,7 +23,7 @@ jobs:
     outputs:
       golang: ${{ steps.changed_files.outputs.golang }}
       rust: ${{ steps.changed_files.outputs.rust }}
-      cpp_cuda: ${{ steps.changed_files.outputs.cpp_cuda }}
+      cpp: ${{ steps.changed_files.outputs.cpp }}
       examples: ${{ steps.changed_files.outputs.examples }}
     steps:
     - name: Checkout Repo
@@ -40,5 +40,5 @@ jobs:
       run: |
         echo "golang=${{ steps.changed-files-yaml.outputs.golang_any_modified }}" >> "$GITHUB_OUTPUT"
         echo "rust=${{ steps.changed-files-yaml.outputs.rust_any_modified }}" >> "$GITHUB_OUTPUT"
-        echo "cpp_cuda=${{ steps.changed-files-yaml.outputs.cpp_any_modified }}" >> "$GITHUB_OUTPUT"
+        echo "cpp=${{ steps.changed-files-yaml.outputs.cpp_any_modified }}" >> "$GITHUB_OUTPUT"
         echo "examples=${{ steps.changed-files-yaml.outputs.examples_any_modified }}" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -4,7 +4,8 @@ on:
   pull_request:
     branches:
       - main
-      - V2
+      - V3
+      - yshekel/V3
 
 jobs:
   spelling-checker:
@@ -15,6 +16,6 @@ jobs:
       - uses: codespell-project/actions-codespell@v2
         with:
           # https://github.com/codespell-project/actions-codespell?tab=readme-ov-file#parameter-skip
-          skip: ./**/target,./**/build,./docs/*.js,./docs/*.json
+          skip: ./**/target,./**/build,./docs/*.js,./docs/*.json,./*.svg
           # https://github.com/codespell-project/actions-codespell?tab=readme-ov-file#parameter-ignore_words_file
           ignore_words_file: .codespellignore
diff --git a/.github/workflows/cpp_cuda.yml → .github/workflows/cpp.yml b/.github/workflows/cpp_cuda.yml → .github/workflows/cpp.yml
@@ -3,12 +3,11 @@ name: C++/CUDA
 on:
   pull_request:
     branches:
-      - main
-      - V2
+      - V3
+      - yshekel/V3     # TODO remove when merged to V3
   push:
     branches:
-      - main
-      - V2
+      - V3
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -26,13 +25,18 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
     - name: Check clang-format
-      if: needs.check-changed-files.outputs.cpp_cuda == 'true'
+      if: needs.check-changed-files.outputs.cpp == 'true'
       run: if [[ $(find ./ \( -path ./icicle/build -prune -o -path ./**/target -prune -o -path ./examples -prune \) -iname *.h -or -iname *.cuh -or -iname *.cu -or -iname *.c -or -iname *.cpp | xargs clang-format --dry-run -ferror-limit=1 -style=file 2>&1) ]]; then echo "Please run clang-format"; exit 1; fi
 
+  extract-cuda-backend-branch:
+    uses: ./.github/workflows/extract-backends.yml
+    with:
+      pr-number: ${{ github.event.pull_request.number }}
+
   test-linux-curve:
     name: Test on Linux
     runs-on: [self-hosted, Linux, X64, icicle]
-    needs: [check-changed-files, check-format]
+    needs: [check-changed-files, check-format, extract-cuda-backend-branch]
     strategy:
       matrix:
         curve:
@@ -50,25 +54,33 @@ jobs:
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v4
+    - name: Checkout CUDA Backend
+      uses: actions/checkout@v4
+      with:
+        repository: ingonyama-zk/icicle-cuda-backend
+        path: ./icicle/backend/cuda
+        token: ${{ secrets.GITHUB_TOKEN }}
+        ssh-key: ${{ secrets.CUDA_PULL_KEY }}
+        ref: ${{ needs.extract-cuda-backend-branch.outputs.cuda-backend-branch }}
     - name: Build curve
       working-directory: ./icicle
-      if: needs.check-changed-files.outputs.cpp_cuda == 'true'
+      if: needs.check-changed-files.outputs.cpp == 'true'
       run: |
         mkdir -p build && rm -rf build/*
-        cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DCURVE=${{ matrix.curve.name }} ${{ matrix.curve.build_args }} -S . -B build
+        cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DCURVE=${{ matrix.curve.name }} ${{ matrix.curve.build_args }} -DCUDA_BACKEND=local -S . -B build
         cmake --build build -j
     - name: Run C++ curve Tests
       working-directory: ./icicle/build/tests
-      if: needs.check-changed-files.outputs.cpp_cuda == 'true'
+      if: needs.check-changed-files.outputs.cpp == 'true'
       run: ctest
 
   test-linux-field:
     name: Test on Linux
     runs-on: [self-hosted, Linux, X64, icicle]
-    needs: [check-changed-files, check-format]
+    needs: [check-changed-files, check-format, extract-cuda-backend-branch]
     strategy:
       matrix:
-        field: 
+        field:
           - name: babybear
             build_args: -DEXT_FIELD=ON
           - name: stark252
@@ -78,14 +90,22 @@ jobs:
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v4
+    - name: Checkout CUDA Backend
+      uses: actions/checkout@v4
+      with:
+        repository: ingonyama-zk/icicle-cuda-backend
+        path: ./icicle/backend/cuda
+        token: ${{ secrets.GITHUB_TOKEN }}
+        ssh-key: ${{ secrets.CUDA_PULL_KEY }}
+        ref: ${{ needs.extract-cuda-backend-branch.outputs.cuda-backend-branch }}
     - name: Build field
       working-directory: ./icicle
-      if: needs.check-changed-files.outputs.cpp_cuda == 'true'
+      if: needs.check-changed-files.outputs.cpp == 'true'
       run: |
         mkdir -p build && rm -rf build/*
-        cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DFIELD=${{ matrix.field.name }} ${{ matrix.field.build_args }} -S . -B build
+        cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DFIELD=${{ matrix.field.name }} ${{ matrix.field.build_args }} -DCUDA_BACKEND=local -S . -B build
         cmake --build build -j
     - name: Run C++ field Tests
       working-directory: ./icicle/build/tests
-      if: needs.check-changed-files.outputs.cpp_cuda == 'true'
+      if: needs.check-changed-files.outputs.cpp == 'true'
       run: ctest
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -1,21 +1,19 @@
 # This workflow is a demo of how to run all examples in the Icicle repository.
-# For each language directory (c++, Rust, etc.) the workflow 
-#   (1) loops over all examples (msm, ntt, etc.) and 
-#   (2) runs ./compile.sh and ./run.sh in each directory.
-# The script ./compile.sh should compile the example and ./run.sh should run it.
+# For each language directory (c++, Rust, etc.) the workflow
+#   (1) loops over all examples (msm, ntt, etc.) and
+#   (2) runs ./run.sh in each directory.
 # Each script should return 0 for success and 1 otherwise.
 
 name: Examples
 
 on:
   pull_request:
     branches:
-      - main
-      - V2
+      - V3
+      - yshekel/V3     # TODO remove when merged to V3
   push:
     branches:
-      - main
-      - V2
+      - V3
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -25,26 +23,38 @@ jobs:
   check-changed-files:
     uses: ./.github/workflows/check-changed-files.yml
 
+  extract-cuda-backend-branch:
+    uses: ./.github/workflows/extract-backends.yml
+    with:
+      pr-number: ${{ github.event.pull_request.number }}
+
   run-examples:
     runs-on: [self-hosted, Linux, X64, icicle, examples]
-    needs: check-changed-files
+    needs: [check-changed-files, extract-cuda-backend-branch]
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+    - name: Checkout CUDA Backend
+      uses: actions/checkout@v4
+      with:
+        repository: ingonyama-zk/icicle-cuda-backend
+        path: ./icicle/backend/cuda
+        token: ${{ secrets.GITHUB_TOKEN }}
+        ssh-key: ${{ secrets.CUDA_PULL_KEY }}
+        ref: ${{ needs.extract-cuda-backend-branch.outputs.cuda-backend-branch }}
     - name: c++ examples
       working-directory: ./examples/c++
-      if: needs.check-changed-files.outputs.cpp_cuda == 'true' || needs.check-changed-files.outputs.examples == 'true'
+      if: needs.check-changed-files.outputs.cpp == 'true' || needs.check-changed-files.outputs.examples == 'true'
       run: |
         # loop over all directories in the current directory
         for dir in $(find . -mindepth 1 -maxdepth 1 -type d); do
           if [ -d "$dir" ]; then
             echo "Running command in $dir"
             cd $dir
-            ./compile.sh
-            ./run.sh
+            ./run.sh -d CUDA
             cd -
           fi
-        done    
+        done
     - name: Rust examples
       working-directory: ./examples/rust
       if: needs.check-changed-files.outputs.rust == 'true' || needs.check-changed-files.outputs.examples == 'true'
@@ -54,7 +64,7 @@ jobs:
           if [ -d "$dir" ]; then
             echo "Running command in $dir"
             cd $dir
-            cargo run --release
+            ./run.sh -d CUDA
             cd -
           fi
-        done      
+        done
diff --git a/.github/workflows/extract-backends.yml b/.github/workflows/extract-backends.yml
@@ -0,0 +1,36 @@
+name: Extract Icicle Backend Branch
+
+on:
+  workflow_call:
+    inputs:
+      pr-number:
+        description: 'The PR number to fetch the description for'
+        required: true
+        type: number
+    outputs:
+      cuda-backend-branch:
+        description: "Branch name for cuda backend"
+        value: ${{ jobs.extract-cuda-backend-branch.outputs.cuda-backend-branch }}
+
+jobs:
+  extract-cuda-backend-branch:
+    name: Extract cuda branch name
+    runs-on: ubuntu-22.04
+    outputs:
+      cuda-backend-branch: ${{ steps.extract.outputs.cuda-backend-branch }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Extract Private Branch from PR Description
+        id: extract
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          DESCRIPTION=$(gh pr view ${{ inputs.pr-number }} --json body -q '.body')
+          echo "PR Description: $DESCRIPTION"
+          CUDA_BE_BRANCH=$(echo "$DESCRIPTION" | grep -oP 'cuda-backend-branch:\s*\K[^\s]+') || true
+          if [ -z "$CUDA_BE_BRANCH" ]; then
+            CUDA_BE_BRANCH="main" # Default branch if not specified
+          fi
+          echo "Extracted CUDA Backend Branch: $CUDA_BE_BRANCH"
+          echo "cuda-backend-branch=$CUDA_BE_BRANCH" >> "$GITHUB_OUTPUT"