From a6c10953326b449127f6dcda2b92d2b1747c7f2d Mon Sep 17 00:00:00 2001 From: Brecht Devos Date: Mon, 6 May 2024 19:35:59 +0200 Subject: [PATCH] feat(raiko): install script + makefile CI integration (#159) * test make in ci * update makefile * add make install * update makefile * back to stable * typo * fixes * caching * fix sp1 hopefully * fixes * fix SP1 * fix sgx * fix sp1 * fix SP1 for sure now ya * now 4 sure * enable testing * feedback fix * feedback * add run script --- .github/workflows/ci.yml | 71 ++++++++++++++++++------------------- README.md | 40 ++++++++------------- makefile | 31 +++++++++------- rust-toolchain | 2 +- scripts/install.sh | 55 ++++++++++++++++++++++++++++ scripts/run.sh | 43 ++++++++++++++++++++++ scripts/toolchain-after.sh | 35 ------------------ scripts/toolchain-before.sh | 51 -------------------------- scripts/toolchain.sh | 53 --------------------------- toolchain.sh | 48 ------------------------- 10 files changed, 165 insertions(+), 264 deletions(-) create mode 100755 scripts/install.sh create mode 100755 scripts/run.sh delete mode 100755 scripts/toolchain-after.sh delete mode 100755 scripts/toolchain-before.sh delete mode 100755 scripts/toolchain.sh delete mode 100755 toolchain.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 557888c9..83dc16de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,47 +24,40 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - run: scripts/toolchain-before.sh risc0 - uses: actions-rs/toolchain@v1 with: - profile: minimal - - run: scripts/toolchain-after.sh risc0 - - uses: risc0/risc0/.github/actions/sccache@release-0.19 + toolchain: stable + profile: minimal - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 - - name: Install cargo-risczero - run: | - cargo binstall -y --force cargo-risczero - cargo risczero install - - name: Build risc0 guest target - run: cargo run --bin risc0-builder + - uses: risc0/risc0/.github/actions/sccache@release-0.19 + - name: Install risc0 + run: TARGET="risc0" CI=1 make install - name: Build risc0 prover - run: cargo build --release --features risc0 + run: TARGET="risc0" make build - name: Test risc0 prover - run: cargo test -p risc0-driver --features enable + run: TARGET="risc0" make test build-test-sp1: name: Build and test sp1 runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v4 - - run: scripts/toolchain-before.sh sp1 - - uses: actions-rs/toolchain@v1 - with: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2024-02-06 profile: minimal - - run: scripts/toolchain-after.sh sp1 - - name: Install sp1 - run: | - curl -L https://sp1.succinct.xyz | bash - echo "/home/runner/.config/.sp1/bin" >> $GITHUB_PATH - /home/runner/.config/.sp1/bin/sp1up - - name: Build sp1 guest target - run: cargo run --bin sp1-builder - - name: Build sp1 prover - run: cargo build --release --features sp1 - - name: Test sp1 prover - run: cargo test -p sp1-driver --features enable --release + - name: Install cargo-binstall + uses: cargo-bins/cargo-binstall@v1.6.4 + - name: Install sp1 + run: TARGET="sp1" CI=1 make install + - name: Build sp1 prover + run: TARGET="sp1" make build + - name: Test sp1 prover + run: TARGET="risc0" make test build-test-sgx: name: Build and test sgx @@ -72,16 +65,20 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 - - run: scripts/toolchain-before.sh sgx + with: + submodules: recursive - uses: actions-rs/toolchain@v1 with: - profile: minimal - - run: scripts/toolchain-after.sh sgx - - name: Setup and build - run: | - cargo build --features sgx - cargo test -p sgx-prover --features enable --release - shell: bash + toolchain: stable + profile: minimal + - name: Install cargo-binstall + uses: cargo-bins/cargo-binstall@v1.6.4 + - name: Install sgx + run: TARGET="sgx" CI=1 make install + - name: Build sgx prover + run: TARGET="sgx" make build + - name: Test sgx prover + run: TARGET="sgx" make test build-test-sgx-with-docker: name: Build and test sgx with Docker @@ -127,4 +124,4 @@ jobs: steps: - uses: actions/checkout@v4 - uses: risc0/risc0/.github/actions/rustup@release-0.19 - - run: cargo fmt --all --check + - run: make fmt diff --git a/README.md b/README.md index c7ef4be8..e056bd25 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,18 @@ ### Building -- Install the `cargo risczero` tool and the `risc0` toolchain: +- To download all dependencies for all provers you can run ```console -$ cargo install cargo-risczero -$ cargo risczero install --version v2024-02-08.1 +$ make install ``` -- Install the `cargo prove` tool and the `succinct` toolchain: + +You can also download all required dependencies for each prover separately, for example for SP1: ```console -$ curl -L https://sp1.succinct.xyz | bash -$ sp1up -$ cargo prove --version +$ TARGET="sp1" make install ``` -- For SGX, install gramine: https://github.com/gramineproject/gramine. If you're running ubuntu 22.04 (or a compatible distro) you can just download and install this deb file: https://packages.gramineproject.io/pool/main/g/gramine/gramine_1.6.2_amd64.deb - - Clone the repository and build with `cargo`: ```console @@ -51,22 +47,16 @@ You can also automatically sync with the tip of the chain and prove all new bloc ## Provers -Provers can be enabled using features. To compile with all of them (using standard options): - -``` -cargo run --release --features "risc0 sp1" -``` - ### risc zero Build using ``` -make risc0 +TARGET="risc0" make build ``` -#### Testing +#### Running ``` -RISC0_DEV_MODE=1 cargo run --release --features risc0 +TARGET="risc0" make run ``` #### Bonsai @@ -79,7 +69,7 @@ prove_block.sh taiko_a7 risc0-bonsai 10 #### CPU ``` -cargo run --release --features risc0 +TARGET="risc0" make run ``` #### GPU @@ -96,20 +86,18 @@ CUDA needs to be installed when using `cuda`: https://docs.nvidia.com/cuda/cuda- ### SP1: ``` -make sp1 -cargo run --release --features sp1 +TARGET="sp1" make build +TARGET="sp1" make run ``` ### SGX: ``` -make sgx -cargo run --release --features sgx +TARGET="sgx" make build +TARGET="sgx" make run ``` -Make sure to first do a cargo build because cargo run does not build the sgx binary! - If your CPU doesn't support SGX, you can still run the SGX code through gramine like it would on an SGX machine: ``` -SGX_DIRECT=1 cargo run --release --features sgx +SGX_DIRECT=1 TARGET="sgx" make run ``` \ No newline at end of file diff --git a/makefile b/makefile index ff9c298a..0fb5fcff 100644 --- a/makefile +++ b/makefile @@ -1,14 +1,19 @@ -risc0: - @./toolchain.sh risc0 - @cargo run --bin risc0-builder - cargo build --release --features risc0 - -sp1: - @./toolchain.sh sp1 - @cargo run --bin sp1-builder - cargo build --release --features sp1 - -sgx: - @./toolchain.sh sgx - cargo build --release --features sgx +install: + ./scripts/install.sh $(TARGET) + +build: + ./scripts/run.sh $(TARGET) + +run: + RUN=1 ./scripts/run.sh $(TARGET) + +test: + TEST=1 ./scripts/run.sh $(TARGET) + +fmt: + @cargo fmt --all --check + +clippy: + @cargo +nightly-2024-02-06 check --features "sgx,sp1,risc0" + @cargo +nightly-2024-02-06 clippy --workspace --features "sgx,sp1,risc0" --all-targets -- -Dwarnings diff --git a/rust-toolchain b/rust-toolchain index 2bf5ad04..870bbe4e 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -stable +stable \ No newline at end of file diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 00000000..11d07785 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# toolchain necessary to compile c-kzg in SP1/risc0 +if [ -z "$1" ] || [ "$1" == "sp1" ] || [ "$1" == "risc0" ]; then + # Check if the RISC-V GCC prebuilt binary archive already exists + if [ -f /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz ]; then + echo "riscv-gcc-prebuilt existed, please check the file manually" + else + # Download the file using wget + wget -O /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz https://github.com/stnolting/riscv-gcc-prebuilt/releases/download/rv32i-131023/riscv32-unknown-elf.gcc-13.2.0.tar.gz + # Check if wget succeeded + if [ $? -ne 0 ]; then + echo "Failed to download riscv-gcc-prebuilt" + exit 1 + fi + # Create the directory if it doesn't exist + if [ ! -d /opt/riscv ]; then + mkdir /opt/riscv + fi + # Extract the downloaded archive + tar -xzf /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz -C /opt/riscv/ + # Check if tar succeeded + if [ $? -ne 0 ]; then + echo "Failed to extract riscv-gcc-prebuilt" + exit 1 + fi + fi +fi + +# SGX +if [ -z "$1" ] || [ "$1" == "sgx" ]; then + # For SGX, install gramine: https://github.com/gramineproject/gramine. + wget -O /tmp/gramine.deb https://packages.gramineproject.io/pool/main/g/gramine/gramine_1.6.2_amd64.deb + sudo apt install /tmp/gramine.deb +fi +# RISC0 +if [ -z "$1" ] || [ "$1" == "risc0" ]; then + cargo install cargo-risczero + cargo risczero install --version v2024-02-08.1 +fi +# SP1 +if [ -z "$1" ] || [ "$1" == "sp1" ]; then + curl -L https://sp1.succinct.xyz | bash + + if [ -z "${CI}" ]; then + # Need to add sp1up to the path here + PROFILE=$HOME/.bashrc + echo ${PROFILE} + source ${PROFILE} + sp1up + else + echo "/home/runner/.config/.sp1/bin" >> $GITHUB_PATH + /home/runner/.config/.sp1/bin/sp1up + fi +fi \ No newline at end of file diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 00000000..c017f09a --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +TOOLCHAIN_RISC0=+stable +TOOLCHAIN_SP1=+nightly-2024-02-06 +TOOLCHAIN_SGX=+stable + +if [ -z "${DEBUG}" ]; then + FLAGS=--release +fi + + +if [ -z "${RUN}" ]; then + COMMAND=build +else + COMMAND=run +fi + +# SGX +if [ -z "$1" ] || [ "$1" == "sgx" ]; then + if [ -z "${TEST}" ]; then + cargo ${TOOLCHAIN_SGX} ${COMMAND} ${FLAGS} --features sgx + else + cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p sgx-prover --features enable + fi +fi +# RISC0 +if [ -z "$1" ] || [ "$1" == "risc0" ]; then + if [ -z "${TEST}" ]; then + cargo ${TOOLCHAIN_RISC0} run --bin risc0-builder + cargo ${TOOLCHAIN_RISC0} ${COMMAND} ${FLAGS} --features risc0 + else + RISC0_DEV_MODE=1 cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p risc0-driver --features enable + fi +fi +# SP1 +if [ -z "$1" ] || [ "$1" == "sp1" ]; then + if [ -z "${TEST}" ]; then + cargo ${TOOLCHAIN_SP1} run --bin sp1-builder + cargo ${TOOLCHAIN_SP1} ${COMMAND} ${FLAGS} --features sp1 + else + cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p sp1-driver --features enable + fi +fi \ No newline at end of file diff --git a/scripts/toolchain-after.sh b/scripts/toolchain-after.sh deleted file mode 100755 index 3acbb09a..00000000 --- a/scripts/toolchain-after.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# This is a workaround tos set up cargo component in CI -CRATE_TOOLCHAIN="" -ROOT_TOOLCHAIN="rust-toolchain" - -# Define paths -if [ "$1" == "native" ]; then - echo "Skip toolchain modification" - exit 1 -elif [ "$1" == "sgx" ]; then - CRATE_TOOLCHAIN="provers/sgx/prover/rust-toolchain.toml" - echo "rust-toolchain.toml has been copied from SGX prover to the workspace root." -elif [ "$1" == "risc0" ]; then - CRATE_TOOLCHAIN="provers/risc0/driver/rust-toolchain.toml" - echo "rust-toolchain.toml has been copied from Risc0 prover to the workspace root." -elif [ "$1" == "sp1" ]; then - CRATE_TOOLCHAIN="provers/sp1/driver/rust-toolchain.toml" - echo "rust-toolchain.toml has been copied from Sp1 prover to the workspace root." -else - echo "Skip toolchain modification" - exit 1 -fi - -# Extract the channel (toolchain version) from the selected crate toolchain file -TOOLCHAIN_VERSION=$(grep "channel = " $CRATE_TOOLCHAIN | cut -d '"' -f 2) - -# Extract components from the selected crate toolchain file and add them using rustup -grep "components = \[" $CRATE_TOOLCHAIN | sed 's/.*\[\(.*\)\].*/\1/' | tr ',' '\n' | while read -r component; do - component=$(echo $component | xargs | tr -d '"') - if [ ! -z "$component" ]; then - echo "Adding component: $component" - rustup component add "$component" --toolchain "$TOOLCHAIN_VERSION" - fi -done \ No newline at end of file diff --git a/scripts/toolchain-before.sh b/scripts/toolchain-before.sh deleted file mode 100755 index 53bb0ed9..00000000 --- a/scripts/toolchain-before.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# Check if the RISC-V GCC prebuilt binary archive already exists -if [ -f /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz ]; then - echo "riscv-gcc-prebuilt existed, please check the file manually" -else - # Download the file using wget - wget -O /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz https://github.com/stnolting/riscv-gcc-prebuilt/releases/download/rv32i-131023/riscv32-unknown-elf.gcc-13.2.0.tar.gz - # Check if wget succeeded - if [ $? -ne 0 ]; then - echo "Failed to download riscv-gcc-prebuilt" - exit 1 - fi - # Create the directory if it doesn't exist - if [ ! -d /opt/riscv ]; then - mkdir /opt/riscv - fi - # Extract the downloaded archive - tar -xzf /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz -C /opt/riscv/ - # Check if tar succeeded - if [ $? -ne 0 ]; then - echo "Failed to extract riscv-gcc-prebuilt" - exit 1 - fi -fi - -# This is a workaround tos set up cargo toolchain in CI -CRATE_TOOLCHAIN="" -ROOT_TOOLCHAIN="rust-toolchain" - -# Define paths -if [ "$1" == "native" ]; then - echo "Skip toolchain modification" - exit 1 -elif [ "$1" == "sgx" ]; then - CRATE_TOOLCHAIN="provers/sgx/prover/rust-toolchain.toml" - echo "rust-toolchain.toml has been copied from SGX prover to the workspace root." -elif [ "$1" == "risc0" ]; then - CRATE_TOOLCHAIN="provers/risc0/driver/rust-toolchain.toml" - echo "rust-toolchain.toml has been copied from Risc0 prover to the workspace root." -elif [ "$1" == "sp1" ]; then - CRATE_TOOLCHAIN="provers/sp1/driver/rust-toolchain.toml" - echo "rust-toolchain.toml has been copied from Sp1 prover to the workspace root." -else - echo "Skip toolchain modification" - exit 1 -fi - -# Extract the channel (toolchain version) from the selected crate toolchain file -TOOLCHAIN_VERSION=$(grep "channel = " $CRATE_TOOLCHAIN | cut -d '"' -f 2) -echo $TOOLCHAIN_VERSION > $ROOT_TOOLCHAIN diff --git a/scripts/toolchain.sh b/scripts/toolchain.sh deleted file mode 100755 index 43c4197e..00000000 --- a/scripts/toolchain.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -# Check if the RISC-V GCC prebuilt binary archive already exists -if [ -f /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz ]; then - echo "riscv-gcc-prebuilt existed, please check the file manually" -else - # Download the file using wget - wget -O /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz https://github.com/stnolting/riscv-gcc-prebuilt/releases/download/rv32i-131023/riscv32-unknown-elf.gcc-13.2.0.tar.gz - # Check if wget succeeded - if [ $? -ne 0 ]; then - echo "Failed to download riscv-gcc-prebuilt" - exit 1 - fi - # Create the directory if it doesn't exist - if [ ! -d /opt/riscv ]; then - mkdir /opt/riscv - fi - # Extract the downloaded archive - tar -xzf /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz -C /opt/riscv/ - # Check if tar succeeded - if [ $? -ne 0 ]; then - echo "Failed to extract riscv-gcc-prebuilt" - exit 1 - fi -fi - -# unzip to dest folder -mkdir -p /tmp/riscv -tar -xzvf /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz -C /tmp/riscv -if [ $? -ne 0 ]; then - echo "failed to unzip riscv-gcc-prebuilt" - exit 1 -fi - -if [ "$1" == "native" ]; then - echo "Skip toolchain modification" -elif [ "$1" == "sgx" ]; then - cp provers/sgx/prover/rust-toolchain.toml ./ - echo "rust-toolchain.toml has been copied from SGX prover to the workspace root." -elif [ "$1" == "risc0" ]; then - export CC=gcc && export CC_riscv32im_risc0_zkvm_elf=/tmp/riscv/bin/riscv32-unknown-elf-gcc - cp provers/risc0/rust-toolchain.toml ./ - echo "rust-toolchain.toml has been copied from Risc0 prover to the workspace root." -elif [ "$1" == "sp1" ]; then - export CC=gcc && export CC_riscv32im_succinct_zkvm_elf=/tmp/riscv/bin/riscv32-unknown-elf-gcc - cp provers/sp1/prover/rust-toolchain.toml ./ - echo "rust-toolchain.toml has been copied from Sp1 prover to the workspace root." -else - echo "Skip toolchain modification" - exit 1 -fi - -rm rust-toolchain.toml diff --git a/toolchain.sh b/toolchain.sh deleted file mode 100755 index a00e3780..00000000 --- a/toolchain.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# Check if the RISC-V GCC prebuilt binary archive already exists -if [ -f /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz ]; then - echo "riscv-gcc-prebuilt existed, please check the file manually" -else - # Download the file using wget - wget -O /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz https://github.com/stnolting/riscv-gcc-prebuilt/releases/download/rv32i-131023/riscv32-unknown-elf.gcc-13.2.0.tar.gz - # Check if wget succeeded - if [ $? -ne 0 ]; then - echo "Failed to download riscv-gcc-prebuilt" - exit 1 - fi - # Create the directory if it doesn't exist - if [ ! -d /opt/riscv ]; then - mkdir /opt/riscv - fi - # Extract the downloaded archive - tar -xzf /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz -C /opt/riscv/ - # Check if tar succeeded - if [ $? -ne 0 ]; then - echo "Failed to extract riscv-gcc-prebuilt" - exit 1 - fi -fi - -if [ "$1" == "native" ]; then - echo "Skip toolchain modification" -elif [ "$1" == "sgx" ]; then - cp provers/sgx/prover/rust-toolchain.toml ./ - echo "rust-toolchain.toml has been copied from SGX prover to the workspace root." -elif [ "$1" == "risc0" ]; then - cp provers/risc0/driver/rust-toolchain.toml ./ - echo "rust-toolchain.toml has been copied from Risc0 prover to the workspace root." -elif [ "$1" == "sp1" ]; then - cp provers/sp1/driver/rust-toolchain.toml ./ - echo "rust-toolchain.toml has been copied from Sp1 prover to the workspace root." -else - echo "Skip toolchain modification" - exit 1 -fi - -# Check if the file exists -if [ -f "rust-toolchain" ]; then - # If the file exists, remove it - rm "rust-toolchain" - echo "File removed successfully." -fi \ No newline at end of file