Merge branch 'main' into deniallugo-update-githooks

matter-labs · Sep 18, 2024 · 3ad4465 · 3ad4465
2 parents 5732b7e + 3f1d1a2
commit 3ad4465
Show file tree

Hide file tree

Showing 31 changed files with 665 additions and 393 deletions.
diff --git a/.env b/.env
diff --git a/.github/workflows/ci-common-reusable.yml b/.github/workflows/ci-common-reusable.yml
@@ -18,13 +18,16 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
 
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
           mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
-          ci_run sccache --start-server
 
       - name: Init
         run: |

diff --git a/.github/workflows/ci-core-lint-reusable.yml b/.github/workflows/ci-core-lint-reusable.yml
@@ -15,13 +15,16 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
           echo "prover_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local_prover" >> $GITHUB_ENV
           echo "core_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local" >> $GITHUB_ENV
 
       - name: Start services
         run: |
           ci_localnet_up
-          ci_run sccache --start-server
 
       - name: Build
         run: |

diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -26,6 +26,10 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
 
       # TODO: Remove when we after upgrade of hardhat-plugins
       - name: pre-download compilers
@@ -48,7 +52,6 @@ jobs:
       - name: Start services
         run: |
           ci_localnet_up
-          ci_run sccache --start-server
 
       - name: Init
         run: |
@@ -84,6 +87,10 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
 
       - name: Loadtest configuration
         run: |
@@ -119,16 +126,14 @@ jobs:
             --set-as-default false \
             --ignore-prerequisites \
             --legacy-bridge
-          
+
           ci_run zk_inception ecosystem init --dev --verbose
           ci_run zk_supervisor contracts --test-contracts
 
       # `sleep 60` because we need to wait until server added all the tokens
       - name: Run server
         run: |
-          ci_run sed -i -e 's/mode: FULL/mode: LIGHTWEIGHT/' chains/legacy/configs/general.yaml
-          ci_run sed -i -e 's/state_keeper_fast_vm_mode:.*/state_keeper_fast_vm_mode: ${{ matrix.vm_mode }}/' chains/legacy/configs/general.yaml
-          ci_run sed -i -e 's/delay_interval:.*/delay_interval: 50/' chains/legacy/configs/general.yaml
+          ci_run zk_supervisor config-writer --path ${{ matrix.vm_mode == 'NEW' && 'etc/env/file_based/overrides/tests/loadtest-new.yaml' || 'etc/env/file_based/overrides/tests/loadtest-old.yaml' }} --chain legacy
           ci_run zk_inception server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
           ci_run sleep 60
 
@@ -159,12 +164,15 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
           echo RUN_CONTRACT_VERIFICATION_TEST=true >> $GITHUB_ENV
 
       - name: Start services
         run: |
           ci_localnet_up
-          ci_run sccache --start-server
 
       - name: Build zk_toolbox
         run: ci_run bash -c "./bin/zkt"
@@ -178,15 +186,15 @@ jobs:
           GENESIS_RECOVERY_LOGS_DIR=logs/genesis_recovery/
           EXTERNAL_NODE_LOGS_DIR=logs/external_node
           REVERT_LOGS_DIR=logs/revert
-          
+
           mkdir -p $SERVER_LOGS_DIR
           mkdir -p $INTEGRATION_TESTS_LOGS_DIR
           mkdir -p $INTEGRATION_TESTS_EN_LOGS_DIR
           mkdir -p $SNAPSHOT_RECOVERY_LOGS_DIR
           mkdir -p $GENESIS_RECOVERY_LOGS_DIR
           mkdir -p $EXTERNAL_NODE_LOGS_DIR
           mkdir -p $REVERT_LOGS_DIR
-          
+
           echo "SERVER_LOGS_DIR=$SERVER_LOGS_DIR" >> $GITHUB_ENV
           echo "INTEGRATION_TESTS_LOGS_DIR=$INTEGRATION_TESTS_LOGS_DIR" >> $GITHUB_ENV
           echo "INTEGRATION_TESTS_EN_LOGS_DIR=$INTEGRATION_TESTS_EN_LOGS_DIR" >> $GITHUB_ENV
@@ -329,7 +337,7 @@ jobs:
 
       - name: Initialize Contract verifier
         run: |
-          ci_run zk_inception contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.18-1.0.1 --only --chain era
+          ci_run zk_inception contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
           ci_run zk_inception contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
 
       - name: Run servers
@@ -354,7 +362,7 @@ jobs:
 
           ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --chain custom_token &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/custom_token.log &
           PID3=$!
-          
+
           ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --chain consensus &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log &
           PID4=$!
 
@@ -367,16 +375,16 @@ jobs:
         run: |
           ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --db-name=zksync_en_localhost_era_rollup --l1-rpc-url=http://localhost:8545 --chain era
-          ci_run zk_inception external-node init --ignore-prerequisites --chain era 
-          
+          ci_run zk_inception external-node init --ignore-prerequisites --chain era
+
           ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --db-name=zksync_en_localhost_era_validium1 --l1-rpc-url=http://localhost:8545 --chain validium
           ci_run zk_inception external-node init --ignore-prerequisites --chain validium
 
           ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --db-name=zksync_en_localhost_era_custom_token --l1-rpc-url=http://localhost:8545 --chain custom_token
           ci_run zk_inception external-node init --ignore-prerequisites --chain custom_token
-          
+
           ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --db-name=zksync_en_localhost_era_consensus --l1-rpc-url=http://localhost:8545 --chain consensus
           ci_run zk_inception external-node init --ignore-prerequisites --chain consensus
@@ -386,13 +394,13 @@ jobs:
 
           ci_run zk_supervisor test recovery --snapshot --no-deps --ignore-prerequisites --verbose --chain era &> ${{ env.SNAPSHOT_RECOVERY_LOGS_DIR }}/rollup.log &
           PID1=$!
-          
+
           ci_run zk_supervisor test recovery --snapshot --no-deps --ignore-prerequisites --verbose --chain validium &> ${{ env.SNAPSHOT_RECOVERY_LOGS_DIR }}/validium.log &
           PID2=$!
-          
+
           ci_run zk_supervisor test recovery --snapshot --no-deps --ignore-prerequisites --verbose --chain custom_token &> ${{ env.SNAPSHOT_RECOVERY_LOGS_DIR }}/custom_token.log &
           PID3=$!
-          
+
           ci_run zk_supervisor test recovery --snapshot --no-deps --ignore-prerequisites --verbose --chain consensus &> ${{ env.SNAPSHOT_RECOVERY_LOGS_DIR }}/consensus.log &
           PID4=$!
 
@@ -411,7 +419,7 @@ jobs:
 
           ci_run zk_supervisor test recovery --no-deps --no-kill --ignore-prerequisites --verbose --chain custom_token &> ${{ env.GENESIS_RECOVERY_LOGS_DIR }}/custom_token.log &
           PID3=$!
-          
+
           ci_run zk_supervisor test recovery --no-deps --no-kill --ignore-prerequisites --verbose --chain consensus &> ${{ env.GENESIS_RECOVERY_LOGS_DIR }}/consensus.log &
           PID4=$!
 
@@ -450,7 +458,7 @@ jobs:
         run: |
           ci_run killall -INT zksync_server || true
           ci_run killall -INT zksync_external_node || true
-          
+
           ci_run zk_supervisor test revert --no-deps --external-node --no-kill --ignore-prerequisites --chain era &> ${{ env.REVERT_LOGS_DIR }}/rollup.log &
           PID1=$!
 
@@ -462,7 +470,7 @@ jobs:
 
           ci_run zk_supervisor test revert --no-deps --external-node --no-kill --ignore-prerequisites --chain consensus &> ${{ env.REVERT_LOGS_DIR }}/consensus.log &
           PID4=$!
-          
+
           wait $PID1
           wait $PID2
           wait $PID3

diff --git a/.github/workflows/ci-docs-reusable.yml b/.github/workflows/ci-docs-reusable.yml
@@ -17,12 +17,16 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
 
       - name: Start services
         run: |
           run_retried docker compose pull zk
           docker compose up -d zk
-      
+
       - name: Build
         run: |
           ci_run ./bin/zkt

diff --git a/.github/workflows/ci-prover-reusable.yml b/.github/workflows/ci-prover-reusable.yml
@@ -17,6 +17,10 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
           echo "prover_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local_prover" >> $GITHUB_ENV
           echo "core_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local" >> $GITHUB_ENV
 
@@ -25,7 +29,6 @@ jobs:
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
           mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
-          ci_run sccache --start-server
 
       - name: Init
         run: |
@@ -50,13 +53,16 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
 
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
           mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
-          ci_run sccache --start-server
 
       - name: Init
         run: |

diff --git a/.github/workflows/vm-perf-comparison.yml b/.github/workflows/vm-perf-comparison.yml
@@ -35,6 +35,10 @@ jobs:
           touch .env
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
 
       - name: init
         run: |

diff --git a/.github/workflows/vm-perf-to-prometheus.yml b/.github/workflows/vm-perf-to-prometheus.yml
@@ -22,7 +22,10 @@ jobs:
       - name: setup-env
         run: |
           echo BENCHMARK_PROMETHEUS_PUSHGATEWAY_URL=${{ secrets.BENCHMARK_PROMETHEUS_PUSHGATEWAY_URL }} >> .env
-
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
 

diff --git a/core/tests/ts-integration/hardhat.config.ts b/core/tests/ts-integration/hardhat.config.ts
@@ -20,8 +20,11 @@ export default {
         }
     },
     solidity: {
-        version: '0.8.18',
-        eraVersion: '1.0.1'
+        version: '0.8.26',
+        eraVersion: '1.0.1',
+        settings: {
+            evmVersion: 'cancun'
+        }
     },
     vyper: {
         version: '0.3.10'

diff --git a/core/tests/ts-integration/tests/api/contract-verification.test.ts b/core/tests/ts-integration/tests/api/contract-verification.test.ts
@@ -12,7 +12,7 @@ const DATE_REGEX = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?/;
 
 const ZKSOLC_VERSION = 'v1.5.3';
 const SOLC_VERSION = '0.8.26';
-const ZK_VM_SOLC_VERSION = 'zkVM-0.8.18-1.0.1';
+const ZK_VM_SOLC_VERSION = 'zkVM-0.8.26-1.0.1';
 
 const ZKVYPER_VERSION = 'v1.5.4';
 const VYPER_VERSION = '0.3.10';

diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml
@@ -36,7 +36,7 @@ api:
     filters_limit: 10000
     subscriptions_limit: 10000
     pubsub_polling_interval: 200
-    max_nonce_ahead: 20
+    max_nonce_ahead: 40
     gas_price_scale_factor: 1.5
     estimate_gas_scale_factor: 1.3
     estimate_gas_acceptable_overestimation: 5000

diff --git a/...file_based/overrides/mainnet/general.yaml → etc/env/file_based/overrides/mainnet.yaml b/...file_based/overrides/mainnet/general.yaml → etc/env/file_based/overrides/mainnet.yaml
@@ -10,12 +10,13 @@ eth:
     aggregated_block_prove_deadline: 300
     aggregated_block_execute_deadline: 300
     timestamp_criteria_max_allowed_lag: 104000 # 29h
+    wait_confirmations: null
   gas_adjuster:
     pricing_formula_parameter_a: 1.06
     internal_l1_pricing_multiplier: 1
     internal_pubdata_pricing_multiplier: 1.50
     poll_period: 60
+  watcher:
+    confirmations_for_eth_event: null
 observability:
   log_directives: zksync=info,zksync_state_keeper=debug,zksync_core=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=debug,zksync_state=debug,zksync_utils=debug,zksync_eth_sender=debug,loadnext=debug,dev_ticker=info,vm=info,block_sizes_test=info,setup_key_generator_and_server=info,zksync_queued_job_processor=debug,slot_index_consistency_checker=debug,zksync_health_check=debug,zksync_consensus_bft=debug,zksync_consensus_network=debug,zksync_consensus_storage=debug,zksync_consensus_executor=debug,
-
-# remove eth_sender_wait_confirmations, eth_watcher_confirmations_for_eth_event variables
diff --git a/etc/env/file_based/overrides/only_real_proofs.yaml b/etc/env/file_based/overrides/only_real_proofs.yaml
@@ -0,0 +1,3 @@
+eth:
+  sender:
+    proof_sending_mode: ONLY_REAL_PROOFS
diff --git a/...file_based/overrides/testnet/general.yaml → etc/env/file_based/overrides/testnet.yaml b/...file_based/overrides/testnet/general.yaml → etc/env/file_based/overrides/testnet.yaml
@@ -10,6 +10,7 @@ eth:
     aggregated_block_prove_deadline: 300
     aggregated_block_execute_deadline: 300
     timestamp_criteria_max_allowed_lag: 104000 # 29h
+    wait_confirmations: null
   gas_adjuster:
     pricing_formula_parameter_a: 1.1
     internal_l1_pricing_multiplier: 1
@@ -18,5 +19,3 @@ eth:
     confirmations_for_eth_event: 10
 observability:
   log_directives: zksync=info,zksync_state_keeper=debug,zksync_core=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=debug,zksync_state=debug,zksync_utils=debug,zksync_eth_sender=debug,loadnext=debug,dev_ticker=info,vm=info,block_sizes_test=info,setup_key_generator_and_server=info,zksync_queued_job_processor=debug,slot_index_consistency_checker=debug,zksync_health_check=debug,zksync_consensus_bft=debug,zksync_consensus_network=debug,zksync_consensus_storage=debug,zksync_consensus_executor=debug,
-
-# remove eth_sender_wait_confirmations variable
diff --git a/etc/env/file_based/overrides/tests/loadtest-new.yaml b/etc/env/file_based/overrides/tests/loadtest-new.yaml
@@ -0,0 +1,7 @@
+db:
+  merkle_tree:
+    mode: LIGHTWEIGHT
+experimental_vm:
+  state_keeper_fast_vm_mode: NEW
+mempool:
+  delay_interval: 50
diff --git a/etc/env/file_based/overrides/tests/loadtest-old.yaml b/etc/env/file_based/overrides/tests/loadtest-old.yaml
@@ -0,0 +1,7 @@
+db:
+  merkle_tree:
+    mode: LIGHTWEIGHT
+experimental_vm:
+  state_keeper_fast_vm_mode: OLD
+mempool:
+  delay_interval: 50
diff --git a/etc/env/file_based/overrides/validium.yaml b/etc/env/file_based/overrides/validium.yaml
@@ -0,0 +1,6 @@
+eth:
+  sender:
+    pubdata_sending_mode: CUSTOM
+state_keeper:
+  pubdata_overhead_part: 0
+  compute_overhead_part: 1
diff --git a/zk_toolbox/Cargo.lock b/zk_toolbox/Cargo.lock
diff --git a/zk_toolbox/crates/common/src/lib.rs b/zk_toolbox/crates/common/src/lib.rs
@@ -14,6 +14,7 @@ pub mod git;
 pub mod server;
 pub mod version;
 pub mod wallets;
+pub mod yaml;
 
 pub use prerequisites::{
     check_general_prerequisites, check_prerequisites, GCLOUD_PREREQUISITE, GPU_PREREQUISITES,