Skip to content

Commit

Permalink
fix: instruction count diff always N/A in VM perf comparison (#1608)
Browse files Browse the repository at this point in the history
Instruction count diff should now be correct. The problem wasn't found
initially because counting instructions legitimately fails if the PR
starts from a version where that wasn't supported yet.

Also found and fixed a bug that hid performance improvements.
  • Loading branch information
joonazan authored Apr 9, 2024
1 parent ef12df7 commit c0f3104
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 4 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/vm-perf-comparison.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
ci_run zk
ci_run zk compiler system-contracts
ci_run cargo bench --package vm-benchmark --bench iai | tee base-iai
ci_run cd core/tests/vm_benchmark && cargo run --release --bin instruction-counts | tee base-opcodes || touch base-opcodes
ci_run cargo run --package vm-benchmark --release --bin instruction-counts | tee base-opcodes || touch base-opcodes
ci_run yarn workspace system-contracts clean
- name: checkout PR
Expand All @@ -58,7 +58,7 @@ jobs:
ci_run zk
ci_run zk compiler system-contracts
ci_run cargo bench --package vm-benchmark --bench iai | tee pr-iai
ci_run cd core/tests/vm_benchmark && cargo run --release --bin instruction-counts | tee pr-opcodes || touch pr-opcodes
ci_run cargo run --package vm-benchmark --release --bin instruction-counts | tee pr-opcodes || touch pr-opcodes
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "speedup<<$EOF" >> $GITHUB_OUTPUT
Expand Down
2 changes: 1 addition & 1 deletion core/tests/vm-benchmark/src/compare_iai_results.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ fn main() {
.intersection(&iai_after.keys().collect())
.flat_map(|&name| {
let diff = percent_difference(iai_before[name], iai_after[name]);
if diff > 2. {
if diff.abs() > 2. {
Some((name, format!("{:+.1}%", diff)))
} else {
None
Expand Down
12 changes: 11 additions & 1 deletion core/tests/vm-benchmark/src/instruction_counts.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
//! Runs all benchmarks and prints out the number of zkEVM opcodes each one executed.

use std::path::Path;

use vm_benchmark_harness::{cut_to_allowed_bytecode_size, get_deploy_tx, BenchmarkingVm};

fn main() {
for path in std::fs::read_dir("deployment_benchmarks").unwrap() {
// using source file location because this is just a script, the binary isn't meant to be reused
let benchmark_folder = Path::new(file!())
.parent()
.unwrap()
.parent()
.unwrap()
.join("deployment_benchmarks");

for path in std::fs::read_dir(benchmark_folder).unwrap() {
let path = path.unwrap().path();

let test_contract = std::fs::read(&path).expect("failed to read file");
Expand Down

0 comments on commit c0f3104

Please sign in to comment.