diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml new file mode 100644 index 00000000..e7fa898d --- /dev/null +++ b/.github/workflows/bencher.yml @@ -0,0 +1,67 @@ +name: Bencher + +on: + push: + branches: main + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + benchmark_base: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: bencherdev/bencher@main + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Base branch benchmarks + run: | + bencher run \ + --branch main \ + --testbed ubuntu-latest \ + --err + env: + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_CMD: cargo bench --all-features + BENCHER_PROJECT: socketioxide + RUSTFLAGS: --cfg=socketioxide_test + + benchmark_pr: + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + permissions: + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - uses: bencherdev/bencher@main + - name: PR benchmarks + run: | + bencher run \ + --branch '${{ github.head_ref }}' \ + --branch-start-point '${{ github.base_ref }}' \ + --branch-start-point-hash '${{ github.event.pull_request.base.sha }}' \ + --testbed ubuntu-latest \ + --err \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' + env: + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_CMD: cargo bench --all-features + BENCHER_PROJECT: socketioxide + RUSTFLAGS: --cfg=socketioxide_test \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 257632d1..a54e2e80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,5 +38,7 @@ pin-project-lite = "0.2.13" # Dev deps tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -criterion = { version = "0.5.1", features = ["html_reports"] } +criterion = { version = "0.5.1", features = [ + "rayon", +], default-features = false } axum = "0.7.2" diff --git a/engineioxide/benches/packet_decode.rs b/engineioxide/benches/packet_decode.rs index ea223048..2984e42f 100644 --- a/engineioxide/benches/packet_decode.rs +++ b/engineioxide/benches/packet_decode.rs @@ -1,31 +1,34 @@ +use bytes::Bytes; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use engineioxide::Packet; fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("Decode packet ping/pong", |b| { + let mut group = c.benchmark_group("engineio_packet/decode"); + group.bench_function("Decode packet ping/pong", |b| { let packet: String = Packet::Ping.try_into().unwrap(); b.iter(|| Packet::try_from(packet.as_str()).unwrap()) }); - c.bench_function("Decode packet ping/pong upgrade", |b| { + group.bench_function("Decode packet ping/pong upgrade", |b| { let packet: String = Packet::PingUpgrade.try_into().unwrap(); b.iter(|| Packet::try_from(packet.as_str()).unwrap()) }); - c.bench_function("Decode packet message", |b| { + group.bench_function("Decode packet message", |b| { let packet: String = Packet::Message(black_box("Hello").to_string()) .try_into() .unwrap(); b.iter(|| Packet::try_from(packet.as_str()).unwrap()) }); - c.bench_function("Decode packet noop", |b| { + group.bench_function("Decode packet noop", |b| { let packet: String = Packet::Noop.try_into().unwrap(); b.iter(|| Packet::try_from(packet.as_str()).unwrap()) }); - c.bench_function("Decode packet binary b64", |b| { - let packet: String = Packet::Binary(black_box(vec![0x00, 0x01, 0x02, 0x03, 0x04, 0x05])) - .try_into() - .unwrap(); + group.bench_function("Decode packet binary b64", |b| { + const BYTES: Bytes = Bytes::from_static(&[0x00, 0x01, 0x02, 0x03, 0x04, 0x05]); + let packet: String = Packet::Binary(BYTES).try_into().unwrap(); b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); + + group.finish(); } criterion_group!(benches, criterion_benchmark); diff --git a/engineioxide/benches/packet_encode.rs b/engineioxide/benches/packet_encode.rs index 499e9e2b..97492aef 100644 --- a/engineioxide/benches/packet_encode.rs +++ b/engineioxide/benches/packet_encode.rs @@ -1,8 +1,10 @@ +use bytes::Bytes; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use engineioxide::{config::EngineIoConfig, sid::Sid, OpenPacket, Packet, TransportType}; fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("Encode packet open", |b| { + let mut group = c.benchmark_group("engineio_packet/encode"); + group.bench_function("Encode packet open", |b| { let packet = Packet::Open(OpenPacket::new( black_box(TransportType::Polling), black_box(Sid::ZERO), @@ -10,26 +12,29 @@ fn criterion_benchmark(c: &mut Criterion) { )); b.iter(|| TryInto::::try_into(packet.clone())) }); - c.bench_function("Encode packet ping/pong", |b| { + group.bench_function("Encode packet ping/pong", |b| { let packet = Packet::Ping; b.iter(|| TryInto::::try_into(packet.clone())) }); - c.bench_function("Encode packet ping/pong upgrade", |b| { + group.bench_function("Encode packet ping/pong upgrade", |b| { let packet = Packet::PingUpgrade; b.iter(|| TryInto::::try_into(packet.clone())) }); - c.bench_function("Encode packet message", |b| { + group.bench_function("Encode packet message", |b| { let packet = Packet::Message(black_box("Hello").to_string()); b.iter(|| TryInto::::try_into(packet.clone())) }); - c.bench_function("Encode packet noop", |b| { + group.bench_function("Encode packet noop", |b| { let packet = Packet::Noop; b.iter(|| TryInto::::try_into(packet.clone())) }); - c.bench_function("Encode packet binary b64", |b| { - let packet = Packet::Binary(black_box(vec![0x00, 0x01, 0x02, 0x03, 0x04, 0x05])); + group.bench_function("Encode packet binary b64", |b| { + const BYTES: Bytes = Bytes::from_static(&[0x00, 0x01, 0x02, 0x03, 0x04, 0x05]); + let packet = Packet::Binary(BYTES); b.iter(|| TryInto::::try_into(packet.clone())) }); + + group.finish(); } criterion_group!(benches, criterion_benchmark); diff --git a/socketioxide/Cargo.toml b/socketioxide/Cargo.toml index d5d89337..02f4961b 100644 --- a/socketioxide/Cargo.toml +++ b/socketioxide/Cargo.toml @@ -57,7 +57,7 @@ criterion.workspace = true hyper = { workspace = true, features = ["server", "http1"] } hyper-util = { workspace = true, features = ["tokio", "client-legacy"] } http-body-util.workspace = true - +rand = { version = "0.8", default-features = false } # docs.rs-specific configuration [package.metadata.docs.rs] features = ["v4", "extensions", "tracing", "state"] @@ -75,6 +75,6 @@ path = "benches/packet_decode.rs" harness = false [[bench]] -name = "itoa_bench" -path = "benches/itoa_bench.rs" +name = "extensions" +path = "benches/extensions.rs" harness = false diff --git a/socketioxide/benches/extensions.rs b/socketioxide/benches/extensions.rs new file mode 100644 index 00000000..8bd9c05d --- /dev/null +++ b/socketioxide/benches/extensions.rs @@ -0,0 +1,39 @@ +use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion}; +use rand::Rng; +use socketioxide::extensions::Extensions; + +fn bench_extensions(c: &mut Criterion) { + let i = black_box(5i32); + let mut group = c.benchmark_group("extensions"); + group.bench_function("concurrent_inserts", |b| { + let ext = Extensions::new(); + b.iter(|| { + ext.insert(i); + }); + }); + group.bench_function("concurrent_get", |b| { + let ext = Extensions::new(); + ext.insert(i); + b.iter(|| { + ext.get::(); + }) + }); + group.bench_function("concurrent_get_inserts", |b| { + let ext = Extensions::new(); + b.iter_batched( + || rand::thread_rng().gen_range(0..3), + |i| { + if i == 0 { + ext.insert(i); + } else { + ext.get::(); + } + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +criterion_group!(benches, bench_extensions); +criterion_main!(benches); diff --git a/socketioxide/benches/itoa_bench.rs b/socketioxide/benches/itoa_bench.rs deleted file mode 100644 index dd84ba0b..00000000 --- a/socketioxide/benches/itoa_bench.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! # itoa_bench, used to compare best solutions related to this issue: https://github.com/Totodore/socketioxide/issues/143 - -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; - -/// This solution doesn't imply additional buffer or dependency call -fn insert_number_reverse(mut v: u32, res: &mut String) { - let mut buf = [0u8; 10]; - let mut i = 1; - while v > 0 { - let n = (v % 10) as u8; - buf[10 - i] = n + 0x30; - v /= 10; - i += 1; - } - res.push_str(unsafe { std::str::from_utf8_unchecked(&buf[10 - (i - 1)..]) }); -} - -/// This solution uses the itoa crate -fn insert_number_itoa(v: u32, res: &mut String) { - let mut buffer = itoa::Buffer::new(); - res.push_str(buffer.format(v)); -} - -fn bench_itoa(c: &mut Criterion) { - let mut group = c.benchmark_group("itoa"); - for i in [u32::MAX / 200000, u32::MAX / 2, u32::MAX].iter() { - group.bench_with_input(BenchmarkId::new("number_reverse", i), i, |b, i| { - b.iter(|| insert_number_reverse(*i, &mut String::new())) - }); - group.bench_with_input(BenchmarkId::new("number_itoa", i), i, |b, i| { - b.iter(|| insert_number_itoa(*i, &mut String::new())) - }); - } - group.finish(); -} - -criterion_group!(benches, bench_itoa); -criterion_main!(benches); diff --git a/socketioxide/benches/packet_decode.rs b/socketioxide/benches/packet_decode.rs index ac190e09..db50c769 100644 --- a/socketioxide/benches/packet_decode.rs +++ b/socketioxide/benches/packet_decode.rs @@ -1,3 +1,4 @@ +use bytes::Bytes; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use engineioxide::sid::Sid; use socketioxide::{ @@ -5,14 +6,15 @@ use socketioxide::{ ProtocolVersion, }; fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("Decode packet connect on /", |b| { + let mut group = c.benchmark_group("socketio_packet/decode"); + group.bench_function("Decode packet connect on /", |b| { let packet: String = Packet::connect(black_box("/"), black_box(Sid::ZERO), ProtocolVersion::V5) .try_into() .unwrap(); b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet connect on /custom_nsp", |b| { + group.bench_function("Decode packet connect on /custom_nsp", |b| { let packet: String = Packet::connect( black_box("/custom_nsp"), black_box(Sid::ZERO), @@ -24,8 +26,8 @@ fn criterion_benchmark(c: &mut Criterion) { }); const DATA: &str = r#"{"_placeholder":true,"num":0}"#; - const BINARY: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; - c.bench_function("Decode packet event on /", |b| { + const BINARY: Bytes = Bytes::from_static(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + group.bench_function("Decode packet event on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::event(black_box("/"), black_box("event"), black_box(data.clone())) @@ -34,7 +36,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet event on /custom_nsp", |b| { + group.bench_function("Decode packet event on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::event( black_box("custom_nsp"), @@ -46,7 +48,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet event with ack on /", |b| { + group.bench_function("Decode packet event with ack on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: Packet = Packet::event(black_box("/"), black_box("event"), black_box(data.clone())); @@ -58,7 +60,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet event with ack on /custom_nsp", |b| { + group.bench_function("Decode packet event with ack on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::event( black_box("/custom_nsp"), @@ -74,7 +76,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet ack on /", |b| { + group.bench_function("Decode packet ack on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::ack(black_box("/"), black_box(data.clone()), black_box(0)) .try_into() @@ -82,7 +84,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet ack on /custom_nsp", |b| { + group.bench_function("Decode packet ack on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::ack( black_box("/custom_nsp"), @@ -94,38 +96,38 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet binary event (b64) on /", |b| { + group.bench_function("Decode packet binary event (b64) on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::bin_event( black_box("/"), black_box("event"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), ) .try_into() .unwrap(); b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet binary event (b64) on /custom_nsp", |b| { + group.bench_function("Decode packet binary event (b64) on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::bin_event( black_box("/custom_nsp"), black_box("event"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), ) .try_into() .unwrap(); b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet binary ack (b64) on /", |b| { + group.bench_function("Decode packet binary ack (b64) on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::bin_ack( black_box("/"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), black_box(0), ) .try_into() @@ -133,18 +135,20 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); - c.bench_function("Decode packet binary ack (b64) on /custom_nsp", |b| { + group.bench_function("Decode packet binary ack (b64) on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet: String = Packet::bin_ack( black_box("/custom_nsp"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), black_box(0), ) .try_into() .unwrap(); b.iter(|| Packet::try_from(packet.clone()).unwrap()) }); + + group.finish(); } criterion_group!(benches, criterion_benchmark); diff --git a/socketioxide/benches/packet_encode.rs b/socketioxide/benches/packet_encode.rs index b55f912a..d55ee92f 100644 --- a/socketioxide/benches/packet_encode.rs +++ b/socketioxide/benches/packet_encode.rs @@ -1,3 +1,4 @@ +use bytes::Bytes; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use engineioxide::sid::Sid; use socketioxide::{ @@ -5,13 +6,14 @@ use socketioxide::{ ProtocolVersion, }; fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("Encode packet connect on /", |b| { + let mut group = c.benchmark_group("socketio_packet/encode"); + group.bench_function("Encode packet connect on /", |b| { let packet = Packet::connect(black_box("/"), black_box(Sid::ZERO), ProtocolVersion::V5); b.iter(|| { let _: String = packet.clone().try_into().unwrap(); }) }); - c.bench_function("Encode packet connect on /custom_nsp", |b| { + group.bench_function("Encode packet connect on /custom_nsp", |b| { let packet = Packet::connect( black_box("/custom_nsp"), black_box(Sid::ZERO), @@ -23,8 +25,9 @@ fn criterion_benchmark(c: &mut Criterion) { }); const DATA: &str = r#"{"_placeholder":true,"num":0}"#; - const BINARY: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; - c.bench_function("Encode packet event on /", |b| { + const BINARY: Bytes = Bytes::from_static(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + + group.bench_function("Encode packet event on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::event(black_box("/"), black_box("event"), black_box(data.clone())); b.iter(|| { @@ -32,7 +35,7 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - c.bench_function("Encode packet event on /custom_nsp", |b| { + group.bench_function("Encode packet event on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::event( black_box("custom_nsp"), @@ -44,7 +47,7 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - c.bench_function("Encode packet event with ack on /", |b| { + group.bench_function("Encode packet event with ack on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::event(black_box("/"), black_box("event"), black_box(data.clone())); match packet.inner { @@ -56,7 +59,7 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - c.bench_function("Encode packet event with ack on /custom_nsp", |b| { + group.bench_function("Encode packet event with ack on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::event( black_box("/custom_nsp"), @@ -72,7 +75,7 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - c.bench_function("Encode packet ack on /", |b| { + group.bench_function("Encode packet ack on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::ack(black_box("/"), black_box(data.clone()), black_box(0)); b.iter(|| { @@ -80,7 +83,7 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - c.bench_function("Encode packet ack on /custom_nsp", |b| { + group.bench_function("Encode packet ack on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::ack( black_box("/custom_nsp"), @@ -92,38 +95,38 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - c.bench_function("Encode packet binary event (b64) on /", |b| { + group.bench_function("Encode packet binary event (b64) on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::bin_event( black_box("/"), black_box("event"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), ); b.iter(|| { let _: String = packet.clone().try_into().unwrap(); }) }); - c.bench_function("Encode packet binary event (b64) on /custom_nsp", |b| { + group.bench_function("Encode packet binary event (b64) on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::bin_event( black_box("/custom_nsp"), black_box("event"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), ); b.iter(|| { let _: String = packet.clone().try_into().unwrap(); }) }); - c.bench_function("Encode packet binary ack (b64) on /", |b| { + group.bench_function("Encode packet binary ack (b64) on /", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::bin_ack( black_box("/"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), black_box(0), ); b.iter(|| { @@ -131,18 +134,20 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - c.bench_function("Encode packet binary ack (b64) on /custom_nsp", |b| { + group.bench_function("Encode packet binary ack (b64) on /custom_nsp", |b| { let data = serde_json::to_value(DATA).unwrap(); let packet = Packet::bin_ack( black_box("/custom_nsp"), black_box(data.clone()), - black_box(vec![BINARY.to_vec().clone()]), + black_box(vec![BINARY.clone()]), black_box(0), ); b.iter(|| { let _: String = packet.clone().try_into().unwrap(); }) }); + + group.finish(); } criterion_group!(benches, criterion_benchmark);