chore(ci/bench): add bencher configuration (#307)

Totodore · Apr 19, 2024 · 22b1722 · 22b1722
1 parent b429c13
commit 22b1722
Show file tree

Hide file tree

Showing 9 changed files with 178 additions and 91 deletions.
diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml
@@ -0,0 +1,67 @@
+name: Bencher
+
+on:
+  push:
+    branches: main
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
+
+jobs:
+  benchmark_base:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: bencherdev/bencher@main
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - name: Base branch benchmarks
+        run: |
+          bencher run \
+          --branch main \
+          --testbed ubuntu-latest \
+          --err
+        env:
+          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+          BENCHER_CMD: cargo bench --all-features
+          BENCHER_PROJECT: socketioxide
+          RUSTFLAGS: --cfg=socketioxide_test
+
+  benchmark_pr:
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - uses: bencherdev/bencher@main
+      - name: PR benchmarks
+        run: |
+          bencher run \
+          --branch '${{ github.head_ref }}' \
+          --branch-start-point '${{ github.base_ref }}' \
+          --branch-start-point-hash '${{ github.event.pull_request.base.sha }}' \
+          --testbed ubuntu-latest \
+          --err \
+          --github-actions '${{ secrets.GITHUB_TOKEN }}'
+        env:
+          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+          BENCHER_CMD: cargo bench --all-features
+          BENCHER_PROJECT: socketioxide
+          RUSTFLAGS: --cfg=socketioxide_test
diff --git a/Cargo.toml b/Cargo.toml
@@ -38,5 +38,7 @@ pin-project-lite = "0.2.13"
 
 # Dev deps
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
-criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.5.1", features = [
+    "rayon",
+], default-features = false }
 axum = "0.7.2"
diff --git a/engineioxide/benches/packet_decode.rs b/engineioxide/benches/packet_decode.rs
@@ -1,31 +1,34 @@
+use bytes::Bytes;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use engineioxide::Packet;
 
 fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("Decode packet ping/pong", |b| {
+    let mut group = c.benchmark_group("engineio_packet/decode");
+    group.bench_function("Decode packet ping/pong", |b| {
         let packet: String = Packet::Ping.try_into().unwrap();
         b.iter(|| Packet::try_from(packet.as_str()).unwrap())
     });
-    c.bench_function("Decode packet ping/pong upgrade", |b| {
+    group.bench_function("Decode packet ping/pong upgrade", |b| {
         let packet: String = Packet::PingUpgrade.try_into().unwrap();
         b.iter(|| Packet::try_from(packet.as_str()).unwrap())
     });
-    c.bench_function("Decode packet message", |b| {
+    group.bench_function("Decode packet message", |b| {
         let packet: String = Packet::Message(black_box("Hello").to_string())
             .try_into()
             .unwrap();
         b.iter(|| Packet::try_from(packet.as_str()).unwrap())
     });
-    c.bench_function("Decode packet noop", |b| {
+    group.bench_function("Decode packet noop", |b| {
         let packet: String = Packet::Noop.try_into().unwrap();
         b.iter(|| Packet::try_from(packet.as_str()).unwrap())
     });
-    c.bench_function("Decode packet binary b64", |b| {
-        let packet: String = Packet::Binary(black_box(vec![0x00, 0x01, 0x02, 0x03, 0x04, 0x05]))
-            .try_into()
-            .unwrap();
+    group.bench_function("Decode packet binary b64", |b| {
+        const BYTES: Bytes = Bytes::from_static(&[0x00, 0x01, 0x02, 0x03, 0x04, 0x05]);
+        let packet: String = Packet::Binary(BYTES).try_into().unwrap();
         b.iter(|| Packet::try_from(packet.clone()).unwrap())
     });
+
+    group.finish();
 }
 
 criterion_group!(benches, criterion_benchmark);

diff --git a/engineioxide/benches/packet_encode.rs b/engineioxide/benches/packet_encode.rs
@@ -1,35 +1,40 @@
+use bytes::Bytes;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use engineioxide::{config::EngineIoConfig, sid::Sid, OpenPacket, Packet, TransportType};
 
 fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("Encode packet open", |b| {
+    let mut group = c.benchmark_group("engineio_packet/encode");
+    group.bench_function("Encode packet open", |b| {
         let packet = Packet::Open(OpenPacket::new(
             black_box(TransportType::Polling),
             black_box(Sid::ZERO),
             &EngineIoConfig::default(),
         ));
         b.iter(|| TryInto::<String>::try_into(packet.clone()))
     });
-    c.bench_function("Encode packet ping/pong", |b| {
+    group.bench_function("Encode packet ping/pong", |b| {
         let packet = Packet::Ping;
         b.iter(|| TryInto::<String>::try_into(packet.clone()))
     });
-    c.bench_function("Encode packet ping/pong upgrade", |b| {
+    group.bench_function("Encode packet ping/pong upgrade", |b| {
         let packet = Packet::PingUpgrade;
         b.iter(|| TryInto::<String>::try_into(packet.clone()))
     });
-    c.bench_function("Encode packet message", |b| {
+    group.bench_function("Encode packet message", |b| {
         let packet = Packet::Message(black_box("Hello").to_string());
         b.iter(|| TryInto::<String>::try_into(packet.clone()))
     });
-    c.bench_function("Encode packet noop", |b| {
+    group.bench_function("Encode packet noop", |b| {
         let packet = Packet::Noop;
         b.iter(|| TryInto::<String>::try_into(packet.clone()))
     });
-    c.bench_function("Encode packet binary b64", |b| {
-        let packet = Packet::Binary(black_box(vec![0x00, 0x01, 0x02, 0x03, 0x04, 0x05]));
+    group.bench_function("Encode packet binary b64", |b| {
+        const BYTES: Bytes = Bytes::from_static(&[0x00, 0x01, 0x02, 0x03, 0x04, 0x05]);
+        let packet = Packet::Binary(BYTES);
         b.iter(|| TryInto::<String>::try_into(packet.clone()))
     });
+
+    group.finish();
 }
 
 criterion_group!(benches, criterion_benchmark);

diff --git a/socketioxide/Cargo.toml b/socketioxide/Cargo.toml
@@ -57,7 +57,7 @@ criterion.workspace = true
 hyper = { workspace = true, features = ["server", "http1"] }
 hyper-util = { workspace = true, features = ["tokio", "client-legacy"] }
 http-body-util.workspace = true
-
+rand = { version = "0.8", default-features = false }
 # docs.rs-specific configuration
 [package.metadata.docs.rs]
 features = ["v4", "extensions", "tracing", "state"]
@@ -75,6 +75,6 @@ path = "benches/packet_decode.rs"
 harness = false
 
 [[bench]]
-name = "itoa_bench"
-path = "benches/itoa_bench.rs"
+name = "extensions"
+path = "benches/extensions.rs"
 harness = false
diff --git a/socketioxide/benches/extensions.rs b/socketioxide/benches/extensions.rs
@@ -0,0 +1,39 @@
+use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion};
+use rand::Rng;
+use socketioxide::extensions::Extensions;
+
+fn bench_extensions(c: &mut Criterion) {
+    let i = black_box(5i32);
+    let mut group = c.benchmark_group("extensions");
+    group.bench_function("concurrent_inserts", |b| {
+        let ext = Extensions::new();
+        b.iter(|| {
+            ext.insert(i);
+        });
+    });
+    group.bench_function("concurrent_get", |b| {
+        let ext = Extensions::new();
+        ext.insert(i);
+        b.iter(|| {
+            ext.get::<i32>();
+        })
+    });
+    group.bench_function("concurrent_get_inserts", |b| {
+        let ext = Extensions::new();
+        b.iter_batched(
+            || rand::thread_rng().gen_range(0..3),
+            |i| {
+                if i == 0 {
+                    ext.insert(i);
+                } else {
+                    ext.get::<i32>();
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+criterion_group!(benches, bench_extensions);
+criterion_main!(benches);
diff --git a/socketioxide/benches/itoa_bench.rs b/socketioxide/benches/itoa_bench.rs