Skip to content

Commit

Permalink
ci: Run benchmarks with Ethernet and max MTUs
Browse files Browse the repository at this point in the history
  • Loading branch information
@larseggert
larseggert committed Oct 15, 2024
1 parent 9b5ec71 commit 025e604
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 40 deletions.
73 changes: 39 additions & 34 deletions .github/workflows/bench.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,10 @@ jobs:
#
# Run all benchmarks at elevated priority.
taskset -c 0 nice -n -20 cargo "+$TOOLCHAIN" bench --workspace --exclude neqo-bin --features bench -- --noplot | tee results.txt
nice -n -20 cargo "+$TOOLCHAIN" bench --package neqo-bin --features bench -- --noplot | tee -a results.txt
for MTU in 1500 65536; do
sudo ip link set dev lo mtu "$MTU"
nice -n -20 cargo "+$TOOLCHAIN" bench --package neqo-bin --features bench -- --noplot | tee -a results.txt
done
# Compare various configurations of neqo against msquic, and gather perf data
# during the hyperfine runs.
Expand Down Expand Up @@ -132,45 +134,48 @@ jobs:
fi
}
for server in msquic neqo; do
for client in msquic neqo; do
# msquic doesn't let us configure the congestion control or pacing.
if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
cc_opt=("")
pacing_opt=("")
else
cc_opt=("reno" "cubic")
pacing_opt=("on" "")
fi
for cc in "${cc_opt[@]}"; do
for pacing in "${pacing_opt[@]}"; do
# Make a tag string for this test, for the results.
TAG="$client,$server,$cc,$pacing"
echo "Running benchmarks for $TAG" | tee -a comparison.txt
transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
# shellcheck disable=SC2086
taskset -c 0 nice -n -20 \
perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
PID=$!
transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
# shellcheck disable=SC2086
taskset -c 1 nice -n -20 \
perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
tee -a comparison.txt
echo >> comparison.txt
kill $PID
cat step.md >> steps.md
# Sanity check the size of the last retrieved file.
[ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
for mtu in 1500 65536; do
sudo ip link set dev lo mtu "$mtu"
for server in msquic neqo; do
for client in msquic neqo; do
# msquic doesn't let us configure the congestion control or pacing.
if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
cc_opt=("")
pacing_opt=("")
else
cc_opt=("reno" "cubic")
pacing_opt=("on" "")
fi
for cc in "${cc_opt[@]}"; do
for pacing in "${pacing_opt[@]}"; do
# Make a tag string for this test, for the results.
TAG="$client,$server,$cc,$pacing,$mtu"
echo "Running benchmarks for $TAG" | tee -a comparison.txt
transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
# shellcheck disable=SC2086
taskset -c 0 nice -n -20 \
perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
PID=$!
transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
# shellcheck disable=SC2086
taskset -c 1 nice -n -20 \
perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
tee -a comparison.txt
echo >> comparison.txt
kill $PID
cat step.md >> steps.md
# Sanity check the size of the last retrieved file.
[ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
done
done
done
done
done
# Merge the results tables generated by hyperfine into a single table.
echo "Transfer of $SIZE bytes over loopback." > comparison.md
awk '(!/^\| Command/ || !c++) && (!/^\|:/ || !d++)' < steps.md |\
sed -E 's/`//g; s/^\|:/\|:---\|:---\|:---\|:/g; s/,/ \| /g; s/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
sed -E 's/`//g; s/^\|:/\|:---\|:---\|:---\|:/g; s/,/ \| /g; s/^\| Command/\| Client \| Server \| CC \| Pacing \| MTU/g' >> comparison.md
rm -r "$TMP"
# Re-enable turboboost, hyperthreading and use powersave governor.
Expand Down
15 changes: 9 additions & 6 deletions neqo-bin/benches/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::{path::PathBuf, str::FromStr};
use std::{env, path::PathBuf, str::FromStr};

use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
use neqo_bin::{client, server};
Expand All @@ -20,18 +20,22 @@ fn transfer(c: &mut Criterion) {
neqo_crypto::init_db(PathBuf::from_str("../test-fixture/db").unwrap()).unwrap();

let done_sender = spawn_server();

let mtu = if let Ok(mtu) = env::var("MTU") {
format!("/mtu-{}", mtu)
} else {
"".to_string()
};
for Benchmark { name, requests } in [
Benchmark {
name: "1-conn/1-100mb-resp (aka. Download)".to_string(),
name: format!("1-conn/1-100mb-resp{} (aka. Download)", mtu),
requests: vec![100 * 1024 * 1024],
},
Benchmark {
name: "1-conn/10_000-parallel-1b-resp (aka. RPS)".to_string(),
name: format!("1-conn/10_000-parallel-1b-resp{} (aka. RPS)", mtu),
requests: vec![1; 10_000],
},
Benchmark {
name: "1-conn/1-1b-resp (aka. HPS)".to_string(),
name: format!("1-conn/1-1b-resp{} (aka. HPS)", mtu),
requests: vec![1; 1],
},
] {
Expand All @@ -57,7 +61,6 @@ fn transfer(c: &mut Criterion) {
done_sender.send(()).unwrap();
}

#[allow(clippy::redundant_pub_crate)] // Bug in clippy nursery? Not sure how this lint could fire here.
fn spawn_server() -> tokio::sync::oneshot::Sender<()> {
let (done_sender, mut done_receiver) = tokio::sync::oneshot::channel();
std::thread::spawn(move || {
Expand Down

0 comments on commit 025e604

Please sign in to comment.