From 9ef03558974f998c713b4295c8508010fb430112 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 10 Oct 2024 23:19:18 -0700 Subject: [PATCH] test: Dump simulator RNG seeds (#2165) * test: Dump simulator RNG seeds Into the directory given in the `DUMP_SIMULATION_SEEDS` environment variable. Also, export them as artifacts from the CI runs. Fixes #1645 * mkdir * Print leading zeros * TypoPrint leading zeros * Fixes * Conditional * Separate out sanitizer runs * Per sanitizer * Update test-fixture/src/sim/rng.rs Co-authored-by: Martin Thomson Signed-off-by: Lars Eggert * Update .github/workflows/check.yml Co-authored-by: Martin Thomson Signed-off-by: Lars Eggert * Update test-fixture/src/sim/mod.rs Co-authored-by: Martin Thomson Signed-off-by: Lars Eggert * Prefix seed with test name * Merge --------- Signed-off-by: Lars Eggert Co-authored-by: Martin Thomson --- .github/workflows/check.yml | 15 ++++++++++++-- .github/workflows/sanitize.yml | 9 ++++++++ test-fixture/src/sim/mod.rs | 38 ++++++++++++++++++++++------------ test-fixture/src/sim/rng.rs | 2 +- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 03bc943338..d39d232892 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -17,6 +17,7 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + DUMP_SIMULATION_SEEDS: /tmp/simulation-seeds concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -82,12 +83,14 @@ jobs: cargo +${{ matrix.rust-toolchain }} check $BUILD_TYPE --all-targets --features ci - name: Run tests and determine coverage + env: + RUST_LOG: trace run: | # shellcheck disable=SC2086 if [ "${{ matrix.rust-toolchain }}" == "stable" ]; then - RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info + cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info else - RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast + cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast fi - name: Run client/server transfer @@ -117,6 +120,14 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} if: matrix.type == 'debug' && matrix.rust-toolchain == 'stable' + - name: Save simulation seeds artifact + if: env.DUMP_SIMULATION_SEEDS + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: simulation-seeds-${{ matrix.os }}-${{ matrix.rust-toolchain }}-${{ matrix.type }} + path: ${{ env.DUMP_SIMULATION_SEEDS }} + compression-level: 9 + bench: needs: [check] if: > diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 0752c7e306..71ca8ad2f2 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -11,6 +11,7 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + DUMP_SIMULATION_SEEDS: /tmp/simulation-seeds concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -76,3 +77,11 @@ jobs: export LSAN_OPTIONS="suppressions=$PWD/suppressions.txt" fi cargo nextest run -Z build-std --features ci --target "$TARGET" + + - name: Save simulation seeds artifact + if: env.DUMP_SIMULATION_SEEDS + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: simulation-seeds-${{ matrix.os }}-sanitizer-${{ matrix.sanitizer }} + path: ${{ env.DUMP_SIMULATION_SEEDS }} + compression-level: 9 diff --git a/test-fixture/src/sim/mod.rs b/test-fixture/src/sim/mod.rs index 17e7f2b465..14c21dd75c 100644 --- a/test-fixture/src/sim/mod.rs +++ b/test-fixture/src/sim/mod.rs @@ -15,12 +15,14 @@ use std::{ cell::RefCell, cmp::min, fmt::Debug, + fs::{create_dir_all, File}, ops::{Deref, DerefMut}, + path::PathBuf, rc::Rc, time::{Duration, Instant}, }; -use neqo_common::{qdebug, qinfo, qtrace, Datagram, Encoder}; +use neqo_common::{qdebug, qerror, qinfo, qtrace, Datagram, Encoder}; use neqo_transport::Output; use rng::Random; use NodeState::{Active, Idle, Waiting}; @@ -64,11 +66,7 @@ macro_rules! simulate { let f: Box _> = Box::new($v); nodes.push(Box::new(f(&fixture))); )* - let mut sim = Simulator::new(stringify!($n), nodes); - if let Ok(seed) = std::env::var("SIMULATION_SEED") { - sim.seed_str(seed); - } - sim.run(); + Simulator::new(stringify!($n), nodes).run(); } }; } @@ -151,23 +149,37 @@ impl Simulator { .into_iter() .chain(it.map(|node| NodeHolder { node, state: Idle })) .collect::>(); - Self { + let mut sim = Self { name, nodes, rng: Rc::default(), + }; + // Seed from the `SIMULATION_SEED` environment variable, if set. + if let Ok(seed) = std::env::var("SIMULATION_SEED") { + sim.seed_str(seed); } - } - - pub fn seed(&mut self, seed: [u8; 32]) { - self.rng = Rc::new(RefCell::new(Random::new(&seed))); + // Dump the seed to a file to the directory in the `DUMP_SIMULATION_SEEDS` environment + // variable, if set. + if let Ok(dir) = std::env::var("DUMP_SIMULATION_SEEDS") { + if create_dir_all(&dir).is_err() { + qerror!("Failed to create directory {dir}"); + } else { + let seed_str = sim.rng.borrow().seed_str(); + let path = PathBuf::from(format!("{dir}/{}-{seed_str}", sim.name)); + if File::create(&path).is_err() { + qerror!("Failed to write seed to {}", path.to_string_lossy()); + } + } + } + sim } /// Seed from a hex string. /// # Panics /// When the provided string is not 32 bytes of hex (64 characters). pub fn seed_str(&mut self, seed: impl AsRef) { - let seed = Encoder::from_hex(seed); - self.seed(<[u8; 32]>::try_from(seed.as_ref()).unwrap()); + let seed = <[u8; 32]>::try_from(Encoder::from_hex(seed).as_ref()).unwrap(); + self.rng = Rc::new(RefCell::new(Random::new(&seed))); } fn next_time(&self, now: Instant) -> Instant { diff --git a/test-fixture/src/sim/rng.rs b/test-fixture/src/sim/rng.rs index 1ad36d2baf..dd94798252 100644 --- a/test-fixture/src/sim/rng.rs +++ b/test-fixture/src/sim/rng.rs @@ -70,7 +70,7 @@ impl Random { #[must_use] pub fn seed_str(&self) -> String { format!( - "{:8x}{:8x}{:8x}{:8x}", + "{:016x}{:016x}{:016x}{:016x}", self.state[0], self.state[1], self.state[2], self.state[3], ) }