feat(sqlsmith, deterministic-test): deterministic fuzz stability

ci/scripts/cron-e2e-test.sh

@@ -7,4 +7,5 @@ source ci/scripts/common.env.sh
 export RUN_COMPACTION=1;
 export RUN_META_BACKUP=1;
 export RUN_DELETE_RANGE=1;
+export RUN_DETERMINISTIC_SQLSMITH=1;
 source ci/scripts/run-e2e-test.sh

ci/scripts/deterministic-e2e-test.sh

@@ -4,6 +4,7 @@
 set -euo pipefail
 
 source ci/scripts/common.env.sh
+source ci/scripts/pr.env.sh
 
 echo "--- Download artifacts"
 buildkite-agent artifact download risingwave_simulation .
@@ -15,6 +16,11 @@ mkdir -p ./test_data
 unzip -o test_data.zip -d .
 cd ../../
 
+echo "--- Extract data for sqlsmith"
+cd ./src/tests/sqlsmith/tests
+unzip freeze.zip
+cd -
+
 export RUST_LOG=info
 export LOGDIR=.risingwave/log
 
@@ -38,5 +44,10 @@ seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2
 echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
 
-echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (pre-generated-queries)"
+seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries src/tests/sqlsmith/tests/freeze/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+
+if [[ "$RUN_DETERMINISTIC_SQLSMITH" -eq "1" ]]; then
+  echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)"
+  seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+fi

ci/scripts/pr.env.sh

@@ -20,6 +20,8 @@ export RUN_COMPACTION=0;
 export RUN_META_BACKUP=0;
 # Don't run delete-range random test
 export RUN_DELETE_RANGE=0;
+# Don't run deterministic e2e fuzzing (only run pre-gen)
+export RUN_DETERMINISTIC_SQLSMITH=0;
 
 if [[ -n "$CHANGED" ]]; then
     echo "origin/main SHA: $(git rev-parse origin/main)";

scripts/sqlsmith/gen_queries.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+export TEST_NUM=32
+export LOGDIR=".risingwave/log"
+export TESTS_FOLDER="./src/tests/sqlsmith/tests"
+export BASE_FOLDER="$TESTS_FOLDER/freeze"
+
+generate_deterministic() {
+    seq "$TEST_NUM" | \
+      parallel "mkdir -p $BASE_FOLDER/{}; \
+       MADSIM_TEST_SEED={} \
+       ./target/sim/ci-sim/risingwave_simulation \
+        --sqlsmith 100 \
+        --generate-sqlsmith-queries $BASE_FOLDER/{} \
+        ./src/tests/sqlsmith/tests/testdata \
+         2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log"
+}
+
+generate_sqlsmith() {
+  mkdir -p "$BASE_FOLDER/$1"
+  ./risedev d
+  ./target/debug/sqlsmith test \
+    --testdata ./src/tests/sqlsmith/tests/testdata \
+    --generate "$BASE_FOLDER/$1"
+}
+
+cargo make sslt-build-all --profile ci-sim
+
+generate_deterministic
+
+cd $TESTS_FOLDER
+zip -r "freeze.zip" freeze
+rm -r freeze
+cd -

src/tests/simulation/src/main.rs

@@ -111,6 +111,20 @@ pub struct Args {
     #[clap(long)]
     sqlsmith: Option<usize>,
 
+    /// Run sqlsmith pre-generated queries with the given [`files`] directory,
+    /// containing `ddl.sql` and `queries.sql`.
+    #[clap(long)]
+    run_sqlsmith_queries: bool,
+
+    /// Run sqlsmith to generate queries with the given testdata [`files`],
+    /// and output the ddl + queries to the given directory,
+    /// indicated by this argument.
+    /// We generate sqlsmith queries via `madsim` because
+    /// it provides a degree of determinism, and we can spawn several
+    /// instances in parallel.
+    #[clap(long)]
+    generate_sqlsmith_queries: Option<String>,
+
     /// Load etcd data from toml file.
     #[clap(long)]
     etcd_data: Option<PathBuf>,
@@ -172,7 +186,30 @@ async fn main() {
                 let rw = RisingWave::connect("frontend".into(), "dev".into())
                     .await
                     .unwrap();
-                risingwave_sqlsmith::runner::run(rw.pg_client(), &args.files, count).await;
+                if let Some(outdir) = args.generate_sqlsmith_queries {
+                    risingwave_sqlsmith::runner::generate(
+                        rw.pg_client(),
+                        &args.files,
+                        count,
+                        &outdir,
+                    )
+                    .await;
+                } else {
+                    risingwave_sqlsmith::runner::run(rw.pg_client(), &args.files, count).await;
+                }
+            })
+            .await;
+        return;
+    }
+
+    if args.run_sqlsmith_queries {
+        let outdir = args.files;
+        cluster
+            .run_on_client(async move {
+                let rw = RisingWave::connect("frontend".into(), "dev".into())
+                    .await
+                    .unwrap();
+                risingwave_sqlsmith::runner::run_pre_generated(rw.pg_client(), &outdir).await;
             })
             .await;
         return;

src/tests/sqlsmith/src/bin/main.rs

@@ -17,7 +17,7 @@ use std::time::Duration;
 
 use clap::Parser as ClapParser;
 use risingwave_sqlsmith::print_function_table;
-use risingwave_sqlsmith::runner::run;
+use risingwave_sqlsmith::runner::{generate, run};
 use tokio_postgres::NoTls;
 
 #[derive(ClapParser, Debug, Clone)]
@@ -56,6 +56,11 @@ struct TestOptions {
     /// The number of test cases to generate.
     #[clap(long, default_value = "100")]
     count: usize,
+
+    /// Output directory - only applicable if we are generating
+    /// query while testing.
+    #[clap(long)]
+    generate: Option<String>,
 }
 
 #[derive(clap::Subcommand, Clone, Debug)]
@@ -73,7 +78,8 @@ async fn main() {
     tracing_subscriber::fmt::init();
 
     let opt = Opt::parse();
-    let opt = match opt.command {
+    let command = opt.command;
+    let opt = match command {
         Commands::PrintFunctionTable => {
             println!("{}", print_function_table());
             return;
@@ -95,5 +101,9 @@ async fn main() {
             tracing::error!("Postgres connection error: {:?}", e);
         }
     });
-    run(&client, &opt.testdata, opt.count).await;
+    if let Some(outdir) = opt.generate {
+        generate(&client, &opt.testdata, opt.count, &outdir).await;
+    } else {
+        run(&client, &opt.testdata, opt.count).await;
+    }
 }