feat(sqlsmith): add statement-level reducer (#8507)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
risingwavelabs · Mar 15, 2023 · 8be23f4 · 8be23f4
1 parent 320d755
commit 8be23f4
Show file tree

Hide file tree

Showing 7 changed files with 506 additions and 41 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml
@@ -20,6 +20,7 @@ clap = { version = "4", features = ["derive"] }
 itertools = "0.10"
 rand = { version = "0.8", features = ["small_rng"] }
 rand_chacha = { version = "0.3.1" }
+regex = "1"
 risingwave_common = { path = "../../common" }
 risingwave_expr = { path = "../../expr" }
 risingwave_frontend = { path = "../../frontend" }

diff --git a/src/tests/sqlsmith/scripts/gen_queries.sh b/src/tests/sqlsmith/scripts/gen_queries.sh
@@ -30,6 +30,7 @@ echo_err() {
 }
 
 ################## EXTRACT
+# TODO(kwannoel): Write tests for these
 
 # Get reason for generation crash.
 get_failure_reason() {
@@ -67,7 +68,7 @@ extract_failing_query() {
   grep "\[EXECUTING .*\]: " | tail -n 1 | sed -E 's/^.*\[EXECUTING .*\]: (.*)$/\1;/' || true
 }
 
-# Extract fail info from logs in log dir
+# Extract fail info from [`generate-*.log`] in log dir
 extract_fail_info_from_logs() {
   for LOGFILENAME in $(ls "$LOGDIR" | grep "generate")
   do
@@ -86,14 +87,17 @@ extract_fail_info_from_logs() {
       FAIL_DIR="$OUTDIR/failed/$SEED"
       mkdir -p "$FAIL_DIR"
       echo -e "$DDL" "\n$GLOBAL_SESSION" "\n$DML" "\n$TEST_SESSION" "\n$QUERY" > "$FAIL_DIR/queries.sql"
-      echo_err "[INFO] WROTE FAIL QUERY to $FAIL_DIR/queries.log"
+      echo_err "[INFO] WROTE FAIL QUERY to $FAIL_DIR/queries.sql"
       echo -e "$REASON" > "$FAIL_DIR/fail.log"
       echo_err "[INFO] WROTE FAIL REASON to $FAIL_DIR/fail.log"
+
       cp "$LOGFILE" "$FAIL_DIR/$LOGFILENAME"
     fi
   done
 }
 
+################# Generate
+
 # Prefer to use [`generate_deterministic`], it is faster since
 # runs with all-in-one binary.
 generate_deterministic() {
@@ -123,6 +127,8 @@ generate_sqlsmith() {
     --generate "$OUTDIR/$1"
 }
 
+############################# Checks
+
 # Check that queries are different
 check_different_queries() {
   if [[ -z $(diff "$OUTDIR/1/queries.sql" "$OUTDIR/2/queries.sql") ]]; then
@@ -142,34 +148,6 @@ check_failed_to_generate_queries() {
   fi
 }
 
-# sync step
-# Some queries maybe be added
-sync_queries() {
-  set +x
-  pushd $OUTDIR
-  git checkout main
-  git pull
-  set +e
-  git branch -D stage
-  set -e
-  git checkout -b stage
-  popd
-  set -x
-}
-
-# Upload step
-upload_queries() {
-  set +x
-  pushd "$OUTDIR"
-  git add .
-  git commit -m 'update queries'
-  git push -f origin stage
-  git checkout -
-  git branch -D stage
-  popd
-  set -x
-}
-
 # Run it to make sure it should have no errors
 run_queries() {
   echo "" > $LOGDIR/run_deterministic.stdout.log
@@ -180,6 +158,7 @@ run_queries() {
       && rm $LOGDIR/fuzzing-{}.log"
 }
 
+# Generated query sets should not fail.
 check_failed_to_run_queries() {
   FAILED_LOGS=$(ls "$LOGDIR" | grep fuzzing || true)
   if [[ -n "$FAILED_LOGS" ]]; then
@@ -223,11 +202,39 @@ validate() {
   echo_err "[INFO] Passed checks"
 }
 
+# sync step
+# Some queries maybe be added
+sync_queries() {
+  set +x
+  pushd $OUTDIR
+  git checkout main
+  git pull
+  set +e
+  git branch -D stage
+  set -e
+  git checkout -b stage
+  popd
+  set -x
+}
+
 sync() {
   sync_queries
   echo_err "[INFO] Synced"
 }
 
+# Upload step
+upload_queries() {
+  set +x
+  pushd "$OUTDIR"
+  git add .
+  git commit -m 'update queries'
+  git push -f origin stage
+  git checkout -
+  git branch -D stage
+  popd
+  set -x
+}
+
 upload() {
   upload_queries
   echo_err "[INFO] Uploaded"
@@ -238,6 +245,8 @@ cleanup() {
   echo_err "[INFO] Success!"
 }
 
+################### MAIN
+
 main() {
   setup
 
@@ -250,4 +259,4 @@ main() {
   cleanup
 }
 
-main
+main
diff --git a/src/tests/sqlsmith/src/lib.rs b/src/tests/sqlsmith/src/lib.rs
@@ -15,6 +15,7 @@
 #![feature(let_chains)]
 #![feature(if_let_guard)]
 #![feature(once_cell)]
+#![feature(box_patterns)]
 
 use rand::prelude::SliceRandom;
 use rand::Rng;
@@ -25,8 +26,10 @@ use risingwave_sqlparser::parser::Parser;
 
 use crate::sql_gen::SqlGenerator;
 
+pub mod reducer;
 pub mod runner;
 mod sql_gen;
+mod utils;
 pub mod validation;
 pub use validation::is_permissible_error;
 
@@ -76,7 +79,8 @@ pub fn session_sql_gen<R: Rng>(rng: &mut R) -> String {
 
 /// Parse SQL
 /// FIXME(Noel): Introduce error type for sqlsmith for this.
-pub fn parse_sql(sql: &str) -> Vec<Statement> {
+pub fn parse_sql<S: AsRef<str>>(sql: S) -> Vec<Statement> {
+    let sql = sql.as_ref();
     Parser::parse_sql(sql).unwrap_or_else(|_| panic!("Failed to parse SQL: {}", sql))
 }