Merge branch 'main' into eric/generate-example-config

Cargo.toml

@@ -11,6 +11,7 @@ members = [
   "src/connector",
   "src/ctl",
   "src/expr",
+  "src/expr/macro",
   "src/frontend",
   "src/frontend/planner_test",
   "src/java_binding",
@@ -32,6 +33,7 @@ members = [
   "src/stream",
   "src/test_runner",
   "src/tests/compaction_test",
+  "src/tests/e2e_extended_mode",
   "src/tests/regress",
   "src/tests/simulation",
   "src/tests/sqlsmith",

Makefile.toml

@@ -622,12 +622,13 @@ tar xf ${TARGET_PATH} -C "${PREFIX_BIN}/connector-node"
 category = "RiseDev - Build in simulation mode"
 description = "Build in simulation mode"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+env = { CARGO_TARGET_DIR = "target/sim" }
 script = """
 #!/usr/bin/env bash
 set -e
 
 cargo build \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
   -p risingwave_batch \
   -p risingwave_common \
   -p risingwave_compute \
@@ -646,12 +647,13 @@ cargo build \
 category = "RiseDev - Deterministic Simulation Test"
 description = "Run unit tests in deterministic simulation mode"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+env = { CARGO_TARGET_DIR = "target/sim" }
 script = """
 #!/usr/bin/env bash
 set -e
 
 cargo nextest run \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
   -p risingwave_batch \
   -p risingwave_common \
   -p risingwave_compute \
@@ -670,12 +672,13 @@ cargo nextest run \
 category = "RiseDev - Simulation scaling tests"
 description = "Run integration scaling tests in deterministic simulation mode"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+env = { CARGO_TARGET_DIR = "target/sim" }
 script = """
 #!/usr/bin/env bash
 set -e
 
 cargo nextest run \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
   -p risingwave_simulation \
   "$@"
 """
@@ -684,12 +687,13 @@ cargo nextest run \
 category = "RiseDev - Simulation scaling tests"
 description = "Archive integration scaling tests in deterministic simulation mode"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+env = { CARGO_TARGET_DIR = "target/sim" }
 script = """
 #!/usr/bin/env bash
 set -e
 
 cargo nextest archive \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
   -p risingwave_simulation \
   --archive-file scale-test.tar.zst \
   "$@"
@@ -699,48 +703,58 @@ cargo nextest archive \
 category = "RiseDev - Deterministic Simulation End-to-end Test"
 description = "Run cargo check in deterministic simulation mode"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+env = { CARGO_TARGET_DIR = "target/sim" }
 script = """
 #!/usr/bin/env bash
 set -e
 
-cargo check -p risingwave_simulation --all-targets "$@"
+cargo check \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
+  -p risingwave_simulation --all-targets "$@"
 """
 
 [tasks.sslt]
 category = "RiseDev - Deterministic Simulation End-to-end Test"
 description = "Run e2e tests in deterministic simulation mode"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+env = { CARGO_TARGET_DIR = "target/sim" }
 script = """
 #!/usr/bin/env bash
 set -e
 
-cargo run -p risingwave_simulation "$@"
+cargo run \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
+  -p risingwave_simulation "$@"
 """
 
 [tasks.sslt-build-all]
 category = "RiseDev - Deterministic Simulation End-to-end Test"
 description = "Build deterministic simulation runner and tests"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+env = { CARGO_TARGET_DIR = "target/sim" }
 script = """
 #!/usr/bin/env bash
 set -e
 
-cargo build -p risingwave_simulation --tests "$@"
+cargo build \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
+  -p risingwave_simulation \
+  --tests "$@"
 """
 
 [tasks.sslt-cov]
 category = "RiseDev - Deterministic Simulation End-to-end Test"
 description = "Run e2e tests in deterministic simulation mode and report code coverage"
 dependencies = ["warn-on-missing-tools"]
-env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim-cov" }
+env = { CARGO_TARGET_DIR = "target/sim-cov" }
 script = """
 #!/usr/bin/env bash
 set -e
 
-cargo llvm-cov run -p risingwave_simulation --html "$@"
+cargo llvm-cov run \
+  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
+  -p risingwave_simulation \
+  --html "$@"
 """
 
 [tasks.check-java]

ci/scripts/build.sh

@@ -43,10 +43,11 @@ cargo build \
     -p risingwave_compaction_test \
     -p risingwave_backup_cmd \
     -p risingwave_java_binding \
+    -p risingwave_e2e_extended_mode_test \
     --features "static-link static-log-level" --profile "$profile"
 
 # the file name suffix of artifact for risingwave_java_binding is so only for linux. It is dylib for MacOS
-artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risedev-dev delete-range-test librisingwave_java_binding.so)
+artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test librisingwave_java_binding.so)
 
 echo "--- Show link info"
 ldd target/"$target"/risingwave

ci/scripts/run-e2e-test.sh

@@ -24,12 +24,15 @@ mkdir -p target/debug
 buildkite-agent artifact download risingwave-"$profile" target/debug/
 buildkite-agent artifact download risedev-dev-"$profile" target/debug/
 buildkite-agent artifact download "e2e_test/generated/*" ./
+buildkite-agent artifact download risingwave_e2e_extended_mode_test-"$profile" target/debug/
 mv target/debug/risingwave-"$profile" target/debug/risingwave
 mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
+mv target/debug/risingwave_e2e_extended_mode_test-"$profile" target/debug/risingwave_e2e_extended_mode_test
 
 echo "--- Adjust permission"
 chmod +x ./target/debug/risingwave
 chmod +x ./target/debug/risedev-dev
+chmod +x ./target/debug/risingwave_e2e_extended_mode_test
 
 echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env
@@ -75,7 +78,10 @@ cargo make ci-kill
 echo "--- e2e, ci-3cn-1fe, extended query"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 cargo make ci-start ci-3cn-1fe
-sqllogictest -p 4566 -d dev -e postgres-extended './e2e_test/extended_query/**/*.slt'
+sqllogictest -p 4566 -d dev -e postgres-extended './e2e_test/extended_mode/**/*.slt'
+RUST_BACKTRACE=1 target/debug/risingwave_e2e_extended_mode_test --host 127.0.0.1 \
+  -p 4566 \
+  -u root 
 
 echo "--- Kill cluster"
 cargo make ci-kill

e2e_test/extended_query/basic.slt → e2e_test/extended_mode/basic.slt

@@ -1,14 +1,4 @@
-# The basic.slt is to cover the path of pgwire.
-#
-# There are two kinds of statement, they run different path of pgwire:
-#   1. un-query statement: SET,CREATE,INSERT,FLUSH,EXPLAIN,DROP..
-#   2. query statement: SELECT,WITH,VALUES,SHOW,DESCRIBE..
-# 
-# We also need to test different type in extended query mode:
-#   smallint,int,bigint
-#   real,double precision,numeric
-#   time,date,timestamp
-
+# Test different statements(DDL,DQL,DML) in extended mode.
 
 statement ok
 SET RW_IMPLICIT_FLUSH TO true;
@@ -78,18 +68,3 @@ with t as (select generate_series(1,3,1)) select * from t;
 1
 2
 3
-
-query III
-select 42::smallint, 42::int, 42::bigint;
-----
-42 42 42
-
-query III
-select 42::real,42::double precision,42::decimal;
-----
-42 42 42
-
-query TTT
-select '20:55:12'::time,'2022-07-12'::date,'2022-07-12 20:55:12'::timestamp;
-----
-20:55:12 2022-07-12 2022-07-12 20:55:12

e2e_test/extended_mode/type.slt

@@ -0,0 +1,28 @@
+# Test binary format of different type. (sqllogitest return binary format in extended mode)
+
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+# RisingWave can't support list and struct now so we skip them.
+# include ../batch/types/array.slt.part
+# include ../batch/types/struct.slt.part
+# include ../batch/types/list.slt.part
+
+# Sqllogitest can't support binary format bytea type so we skip it.
+# include ../batch/types/bytea.slt.part
+
+# Can't support inf,-inf binary format now so we skip it.
+# include ../batch/types/decimal.slt.part
+
+# Sqllogitest can't support binary format jsonb type so we skip it.
+# include ../batch/types/jsonb_ord.slt.part
+# include ../batch/types/jsonb.slt.part
+
+include ../batch/types/boolean.slt.part
+include ../batch/types/cast.slt.part
+include ../batch/types/date.slt
+include ../batch/types/intercal.slt.part
+include ../batch/types/number_arithmetic.slt.part
+include ../batch/types/temporal_arithmetic.slt.part
+include ../batch/types/time.slt.part
+include ../batch/types/timestamptz_utc.slt.part

e2e_test/source/basic/kafka.slt

@@ -384,7 +384,7 @@ select id, first_name, last_name, email from s8;
 query IITFFBTT
 select id, sequence_id, name, score, avg_score, is_lasted, entrance_date, birthday, passed from s9;
 ----
-32 64 str_value 32 64 t 1970-01-01 1970-01-01 00:00:00 1 mon 1 day 00:00:01
+32 64 str_value 32 64 t 1970-01-01 1970-01-01 00:00:00+00:00 1 mon 1 day 00:00:01
 
 query ITITT
 select id, code, timestamp, xfas, contacts, sex from s10;

proto/expr.proto

@@ -120,6 +120,7 @@ message ExprNode {
     ARRAY_PREPEND = 533;
     FORMAT_TYPE = 534;
     ARRAY_DISTINCT = 535;
+    ARRAY_LENGTH = 536;
 
     // Jsonb functions