Merge remote-tracking branch 'origin/main' into bz/allow-in-memory-st…

…ate-store

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

.github/workflows/stale.yml

@@ -13,9 +13,15 @@ jobs:
       pull-requests: write
 
     steps:
-    - uses: actions/stale@v4
+    - uses: actions/stale@v5
       with:
-        repo-token: ${{ secrets.WUTAO_PAT }}
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: >
+          This issue has been open for 60 days with no activity.
+          Could you please update the status? Feel free to continue discussion or close as not planned.
+        stale-pr-message: >
+          This PR has been open for 60 days with no activity.
+          Could you please update the status? Feel free to ping a reviewer if you are waiting for review.
         stale-issue-label: 'no-issue-activity'
         stale-pr-label: 'no-pr-activity'
         days-before-close: -1

.gitignore

@@ -55,3 +55,5 @@ ft.txt
 # generated output from regress tests
 src/tests/regress/output/*
 src/tests/sync_point/slt/e2e_test
+# generated e2e tests
+e2e_test/generated

Cargo.toml

@@ -25,6 +25,7 @@ members = [
   "src/storage/compactor",
   "src/storage/hummock_sdk",
   "src/storage/hummock_test",
+  "src/storage/hummock_test/sync_point_unit_test",
   "src/stream",
   "src/test_runner",
   "src/tests/regress",

Makefile.toml

@@ -649,10 +649,25 @@ extend = "slt"
 args = ["${@}", "./e2e_test/batch/**/*.slt"]
 description = "Run all batch e2e tests"
 
+[tasks.slt-generated]
+category = "RiseDev - SQLLogicTest"
+extend = "slt"
+args = ["${@}", "./e2e_test/generated/**/*.slt"]
+description = "Run all generated e2e tests"
+
 [tasks.slt-all]
 category = "RiseDev - SQLLogicTest"
-run_task = { name = ["slt-streaming", "slt-batch"] }
-description = "Run all batch and streaming e2e tests"
+run_task = { name = ["slt-streaming", "slt-batch", "slt-generated"] }
+description = "Run all e2e tests"
+
+[tasks.docslt]
+category = "RiseDev - SQLLogicTest"
+description = "Extract SQL examples written in SQLLogicTest syntax from Rust doc comments"
+script = '''
+#!/bin/bash
+set -e
+cargo run --bin risedev-docslt -- "$@"
+'''
 
 [tasks.compose]
 category = "RiseDev - Compose"

ci/scripts/docslt.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+
+echo "--- Extract DocSlt end-to-end tests"
+cargo run --bin risedev-docslt
+
+echo "--- Upload generated end-to-end tests"
+buildkite-agent artifact upload "e2e_test/generated/**/*"

ci/scripts/e2e-test-parallel.sh

@@ -25,6 +25,7 @@ echo "--- Download artifacts"
 mkdir -p target/debug
 buildkite-agent artifact download risingwave-"$profile" target/debug/
 buildkite-agent artifact download risedev-dev-"$profile" target/debug/
+buildkite-agent artifact download "e2e_test/generated/*" ./
 mv target/debug/risingwave-"$profile" target/debug/risingwave
 mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
 
@@ -43,15 +44,22 @@ host_args="-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567"
 
 echo "--- e2e, ci-3cn-3fe, streaming"
 cargo make ci-start ci-3cn-3fe
-sqllogictest ${host_args} -d dev  './e2e_test/streaming/**/*.slt' -j 16 --junit "parallel-streaming-${profile}"
+sqllogictest ${host_args} -d dev './e2e_test/streaming/**/*.slt' -j 16 --junit "parallel-streaming-${profile}"
 
 echo "--- Kill cluster"
 cargo make ci-kill
 
 echo "--- e2e, ci-3cn-3fe, batch"
 cargo make ci-start ci-3cn-3fe
-sqllogictest ${host_args} -d dev  './e2e_test/ddl/**/*.slt' --junit "parallel-batch-ddl-${profile}"
-sqllogictest ${host_args} -d dev  './e2e_test/batch/**/*.slt' -j 16 --junit "parallel-batch-${profile}"
+sqllogictest ${host_args} -d dev './e2e_test/ddl/**/*.slt' --junit "parallel-batch-ddl-${profile}"
+sqllogictest ${host_args} -d dev './e2e_test/batch/**/*.slt' -j 16 --junit "parallel-batch-${profile}"
+
+echo "--- Kill cluster"
+cargo make ci-kill
+
+echo "--- e2e, ci-3cn-3fe, generated"
+cargo make ci-start ci-3cn-3fe
+sqllogictest ${host_args} -d dev './e2e_test/generated/**/*.slt' -j 16 --junit "parallel-generated-${profile}"
 
 echo "--- Kill cluster"
 cargo make ci-kill

ci/scripts/run-e2e-test.sh

@@ -23,6 +23,7 @@ echo "--- Download artifacts"
 mkdir -p target/debug
 buildkite-agent artifact download risingwave-"$profile" target/debug/
 buildkite-agent artifact download risedev-dev-"$profile" target/debug/
+buildkite-agent artifact download "e2e_test/generated/*" ./
 mv target/debug/risingwave-"$profile" target/debug/risingwave
 mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
 
@@ -55,6 +56,13 @@ sqllogictest -p 4566 -d test './e2e_test/database/test.slt'
 echo "--- Kill cluster"
 cargo make ci-kill
 
+echo "--- e2e, ci-3cn-1fe, generated"
+cargo make ci-start ci-3cn-1fe
+sqllogictest -p 4566 -d dev './e2e_test/generated/**/*.slt' --junit "generated-${profile}"
+
+echo "--- Kill cluster"
+cargo make ci-kill
+
 echo "--- e2e, ci-3cn-1fe, extended query"
 cargo make ci-start ci-3cn-1fe
 sqllogictest -p 4566 -d dev -e postgres-extended './e2e_test/extended_query/**/*.slt'

ci/workflows/main-cron.yml

@@ -43,10 +43,24 @@ steps:
           mount-buildkite-agent: true
     timeout_in_minutes: 10
     retry: *auto-retry
+
+  - label: "docslt"
+    command: "ci/scripts/docslt.sh"
+    key: "docslt"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+    timeout_in_minutes: 10
+    retry: *auto-retry
 
   - label: "end-to-end test"
     command: "ci/scripts/cron-e2e-test.sh -p ci-dev"
-    depends_on: "build"
+    depends_on:
+      - "build"
+      - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:

ci/workflows/main.yml

@@ -43,10 +43,24 @@ steps:
           mount-buildkite-agent: true
     timeout_in_minutes: 10
     retry: *auto-retry
+
+  - label: "docslt"
+    command: "ci/scripts/docslt.sh"
+    key: "docslt"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+    timeout_in_minutes: 10
+    retry: *auto-retry
 
   - label: "end-to-end test (release mode)"
     command: "ci/scripts/e2e-test.sh -p ci-release"
-    depends_on: "build"
+    depends_on:
+      - "build"
+      - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - seek-oss/aws-sm#v2.3.1:
@@ -65,7 +79,9 @@ steps:
 
   - label: "end-to-end test (parallel) (release mode)"
     command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
-    depends_on: "build"
+    depends_on:
+      - "build"
+      - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - seek-oss/aws-sm#v2.3.1:

ci/workflows/pull-request.yml

@@ -55,9 +55,23 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "docslt"
+    command: "ci/scripts/docslt.sh"
+    key: "docslt"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "end-to-end test"
     command: "ci/scripts/e2e-test.sh -p ci-dev"
-    depends_on: "build"
+    depends_on:
+      - "build"
+      - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:
@@ -70,7 +84,9 @@ steps:
 
   - label: "end-to-end test (parallel)"
     command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
-    depends_on: "build"
+    depends_on:
+      - "build"
+      - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:

docs/developer-guide.md

@@ -33,6 +33,8 @@ To report bugs, create a [GitHub issue](https://github.com/risingwavelabs/rising
     - [Planner tests](#planner-tests)
     - [End-to-end tests](#end-to-end-tests)
     - [End-to-end tests on CI](#end-to-end-tests-on-ci)
+    - [DocSlt tests](#docslt-tests)
+    - [Deterministic simulation tests](#deterministic-simulation-tests)
   - [Miscellaneous checks](#miscellaneous-checks)
   - [Update Grafana dashboard](#update-grafana-dashboard)
   - [Add new files](#add-new-files)
@@ -345,6 +347,71 @@ Basically, CI is using the following two configurations to run the full e2e test
 
 You can adjust the environment variable to enable some specific code to make all e2e tests pass. Refer to GitHub Action workflow for more information.
 
+### DocSlt tests
+
+As introduced in [#5117](https://github.com/risingwavelabs/risingwave/issues/5117), DocSlt tool allows you to write SQL examples in sqllogictest syntax in Rust doc comments. After adding or modifying any such SQL examples, you should run the following commands to generate and run e2e tests for them.
+
+```shell
+# generate e2e tests from doc comments for all default packages
+./risedev docslt
+# or, generate for only modified package
+./risedev docslt -p risingwave_expr
+
+# run all generated e2e tests
+./risedev slt-generated -p 4566 -d dev
+# or, run only some of them
+./risedev slt -p 4566 -d dev './e2e_test/generated/docslt/risingwave_expr/**/*.slt'
+```
+
+These will be run on CI as well.
+
+### Deterministic simulation tests
+
+Deterministic simulation is a powerful tool to efficiently search bugs and reliably reproduce them.
+In case you are not familiar with this technique, here is a [talk](https://www.youtube.com/watch?v=4fFDFbi3toc) and a [blog post](https://sled.rs/simulation.html) for brief introduction.
+
+In RisingWave, deterministic simulation is supported in both unit test and end-to-end test. You can run them using the following commands:
+
+```sh
+# run deterministic unit test
+./risedev stest
+# run deterministic end-to-end test
+./risedev sslt './e2e_test/path/to/directory/**/*.slt'
+```
+
+When your program panics, the simulator will print the random seed of this run:
+
+```sh
+thread '<unnamed>' panicked at '...',
+note: run with `MADSIM_TEST_SEED=1` environment variable to reproduce this error
+```
+
+Then you can reproduce the bug with the given seed:
+
+```sh
+# set the random seed to reproduce a run
+MADSIM_TEST_SEED=1 RUST_LOG=info ./risedev sslt -- './e2e_test/path/to/directory/**/*.slt'
+```
+
+More advanced usages are listed below:
+
+```sh
+# run multiple times with different seeds to test reliability
+# it's recommended to build in release mode for a fast run
+MADSIM_TEST_NUM=100 ./risedev sslt --release -- './e2e_test/path/to/directory/**/*.slt'
+
+# configure cluster nodes (by default: 2fe+3cn)
+./risedev sslt -- --compute-nodes 2 './e2e_test/path/to/directory/**/*.slt'
+
+# inject failures to test fault recovery
+./risedev sslt -- --kill-meta --etcd-timeout-rate=0.01 './e2e_test/path/to/directory/**/*.slt'
+
+# see more usages
+./risedev sslt -- --help  
+```
+
+Deterministic test is included in CI as well. See [CI script](../ci/scripts/deterministic-e2e-test.sh) for details.
+
 ## Miscellaneous checks
 
 For shell code, please run:

e2e_test/source/basic_test.slt

@@ -90,6 +90,19 @@ create materialized source s7 (v1 int, v2 varchar) with (
 statement ok
 select * from s7
 
+# we cannot create debezium source without pk
+statement error
+create materialized source s8 (
+  id integer,
+  first_name varchar,
+  last_name varchar,
+  email varchar
+) with (
+  connector = 'kafka',
+  topic = 'debezium_log',
+  properties.bootstrap.server = '127.0.0.1:29092'
+) row format debezium_json
+
 statement ok
 create materialized source s8 (
   id integer,