Merge remote-tracking branch 'origin/main' into siyuan/aws-private-links

.github/labeler.yml

@@ -22,6 +22,8 @@ labels:
   title: "^test.*"
 - label: "component/doc"
   title: "^doc.*"
+- label: "type/deprecate"
+  title: "^deprecate.*"
 
 - label: "user-facing-changes"
   negate: true

.github/pr-title-checker-config.json

@@ -4,7 +4,7 @@
     "color": "B60205"
   },
   "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert)(\\(.*\\))?:.*",
+    "regexp": "^(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert|deprecate)(\\(.*\\))?:.*",
     "ignoreLabels" : ["ignore-title"]
   }
 }

.github/workflows/intergration_tests.yml

@@ -39,7 +39,7 @@ jobs:
           - schema-registry
           - mysql-cdc
           - postgres-cdc
-          #- mysql-sink
+          - mysql-sink
           - postgres-sink
           - iceberg-sink
         format: ["json", "protobuf"]

.gitignore

@@ -49,6 +49,8 @@ src/log/
 
 log/
 
+*.log
+
 .risingwave/
 .bin/
 

Cargo.toml

@@ -35,6 +35,7 @@ members = [
   "src/tests/regress",
   "src/tests/simulation",
   "src/tests/sqlsmith",
+  "src/tests/state_cleaning_test",
   "src/tracing",
   "src/udf",
   "src/utils/local_stats_alloc",

Makefile.toml

@@ -371,7 +371,6 @@ description = "Download all available components at once"
 dependencies = [
   "download-connector",
   "download-maven",
-  "build-connector-node",
   "download-etcd",
   "download-grafana",
   "download-jaeger",
@@ -415,6 +414,7 @@ dependencies = [
   "create-user-profiles-file",
   "download-all",
   "build-risingwave",
+  "build-connector-node",
   "post-build-risingwave",
   "extract-dashboard-artifact",
   "export-dashboard-v2",
@@ -441,6 +441,7 @@ alias = "playground"
 [tasks.playground]
 category = "RiseDev - Start"
 description = "Start a lite RisingWave playground using risingwave all-in-one binary"
+dependencies = ["download-connector"]
 script = '''
 #!/usr/bin/env bash
 

ci/scripts/deterministic-e2e-test.sh

@@ -45,4 +45,4 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (pre-generated-queries)"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+seq 64 | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'

ci/scripts/e2e-sink-test.sh

@@ -59,7 +59,7 @@ apt-get -y install postgresql-client
 export PGPASSWORD=postgres
 psql -h db -U postgres -c "CREATE ROLE test LOGIN SUPERUSER PASSWORD 'connector';"
 createdb -h db -U postgres test
-psql -h db -U postgres -d test -c "CREATE TABLE t4 (v1 int, v2 int);"
+psql -h db -U postgres -d test -c "CREATE TABLE t4 (v1 int PRIMARY KEY, v2 int);"
 psql -h db -U postgres -d test -c "CREATE TABLE t_remote (id serial PRIMARY KEY, name VARCHAR (50) NOT NULL);"
 
 node_port=50051
@@ -88,29 +88,29 @@ cargo make ci-start ci-1cn-1fe
 
 echo "--- testing sinks"
 sqllogictest -p 4566 -d dev './e2e_test/sink/append_only_sink.slt'
-sqllogictest -p 4566 -d dev './e2e_test/sink/create_sink_as.slt'
+# sqllogictest -p 4566 -d dev './e2e_test/sink/create_sink_as.slt'
 sqllogictest -p 4566 -d dev './e2e_test/sink/blackhole_sink.slt'
 sleep 1
 
 # check sink destination postgres
-sqllogictest -p 4566 -d dev './e2e_test/sink/remote/jdbc.load.slt'
-sleep 1
-sqllogictest -h db -p 5432 -d test './e2e_test/sink/remote/jdbc.check.pg.slt'
-sleep 1
+# sqllogictest -p 4566 -d dev './e2e_test/sink/remote/jdbc.load.slt'
+# sleep 1
+# sqllogictest -h db -p 5432 -d test './e2e_test/sink/remote/jdbc.check.pg.slt'
+# sleep 1
 
 # check sink destination mysql using shell
-if mysql  --host=mysql --port=3306 -u root -p123456 -sN -e "SELECT * FROM test.t_remote ORDER BY id;" | awk '{
-if ($1 == 1 && $2 == "Alex") c1++;
- if ($1 == 3 && $2 == "Carl") c2++;
-  if ($1 == 4 && $2 == "Doris") c3++;
-   if ($1 == 5 && $2 == "Eve") c4++;
-    if ($1 == 6 && $2 == "Frank") c5++; }
-     END { exit !(c1 == 1 && c2 == 1 && c3 == 1 && c4 == 1 && c5 == 1); }'; then
-  echo "mysql sink check passed"
-else
-  echo "The output is not as expected."
-  exit 1
-fi
+# if mysql  --host=mysql --port=3306 -u root -p123456 -sN -e "SELECT * FROM test.t_remote ORDER BY id;" | awk '{
+# if ($1 == 1 && $2 == "Alex") c1++;
+#  if ($1 == 3 && $2 == "Carl") c2++;
+#   if ($1 == 4 && $2 == "Doris") c3++;
+#    if ($1 == 5 && $2 == "Eve") c4++;
+#     if ($1 == 6 && $2 == "Frank") c5++; }
+#      END { exit !(c1 == 1 && c2 == 1 && c3 == 1 && c4 == 1 && c5 == 1); }'; then
+#   echo "mysql sink check passed"
+# else
+#   echo "The output is not as expected."
+#   exit 1
+# fi
 
 echo "--- Kill cluster"
 pkill -f connector-node

ci/scripts/e2e-test-parallel-for-opendal.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+
+while getopts 'p:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+echo "--- Download artifacts"
+mkdir -p target/debug
+buildkite-agent artifact download risingwave-"$profile" target/debug/
+buildkite-agent artifact download risedev-dev-"$profile" target/debug/
+buildkite-agent artifact download "e2e_test/generated/*" ./
+mv target/debug/risingwave-"$profile" target/debug/risingwave
+mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
+
+echo "--- Adjust permission"
+chmod +x ./target/debug/risingwave
+chmod +x ./target/debug/risedev-dev
+
+echo "--- Generate RiseDev CI config"
+cp ci/risedev-components.ci.env risedev-components.user.env
+
+echo "--- Prepare RiseDev dev cluster"
+cargo make pre-start-dev
+cargo make link-all-in-one-binaries
+
+host_args="-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567"
+
+echo "--- e2e, ci-3cn-3fe-opendal-fs-backend, streaming"
+RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+cargo make ci-start ci-3cn-3fe-opendal-fs-backend
+sqllogictest ${host_args} -d dev  './e2e_test/streaming/**/*.slt' -j 16 --junit "parallel-opendal-fs-backend-${profile}"
+
+echo "--- Kill cluster"
+rm -rf /tmp/rw_ci
+cargo make ci-kill
+
+
+echo "--- e2e, ci-3cn-3fe-opendal-fs-backend, batch"
+RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+cargo make ci-start ci-3cn-3fe-opendal-fs-backend
+sqllogictest ${host_args} -d dev  './e2e_test/ddl/**/*.slt' --junit "parallel-opendal-fs-backend-ddl-${profile}"
+sqllogictest ${host_args} -d dev  './e2e_test/batch/**/*.slt' -j 16 --junit "parallel-opendal-fs-backend-batch-${profile}"
+
+echo "--- Kill cluster"
+rm -rf /tmp/rw_ci
+cargo make ci-kill

ci/scripts/e2e-test-parallel-in-memory.sh

@@ -54,4 +54,4 @@ sqllogictest ${host_args} -d dev  './e2e_test/ddl/**/*.slt' --junit "parallel-in
 sqllogictest ${host_args} -d dev  './e2e_test/batch/**/*.slt' -j 16 --junit "parallel-in-memory-batch-${profile}"
 
 echo "--- Kill cluster"
-cargo make ci-kill
+cargo make ci-kill

ci/scripts/java-binding-test.sh

@@ -53,3 +53,6 @@ cargo make ingest-data-and-run-java-binding
 
 echo "--- Kill cluster"
 cargo make ci-kill
+
+echo "--- run stream chunk java binding"
+cargo make run-java-binding-stream-chunk-demo

ci/workflows/main.yml

@@ -228,15 +228,15 @@ steps:
     retry: *auto-retry
 
   - label: "scaling test (deterministic simulation)"
-    command: "TEST_NUM=30 timeout 25m ci/scripts/deterministic-scale-test.sh"
+    command: "TEST_NUM=30 timeout 40m ci/scripts/deterministic-scale-test.sh"
     depends_on: "build-simulation"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v4.9.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
-    timeout_in_minutes: 25
+    timeout_in_minutes: 40 # TODO: split into multiple jobs
     retry: *auto-retry
 
   - label: "end-to-end test (deterministic simulation)"

ci/workflows/pull-request.yml

@@ -113,6 +113,21 @@ steps:
     timeout_in_minutes: 12
     retry: *auto-retry
 
+  - label: "end-to-end test for opendal (parallel)"
+    command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-dev"
+    depends_on:
+      - "build"
+      - "docslt"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 12
+    retry: *auto-retry
+
   - label: "end-to-end test (parallel, in-memory)"
     command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-dev"
     depends_on: "build"
@@ -228,6 +243,7 @@ steps:
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 15
     retry: *auto-retry
+    soft_fail: true
 
   - label: "check"
     command: "ci/scripts/check.sh"