Merge branch 'main' into xinjing/bind_ordered_set_agg

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -6,15 +6,56 @@ body:
     attributes:
       label: Describe the bug
       description: A clear and concise description of what the bug is.
+  - type: textarea
+    attributes:
+      label: Error message/log
+      description: The error message you see.
+      render: text
   - type: textarea
     attributes:
       label: To Reproduce
-      description: Steps to reproduce the behavior.
+      description: Steps to reproduce the behavior, including the SQLs you run and/or the operations you have done to trigger the bug.
+      placeholder: |
+        First create the tables/sources and materialized views with
+        
+        ```sql
+        CREATE TABLE ...
+        CREATE MATERIALIZED VIEW ...
+        ```
+        
+        Then the bug is triggered after ...
   - type: textarea
     attributes:
       label: Expected behavior
       description: A clear and concise description of what you expected to happen.
+      placeholder: |
+        I expected to see this happen: *explanation*
+        
+        Instead, this happened: *explanation*
+  - type: textarea
+    attributes:
+      label: How did you deploy RisingWave?
+      description: Do you run RisingWave via Docker / Homebrew / RiseDev / RisingWave Cloud / ...?
+      placeholder: |
+        via Docker Compose. My `docker-compose.yml` is: ...
+  - type: textarea
+    attributes:
+      label: The version of RisingWave
+      description: The output of `select version()` and/or the docker image tag and ID.
+      placeholder: |
+        ```
+        dev=> select version();
+                                   version
+        -----------------------------------------------------------------------------
+        PostgreSQL 8.3-RisingWave-0.19.0 (01659936e12307e28e13287dcc3ca899b7f701e3)
+
+
+        docker image ls
+        REPOSITORY                                 TAG         IMAGE ID       CREATED         SIZE
+        risingwavelabs/risingwave                  latest      c0fb5556d7cb   6 days ago      1.99GB
+        ```
   - type: textarea
     attributes:
       label: Additional context
-      description: Add any other context about the problem here.
+      description: Add any other context about the problem here. e.g., the full log files.
+

Makefile.toml

@@ -98,19 +98,16 @@ alias = "logs"
 
 [tasks.logs]
 category = "Misc"
-description = "Open logs with VSCode and dump in console"
+description = "Open logs with VSCode or dump in console"
 script = '''
 #!/usr/bin/env bash
 set -e
 
-code "${PREFIX_LOG}" || true
+code "${PREFIX_LOG}" 2> /dev/null || true
 
 for out_file in ${PREFIX_LOG}/*.log
 do
-  echo ""
-  echo "==="
-  echo "=== Dump log file $out_file ==="
-  echo "==="
+  echo "~~~ Dump log file $out_file"
   echo ""
   cat "$out_file" | tail -n 300
 done
@@ -197,23 +194,26 @@ tail -f -n 5 ${PREFIX_LOG}/$1
 '''
 
 [tasks.check-logs]
-category = "Misc"
+category = "RiseDev - CI"
 description = "Check if there is panic in log or significant log size issue"
 script = '''
 #!/usr/bin/env bash
 set -e
 
+echo "~~~ Check logs"
+du -ah ${PREFIX_LOG}
+
 for out_file in ${PREFIX_LOG}/*.log
 do
   if grep "panicked at" "$out_file" -C 100; then
-    echo "$(tput setaf 3)\"panicked at\" found in $out_file$(tput sgr0), please check if there's any bugs in this PR."
+    echo "$(tput setaf 1)\"panicked at\" found in $out_file$(tput sgr0), please check if there's any bugs in this PR."
     echo "You may find \"risedev-logs\" artifacts and download logs after all workflows finish."
     exit 1
   fi
 done
 
 if (( "$(du -sk ${PREFIX_LOG} | cut -f1)" > 2000 )) ; then
-    echo "$(tput setaf 3)log size is significantly large ($(du -sh ${PREFIX_LOG} | cut -f1)).$(tput sgr0) Please disable unnecessary logs."
+    echo "$(tput setaf 1)ERROR: log size is significantly large ($(du -sh ${PREFIX_LOG} | cut -f1)).$(tput sgr0) Please disable unnecessary logs."
     exit 1
 fi
 '''

ci/scripts/remove-micro-bench-results.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Requirements:
+# - s3 certs,
+# - qa binary
+
+# Example
+# remove-micro-bench-results.sh
+
+./qa ctl -I 52.207.243.214:8081 execution delete -i "$1"

ci/scripts/upload-micro-bench-results.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+# EXAMPLE USAGE:
+# BUILDKITE_BUILD_NUMBER=511 ./ci/scripts/upload-micro-bench-results.sh
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+setup() {
+  echo "--- Installing utils"
+  echo ">>> Installing jq"
+  wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+  chmod +x jq-linux64
+  mv jq-linux64 /usr/local/bin/jq
+}
+
+get_branch() {
+   curl -H "Authorization: Bearer $BUILDKITE_TOKEN" \
+   "https://api.buildkite.com/v2/organizations/risingwavelabs/pipelines/main-cron/builds/$BUILDKITE_BUILD_NUMBER" \
+  | jq '.branch' \
+  | sed 's/\"//g'
+}
+
+get_date() {
+  curl -H "Authorization: Bearer $BUILDKITE_TOKEN" \
+   "https://api.buildkite.com/v2/organizations/risingwavelabs/pipelines/main-cron/builds/$BUILDKITE_BUILD_NUMBER" \
+  | jq '.jobs | .[] | select ( .name | contains("micro benchmark")) | .finished_at' \
+  | sed 's/\.[0-9]*Z/Z/' \
+  | sed 's/\"//g'
+}
+
+get_commit() {
+  curl -H "Authorization: Bearer $BUILDKITE_TOKEN" \
+   "https://api.buildkite.com/v2/organizations/risingwavelabs/pipelines/main-cron/builds/$BUILDKITE_BUILD_NUMBER" \
+  | jq '.commit' \
+  | sed 's/\"//g'
+}
+
+setup
+
+BUILDKITE_BUILD_URL="https://buildkite.com/risingwavelabs/main-cron/builds/$BUILDKITE_BUILD_NUMBER"
+END_DATE=$(get_date)
+COMMIT=$(get_commit)
+BRANCH=$(get_branch)
+
+curl -L https://rw-qa-infra-public.s3.us-west-2.amazonaws.com/scripts/download-qa.sh | bash
+
+git clone --depth 1 https://"$GITHUB_TOKEN"@github.com/risingwavelabs/qa-infra.git
+cp -r qa-infra/certs ./certs
+rm -rf qa-infra
+
+echo "--- Uploading results for $BUILDKITE_BUILD_URL"
+echo "Commit: $COMMIT"
+echo "Branch: $BRANCH"
+echo "Date: $END_DATE"
+
+./qa ctl -I 52.207.243.214:8081 execution create-micro-benchmark-executions \
+  --exec-url "${BUILDKITE_BUILD_URL}" \
+  --branch "$BRANCH" \
+  --tag latest \
+  --commit "$COMMIT" \
+  --end-time "$END_DATE" \
+  --status SUCCESS

ci/workflows/main-cron.yml

@@ -33,7 +33,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
     timeout_in_minutes: 15
-  
+
   - label: "docslt"
     command: "ci/scripts/docslt.sh"
     key: "docslt"
@@ -260,13 +260,32 @@ steps:
 
   - label: "micro benchmark"
     command: "ci/scripts/run-micro-benchmarks.sh"
+    key: "run-micro-benchmarks"
     plugins:
       - docker-compose#v4.9.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
     timeout_in_minutes: 35
 
+  - label: "upload micro-benchmark"
+    command:
+      - "BUILDKITE_BUILD_NUMBER=$BUILDKITE_BUILD_NUMBER ci/scripts/upload-micro-bench-results.sh"
+    depends_on: "run-micro-benchmarks"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            BUILDKITE_TOKEN: buildkite_token
+            GITHUB_TOKEN: github-token
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - BUILDKITE_TOKEN
+            - GITHUB_TOKEN
+    timeout_in_minutes: 5
+
   # Backwards compatibility tests
   - label: "Backwards compatibility tests"
     command: "RW_COMMIT=$BUILDKITE_COMMIT ci/scripts/backwards-compat-test.sh -p ci-release"