From 13641c88e1c3279addad5152941e865af2fd7c1d Mon Sep 17 00:00:00 2001
From: congyi wang <58715567+wcy-fdu@users.noreply.github.com>
Date: Mon, 20 Mar 2023 15:16:30 +0800
Subject: [PATCH] feat(test): test opendal fs engine on main cron with large
 dataset (#8637)

---
 .../s3-source-test-for-opendal-fs-engine.sh   | 56 +++++++++++++++++++
 ci/workflows/main-cron.yml                    | 17 ++++++
 risedev.yml                                   |  4 +-
 3 files changed, 75 insertions(+), 2 deletions(-)
 create mode 100755 ci/scripts/s3-source-test-for-opendal-fs-engine.sh

diff --git a/ci/scripts/s3-source-test-for-opendal-fs-engine.sh b/ci/scripts/s3-source-test-for-opendal-fs-engine.sh
new file mode 100755
index 000000000000..20fd77483a7a
--- /dev/null
+++ b/ci/scripts/s3-source-test-for-opendal-fs-engine.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+
+while getopts 'p:s:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        s )
+            script=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+
+
+echo "--- Download artifacts"
+mkdir -p target/debug
+buildkite-agent artifact download risingwave-"$profile" target/debug/
+buildkite-agent artifact download risedev-dev-"$profile" target/debug/
+
+mv target/debug/risingwave-"$profile" target/debug/risingwave
+mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
+
+echo "--- Adjust permission"
+chmod +x ./target/debug/risingwave
+chmod +x ./target/debug/risedev-dev
+
+echo "--- Generate RiseDev CI config"
+cp ci/risedev-components.ci.env risedev-components.user.env
+
+echo "--- Prepare RiseDev dev cluster"
+cargo make pre-start-dev
+cargo make link-all-in-one-binaries
+
+echo "--- starting risingwave cluster with connector node"
+cargo make ci-start ci-3cn-3fe-opendal-fs-backend
+
+echo "--- Run test"
+python3 -m pip install minio psycopg2-binary
+python3 e2e_test/s3/$script.py
+
+echo "--- Kill cluster"
+rm -rf /tmp/rw_ci
+cargo make ci-kill
diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
index 5a139ee74d81..88854253efed 100644
--- a/ci/workflows/main-cron.yml
+++ b/ci/workflows/main-cron.yml
@@ -215,6 +215,7 @@ steps:
             - S3_SOURCE_TEST_CONF
     timeout_in_minutes: 20
     retry: *auto-retry  
+
   - label: "S3 source check on AWS (csv parser)"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s run_csv"
     depends_on: build
@@ -229,4 +230,20 @@ steps:
           environment:
             - S3_SOURCE_TEST_CONF
     timeout_in_minutes: 20
+    retry: *auto-retry  
+
+  - label: "S3 source on OpenDAL fs engine"
+    command: "ci/scripts/s3-source-test-for-opendal-fs-engine.sh -p ci-release -s run"
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+    timeout_in_minutes: 20
     retry: *auto-retry  
\ No newline at end of file
diff --git a/risedev.yml b/risedev.yml
index bc822805d011..1215d32ad8e6 100644
--- a/risedev.yml
+++ b/risedev.yml
@@ -502,8 +502,8 @@ profile:
   ci-3cn-3fe-opendal-fs-backend:
     config-path: src/config/ci.toml
     steps:
-      # - use: etcd
-      #   unsafe-no-fsync: true
+      - use: etcd
+        unsafe-no-fsync: true
       - use: meta-node
       - use: opendal
         engine: fs