From 3017aa26300b1511ffa2dc28c99c50fcf9142b40 Mon Sep 17 00:00:00 2001
From: xxchan <xxchan22f@gmail.com>
Date: Tue, 13 Jun 2023 20:58:37 +0200
Subject: [PATCH] ci: download dependencies from s3 (#9782)

---
 ci/scripts/common.sh                | 3 +++
 ci/scripts/e2e-iceberg-sink-test.sh | 2 +-
 src/risedevtool/gcloud-pubsub.toml  | 6 ++----
 src/risedevtool/minio.toml          | 4 ++--
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
index 7f9d926301948..facef8aa3532d 100644
--- a/ci/scripts/common.sh
+++ b/ci/scripts/common.sh
@@ -10,6 +10,9 @@ export SCCACHE_REGION=us-east-2
 export SCCACHE_IDLE_TIMEOUT=0
 export CARGO_INCREMENTAL=0
 export CARGO_MAKE_PRINT_TIME_SUMMARY=true
+export MINIO_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/minio
+export MCLI_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/mc
+export GCLOUD_DOWNLOAD_TGZ=https://ci-deps-dist.s3.amazonaws.com/google-cloud-cli-406.0.0-linux-x86_64.tar.gz
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT
diff --git a/ci/scripts/e2e-iceberg-sink-test.sh b/ci/scripts/e2e-iceberg-sink-test.sh
index 1b366cc9e0a56..64aa2859c0688 100755
--- a/ci/scripts/e2e-iceberg-sink-test.sh
+++ b/ci/scripts/e2e-iceberg-sink-test.sh
@@ -41,7 +41,7 @@ sleep 1
 # prepare minio iceberg sink
 echo "--- preparing iceberg"
 .risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/iceberg
-wget https://iceberg-ci-spark-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
+wget https://ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
 tar -xf spark-3.3.1-bin-hadoop3.tgz --no-same-owner
 DEPENDENCIES=org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.0.0,org.apache.hadoop:hadoop-aws:3.3.2
 spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \
diff --git a/src/risedevtool/gcloud-pubsub.toml b/src/risedevtool/gcloud-pubsub.toml
index 771c8648b18e5..313561b287cba 100644
--- a/src/risedevtool/gcloud-pubsub.toml
+++ b/src/risedevtool/gcloud-pubsub.toml
@@ -2,9 +2,7 @@ extend = "common.toml"
 
 [env]
 GCLOUD_DOWNLOAD_PATH = "${PREFIX_TMP}/gcloud.tgz"
-GCLOUD_VERSION = "406.0.0"
-GCLOUD_RELEASE = "google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz"
-GCLOUD_DOWNLOAD_TGZ = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/${GCLOUD_RELEASE}"
+GCLOUD_DOWNLOAD_TGZ = { value = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-406.0.0-linux-x86_64.tar.gz", condition = { env_not_set = [ "GCLOUD_DOWNLOAD_TGZ" ] } }
 GCLOUD_SDK_DIR = "google-cloud-sdk"
 
 [tasks.download-pubsub]
@@ -19,7 +17,7 @@ set -e
 if [ -d "${PREFIX_BIN}/gcloud" ]; then
     exit 0
 fi
-echo "gcloud not found, download ${GCLOUD_RELEASE}"
+echo "gcloud not found, downloading"
 curl -fL -o "${GCLOUD_DOWNLOAD_PATH}" "${GCLOUD_DOWNLOAD_TGZ}"
 tar -xf "${GCLOUD_DOWNLOAD_PATH}" -C "${PREFIX_TMP}"
 echo "install the pubsub-emulator"
diff --git a/src/risedevtool/minio.toml b/src/risedevtool/minio.toml
index 34c5ae1f3569a..fea898ad6d233 100644
--- a/src/risedevtool/minio.toml
+++ b/src/risedevtool/minio.toml
@@ -2,8 +2,8 @@ extend = "common.toml"
 
 [env]
 MINIO_SYSTEM = "${SYSTEM}"
-MCLI_DOWNLOAD_BIN = "https://dl.min.io/client/mc/release/${MINIO_SYSTEM}/mc"
-MINIO_DOWNLOAD_BIN = "https://dl.min.io/server/minio/release/${MINIO_SYSTEM}/minio"
+MCLI_DOWNLOAD_BIN = { value = "https://dl.min.io/client/mc/release/${MINIO_SYSTEM}/mc", condition = { env_not_set = [ "MCLI_DOWNLOAD_BIN" ] } }
+MINIO_DOWNLOAD_BIN = { value = "https://dl.min.io/server/minio/release/${MINIO_SYSTEM}/minio", condition = { env_not_set = [ "MINIO_DOWNLOAD_BIN" ] } }
 
 [tasks.download-minio]
 private = true