ci: download dependencies from s3 (#9782)

risingwavelabs · Jun 13, 2023 · 3017aa2 · 3017aa2
1 parent f971965
commit 3017aa2
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 7 deletions.
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
@@ -10,6 +10,9 @@ export SCCACHE_REGION=us-east-2
 export SCCACHE_IDLE_TIMEOUT=0
 export CARGO_INCREMENTAL=0
 export CARGO_MAKE_PRINT_TIME_SUMMARY=true
+export MINIO_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/minio
+export MCLI_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/mc
+export GCLOUD_DOWNLOAD_TGZ=https://ci-deps-dist.s3.amazonaws.com/google-cloud-cli-406.0.0-linux-x86_64.tar.gz
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT

diff --git a/ci/scripts/e2e-iceberg-sink-test.sh b/ci/scripts/e2e-iceberg-sink-test.sh
@@ -41,7 +41,7 @@ sleep 1
 # prepare minio iceberg sink
 echo "--- preparing iceberg"
 .risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/iceberg
-wget https://iceberg-ci-spark-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
+wget https://ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
 tar -xf spark-3.3.1-bin-hadoop3.tgz --no-same-owner
 DEPENDENCIES=org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.0.0,org.apache.hadoop:hadoop-aws:3.3.2
 spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \

diff --git a/src/risedevtool/gcloud-pubsub.toml b/src/risedevtool/gcloud-pubsub.toml
@@ -2,9 +2,7 @@ extend = "common.toml"
 
 [env]
 GCLOUD_DOWNLOAD_PATH = "${PREFIX_TMP}/gcloud.tgz"
-GCLOUD_VERSION = "406.0.0"
-GCLOUD_RELEASE = "google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz"
-GCLOUD_DOWNLOAD_TGZ = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/${GCLOUD_RELEASE}"
+GCLOUD_DOWNLOAD_TGZ = { value = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-406.0.0-linux-x86_64.tar.gz", condition = { env_not_set = [ "GCLOUD_DOWNLOAD_TGZ" ] } }
 GCLOUD_SDK_DIR = "google-cloud-sdk"
 
 [tasks.download-pubsub]
@@ -19,7 +17,7 @@ set -e
 if [ -d "${PREFIX_BIN}/gcloud" ]; then
     exit 0
 fi
-echo "gcloud not found, download ${GCLOUD_RELEASE}"
+echo "gcloud not found, downloading"
 curl -fL -o "${GCLOUD_DOWNLOAD_PATH}" "${GCLOUD_DOWNLOAD_TGZ}"
 tar -xf "${GCLOUD_DOWNLOAD_PATH}" -C "${PREFIX_TMP}"
 echo "install the pubsub-emulator"

diff --git a/src/risedevtool/minio.toml b/src/risedevtool/minio.toml
@@ -2,8 +2,8 @@ extend = "common.toml"
 
 [env]
 MINIO_SYSTEM = "${SYSTEM}"
-MCLI_DOWNLOAD_BIN = "https://dl.min.io/client/mc/release/${MINIO_SYSTEM}/mc"
-MINIO_DOWNLOAD_BIN = "https://dl.min.io/server/minio/release/${MINIO_SYSTEM}/minio"
+MCLI_DOWNLOAD_BIN = { value = "https://dl.min.io/client/mc/release/${MINIO_SYSTEM}/mc", condition = { env_not_set = [ "MCLI_DOWNLOAD_BIN" ] } }
+MINIO_DOWNLOAD_BIN = { value = "https://dl.min.io/server/minio/release/${MINIO_SYSTEM}/minio", condition = { env_not_set = [ "MINIO_DOWNLOAD_BIN" ] } }
 
 [tasks.download-minio]
 private = true