From 868612dfefe7162de21146bf670b28d4cd37768a Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 16 Mar 2020 13:35:14 +0800 Subject: [PATCH] Support sync bucket in lightning (#1629) * Support sync bucket in lightning Signed-off-by: Aylei * fix nodeSelector is not respected in tidb-lightning chart Signed-off-by: Aylei * Fix nodeSelector indention Signed-off-by: Aylei --- charts/tidb-lightning/templates/job.yaml | 4 ++++ .../templates/scripts/_start_data_retriever.sh.tpl | 5 +++++ .../templates/scripts/_start_lightning.sh.tpl | 10 ++++++++++ charts/tidb-lightning/values.yaml | 2 ++ 4 files changed, 21 insertions(+) diff --git a/charts/tidb-lightning/templates/job.yaml b/charts/tidb-lightning/templates/job.yaml index 4a4b46433d..f9e636064f 100644 --- a/charts/tidb-lightning/templates/job.yaml +++ b/charts/tidb-lightning/templates/job.yaml @@ -134,6 +134,10 @@ spec: {{- if .Values.affinity }} affinity: {{ toYaml .Values.affinity | indent 6 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} {{- end }} {{- if .Values.tolerations }} tolerations: diff --git a/charts/tidb-lightning/templates/scripts/_start_data_retriever.sh.tpl b/charts/tidb-lightning/templates/scripts/_start_data_retriever.sh.tpl index 96e16a1bc9..4529f7ce09 100644 --- a/charts/tidb-lightning/templates/scripts/_start_data_retriever.sh.tpl +++ b/charts/tidb-lightning/templates/scripts/_start_data_retriever.sh.tpl @@ -1,4 +1,8 @@ set -euo pipefail +{{ if .Values.dataSource.remote.directory }} +# rclone sync skip identical files automatically +rclone --config /etc/rclone/rclone.conf sync -P {{ .Values.dataSource.remote.directory}} /data +{{- else -}} filename=$(basename {{ .Values.dataSource.remote.path }}) if find /data -name metadata | egrep '.*'; then echo "data already exist" @@ -7,3 +11,4 @@ else rclone --config /etc/rclone/rclone.conf copy -P {{ .Values.dataSource.remote.path }} /data cd /data && tar xzvf ${filename} fi +{{- end -}} diff --git a/charts/tidb-lightning/templates/scripts/_start_lightning.sh.tpl b/charts/tidb-lightning/templates/scripts/_start_lightning.sh.tpl index ba8d2708cc..32ec7be54b 100644 --- a/charts/tidb-lightning/templates/scripts/_start_lightning.sh.tpl +++ b/charts/tidb-lightning/templates/scripts/_start_lightning.sh.tpl @@ -2,6 +2,16 @@ data_dir={{ .Values.dataSource.local.hostPath }} {{- else if .Values.dataSource.adhoc.pvcName -}} data_dir=/var/lib/tidb-lightning/{{ .Values.dataSource.adhoc.backupName | default .Values.dataSource.adhoc.pvcName }} +{{- else if .Values.dataSource.remote.directory -}} +data_dir=/var/lib/tidb-lightning +if [ -z "$(ls -A ${data_dir})" ]; then + if [ ! -z ${FAIL_FAST} ]; then + exit 1 + else + echo "No files in data dir, please exec into my container to diagnose" + tail -f /dev/null + fi +fi {{- else -}} data_dir=$(dirname $(find /var/lib/tidb-lightning -name metadata 2>/dev/null) 2>/dev/null) if [ -z $data_dir ]; then diff --git a/charts/tidb-lightning/values.yaml b/charts/tidb-lightning/values.yaml index ace1abfd90..e5a151e205 100644 --- a/charts/tidb-lightning/values.yaml +++ b/charts/tidb-lightning/values.yaml @@ -30,6 +30,8 @@ dataSource: storage: 100Gi secretName: cloud-storage-secret path: s3:bench-data-us/sysbench/sbtest_16_1e7.tar.gz + # Directory support downloading all files in a remote directory, shadow dataSoure.remote.path if present + # directory: s3:bench-data-us # If rcloneConfig is configured, then `secretName` will be ignored, # `rcloneConfig` should only be used for the cases where no sensitive # information need to be configured, e.g. the configuration as below,