Support sync bucket in lightning (#1629)

* Support sync bucket in lightning Signed-off-by: Aylei <rayingecho@gmail.com> * fix nodeSelector is not respected in tidb-lightning chart Signed-off-by: Aylei <rayingecho@gmail.com> * Fix nodeSelector indention Signed-off-by: Aylei <rayingecho@gmail.com>
pingcap · Mar 16, 2020 · 868612d · 868612d
1 parent 784a9bc
commit 868612d
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 0 deletions.
diff --git a/charts/tidb-lightning/templates/job.yaml b/charts/tidb-lightning/templates/job.yaml
@@ -134,6 +134,10 @@ spec:
     {{- if .Values.affinity }}
       affinity:
 {{ toYaml .Values.affinity | indent 6 }}
+    {{- end }}
+    {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
     {{- end }}
     {{- if .Values.tolerations }}
       tolerations:

diff --git a/charts/tidb-lightning/templates/scripts/_start_data_retriever.sh.tpl b/charts/tidb-lightning/templates/scripts/_start_data_retriever.sh.tpl
@@ -1,4 +1,8 @@
 set -euo pipefail
+{{ if .Values.dataSource.remote.directory }}
+# rclone sync skip identical files automatically
+rclone --config /etc/rclone/rclone.conf sync -P {{ .Values.dataSource.remote.directory}} /data
+{{- else -}}
 filename=$(basename {{ .Values.dataSource.remote.path }})
 if find /data -name metadata | egrep '.*'; then
     echo "data already exist"
@@ -7,3 +11,4 @@ else
     rclone --config /etc/rclone/rclone.conf copy -P {{ .Values.dataSource.remote.path }} /data
     cd /data && tar xzvf ${filename}
 fi
+{{- end -}}
diff --git a/charts/tidb-lightning/templates/scripts/_start_lightning.sh.tpl b/charts/tidb-lightning/templates/scripts/_start_lightning.sh.tpl
@@ -2,6 +2,16 @@
 data_dir={{ .Values.dataSource.local.hostPath }}
 {{- else if .Values.dataSource.adhoc.pvcName -}}
 data_dir=/var/lib/tidb-lightning/{{ .Values.dataSource.adhoc.backupName | default .Values.dataSource.adhoc.pvcName }}
+{{- else if .Values.dataSource.remote.directory -}}
+data_dir=/var/lib/tidb-lightning
+if [ -z "$(ls -A ${data_dir})" ]; then
+    if [ ! -z ${FAIL_FAST} ]; then
+        exit 1
+    else
+        echo "No files in data dir, please exec into my container to diagnose"
+        tail -f /dev/null
+    fi
+fi
 {{- else -}}
 data_dir=$(dirname $(find /var/lib/tidb-lightning -name metadata 2>/dev/null) 2>/dev/null)
 if [ -z $data_dir ]; then

diff --git a/charts/tidb-lightning/values.yaml b/charts/tidb-lightning/values.yaml
@@ -30,6 +30,8 @@ dataSource:
     storage: 100Gi
     secretName: cloud-storage-secret
     path: s3:bench-data-us/sysbench/sbtest_16_1e7.tar.gz
+    # Directory support downloading all files in a remote directory, shadow dataSoure.remote.path if present
+    # directory: s3:bench-data-us
     # If rcloneConfig is configured, then `secretName` will be ignored,
     # `rcloneConfig` should only be used for the cases where no sensitive
     # information need to be configured, e.g. the configuration as below,