From 84ec6a8d750ebd86c7eaea45d2902cb09a8340f7 Mon Sep 17 00:00:00 2001 From: Jacob Lerche Date: Fri, 12 Jul 2019 13:08:03 -0700 Subject: [PATCH 1/3] charts: backup: flexible credentials allow retrieving credentials from the environment on GCP, stream the backup upload --- charts/tidb-backup/templates/backup-job.yaml | 15 ++++++-- .../templates/scripts/_start_backup.sh.tpl | 38 +++++++++++++++---- charts/tidb-backup/values.yaml | 17 ++++++++- 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/charts/tidb-backup/templates/backup-job.yaml b/charts/tidb-backup/templates/backup-job.yaml index 4d2acc4c00..efb6fbf508 100644 --- a/charts/tidb-backup/templates/backup-job.yaml +++ b/charts/tidb-backup/templates/backup-job.yaml @@ -2,7 +2,11 @@ apiVersion: batch/v1 kind: Job metadata: + {{- if .Values.jobName }} + name: {{ tpl .Values.jobName . }} + {{- else }} name: {{ .Values.clusterName }}-{{ tpl .Values.name . }} + {{- end }} labels: app.kubernetes.io/name: {{ template "chart.name" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} @@ -22,6 +26,9 @@ spec: {{ toYaml .Values.extraLabels | indent 8 }} {{- end }} spec: + {{- if .Values.serviceAccount }} + serviceAccount: {{ .Values.serviceAccount }} + {{- end }} containers: - name: backup image: {{ .Values.image.backup }} @@ -38,7 +45,7 @@ spec: volumeMounts: - name: data mountPath: "/data" - {{- if .Values.gcp }} + {{- if .Values.gcp.secretName }} - name: gcp-credentials mountPath: "/gcp" readOnly: true @@ -50,11 +57,11 @@ spec: - name: TZ value: {{ .Values.timezone | default "UTC" }} {{- end }} - {{- if .Values.gcp }} + {{- if .Values.gcp.secretName }} - name: GOOGLE_APPLICATION_CREDENTIALS value: /gcp/credentials.json {{- end }} - {{- if or .Values.ceph .Values.s3 }} + {{- if or .Values.ceph.secretName .Values.s3.secretName }} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -81,7 +88,7 @@ spec: - name: data persistentVolumeClaim: claimName: {{ tpl .Values.name . }} - {{- if .Values.gcp }} + {{- if .Values.gcp.secretName }} - name: gcp-credentials secret: secretName: {{ .Values.gcp.secretName }} diff --git a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl index e4e495072a..84ca30429d 100644 --- a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl +++ b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl @@ -1,6 +1,10 @@ set -euo pipefail +{{- if .Values.host }} +host={{ .Values.host }} +{{- else }} host=$(getent hosts {{ .Values.clusterName }}-tidb | head | awk '{print $1}') +{{- end }} dirname=/data/${BACKUP_NAME} echo "making dir ${dirname}" @@ -15,6 +19,13 @@ fi gc_life_time=`/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_value from mysql.tidb where variable_name='tikv_gc_life_time';"` echo "Old TiKV GC life time is ${gc_life_time}" +function reset_gc_lifetime() { +echo "Reset TiKV GC life time to ${gc_life_time}" +/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "update mysql.tidb set variable_value='${gc_life_time}' where variable_name='tikv_gc_life_time';" +/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_name,variable_value from mysql.tidb where variable_name='tikv_gc_life_time';" +} +trap "reset_gc_lifetime" EXIT + echo "Increase TiKV GC life time to {{ .Values.tikvGCLifeTime | default "720h" }}" /usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "update mysql.tidb set variable_value='{{ .Values.tikvGCLifeTime | default "720h" }}' where variable_name='tikv_gc_life_time';" /usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_name,variable_value from mysql.tidb where variable_name='tikv_gc_life_time';" @@ -36,15 +47,28 @@ fi --tidb-force-priority=LOW_PRIORITY \ {{ .Values.backupOptions }} ${snapshot_args:-} -echo "Reset TiKV GC life time to ${gc_life_time}" -/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "update mysql.tidb set variable_value='${gc_life_time}' where variable_name='tikv_gc_life_time';" -/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_name,variable_value from mysql.tidb where variable_name='tikv_gc_life_time';" +backup_name="$(basename "${dirname}")" +backup_base_dir="$(dirname "${dirname}")" {{- if .Values.gcp }} -uploader \ - --cloud=gcp \ - --bucket={{ .Values.gcp.bucket }} \ - --backup-dir=${dirname} +# Once we know there are no more credentials that will be logged we can run with -x +set -x +bucket={{ .Values.gcp.bucket }} +creds=${GOOGLE_APPLICATION_CREDENTIALS:-""} +if ! [[ -z $creds ]] ; then +creds = "service_account_file = ${creds}" +fi + +cat < /tmp/rclone.conf +[gcp] +type = google cloud storage +bucket_policy_only = true +$creds +EOF + + cd "${backup_base_dir}" + tar -cf - "${backup_name}" | pigz -p 16 \ + | rclone --config /tmp/rclone.conf rcat gcp:${bucket}/${backup_name}/${backup_name}.tgz {{- end }} {{- if .Values.ceph }} diff --git a/charts/tidb-backup/values.yaml b/charts/tidb-backup/values.yaml index b54446f871..82953e9427 100644 --- a/charts/tidb-backup/values.yaml +++ b/charts/tidb-backup/values.yaml @@ -13,12 +13,19 @@ mode: backup # backup | restore | scheduled-restore # name is the backup dir name and pvc name for ad-hoc backup and restore name: fullbackup-{{ date "200601021504" .Release.Time }} +# The default jobName is clusterName-name +# jobName: + +# The TiDB host to connect to +# By default this will match a normal tidb-operator deploy +# host: + # scheduledBackupName is the name of a scheduled backup directory, # used to restore the tidb cluster from scheduled backup. # scheduledBackupName: scheduled-backup-20190822-041004 image: - pullPolicy: IfNotPresent + pullPolicy: Always # https://github.com/pingcap/tidb-cloud-backup backup: pingcap/tidb-cloud-backup:20190828 @@ -40,6 +47,9 @@ resources: cpu: 2000m memory: 4Gi +# Kubernetes service account to use +# serviceAccount: + storage: className: local-storage size: 100Gi @@ -48,7 +58,8 @@ storage: # -F is the chunk size, a big table is partitioned into many chunks. # Other useful options are -B for database, and -T for tables. # See https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options for more options. -backupOptions: "-t 16 -F 256 --skip-tz-utc --verbose=3" +backupOptions: "--compress-protocol -t 16 -F 256 --skip-tz-utc --verbose=3" + # Set the tidb_snapshot to be used for the backup # Use `show master status` to get the ts: # MySQL [(none)]> show master status; @@ -72,6 +83,7 @@ tikvGCLifeTime: 720h # backup to or restore from gcp bucket, the backup path is in the form of - gcp: {} # bucket: "" + # secretName is not necessary on GKE if you use the workload identity feature # secretName is the name of the secret which stores the gcp service account credentials json file # The service account must have read/write permission to the above bucket. # Read the following document to create the service account and download the credentials file as credentials.json: @@ -94,6 +106,7 @@ s3: {} # region: "" # bucket: "" # secretName is the name of the secret which stores s3 object store access key and secret key + # This is not necessary on AWS. Instead you should be able to get the credentials from the EKS service IAM role. # You can create the secret by: # kubectl create secret generic s3-backup-secret --namespace= --from-literal=access_key= --from-literal=secret_key= # secretName: s3-backup-secret From 513bc725124799788b62fbbc45d48d2639b09cc5 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Mon, 2 Dec 2019 18:24:33 -0800 Subject: [PATCH 2/3] Apply suggestions from code review Co-Authored-By: Tennix --- charts/tidb-backup/templates/scripts/_start_backup.sh.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl index 84ca30429d..f1cd112c68 100644 --- a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl +++ b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl @@ -56,7 +56,7 @@ set -x bucket={{ .Values.gcp.bucket }} creds=${GOOGLE_APPLICATION_CREDENTIALS:-""} if ! [[ -z $creds ]] ; then -creds = "service_account_file = ${creds}" +creds="service_account_file = ${creds}" fi cat < /tmp/rclone.conf From 09b740d18e9a92f0fdb826ada2bb7459a1f64899 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Mon, 2 Dec 2019 18:27:36 -0800 Subject: [PATCH 3/3] code review suggestions --- charts/tidb-backup/templates/scripts/_start_backup.sh.tpl | 4 ++-- charts/tidb-backup/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl index f1cd112c68..f33b0d1b14 100644 --- a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl +++ b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl @@ -66,8 +66,8 @@ bucket_policy_only = true $creds EOF - cd "${backup_base_dir}" - tar -cf - "${backup_name}" | pigz -p 16 \ +cd "${backup_base_dir}" +tar -cf - "${backup_name}" | pigz -p 16 \ | rclone --config /tmp/rclone.conf rcat gcp:${bucket}/${backup_name}/${backup_name}.tgz {{- end }} diff --git a/charts/tidb-backup/values.yaml b/charts/tidb-backup/values.yaml index 82953e9427..42c72a20ba 100644 --- a/charts/tidb-backup/values.yaml +++ b/charts/tidb-backup/values.yaml @@ -25,7 +25,7 @@ name: fullbackup-{{ date "200601021504" .Release.Time }} # scheduledBackupName: scheduled-backup-20190822-041004 image: - pullPolicy: Always + pullPolicy: IfNotPresent # https://github.com/pingcap/tidb-cloud-backup backup: pingcap/tidb-cloud-backup:20190828