backup: support kms decryption secret (#1908)

images/tidb-backup-manager/Dockerfile

@@ -1,5 +1,6 @@
 FROM pingcap/tidb-enterprise-tools:latest
 ARG VERSION=v1.51.0
+ARG SHUSH_VERSION=v1.4.0
 RUN apk update && apk add ca-certificates
 
 RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \
@@ -14,6 +15,10 @@ RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \
     && chmod 755 /usr/local/bin/br \
     && rm -rf br-latest-linux-amd64.tar.gz
 
+RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \
+  && mv shush_linux_amd64 /usr/local/bin/shush \
+  && chmod 755 /usr/local/bin/shush
+
 COPY bin/tidb-backup-manager /tidb-backup-manager
 COPY entrypoint.sh /entrypoint.sh
 

images/tidb-backup-manager/entrypoint.sh

@@ -51,33 +51,40 @@ else
 fi
 
 BACKUP_BIN=/tidb-backup-manager
+if [[ -n "${AWS_DEFAULT_REGION}"]]; then
+	EXEC_COMMAND="exec"
+else
+	EXEC_COMMAND="/usr/local/bin/shush exec --"
+fi
+
+cat /tmp/rclone.conf
 
 # exec command
 case "$1" in
     backup)
         shift 1
         echo "$BACKUP_BIN backup $@"
-        exec $BACKUP_BIN backup "$@"
+        $EXEC_COMMAND $BACKUP_BIN backup "$@"
         ;;
     export)
         shift 1
         echo "$BACKUP_BIN export $@"
-        exec $BACKUP_BIN export "$@"
+        $EXEC_COMMAND $BACKUP_BIN export "$@"
         ;;
     restore)
         shift 1
         echo "$BACKUP_BIN restore $@"
-        exec $BACKUP_BIN restore "$@"
+        $EXEC_COMMAND $BACKUP_BIN restore "$@"
         ;;
     import)
         shift 1
         echo "$BACKUP_BIN import $@"
-        exec $BACKUP_BIN import "$@"
+        $EXEC_COMMAND $BACKUP_BIN import "$@"
         ;;
     clean)
         shift 1
         echo "$BACKUP_BIN clean $@"
-        exec $BACKUP_BIN clean "$@"
+        $EXEC_COMMAND $BACKUP_BIN clean "$@"
         ;;
     *)
         echo "Usage: $0 {backup|restore|clean}"

manifests/backup/backup-aws-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
     # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
   # backupType: full
+  # useKMS: false
   # serviceAccount: myServiceAccount
   br:
     cluster: myCluster

manifests/backup/backup-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
     # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
   # backupType: full
+  # useKMS: false
   # serviceAccount: myServiceAccount
   br:
     cluster: myCluster

manifests/backup/backup-schedule-aws-s3-br.yaml

@@ -13,6 +13,7 @@ spec:
   schedule: "*/2 * * * *"
   backupTemplate:
     #backupType: full
+    # useKMS: false
     # serviceAccount: myServiceAccount
     br:
       cluster: myCluster

manifests/backup/backup-schedule-s3-br.yaml

@@ -13,6 +13,7 @@ spec:
   schedule: "*/2 * * * *"
   backupTemplate:
     #backupType: full
+    # useKMS: false
     # serviceAccount: myServiceAccount
     br:
       cluster: myCluster

manifests/backup/restore-aws-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
     # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
   # backupType: full
+  # useKMS: false
   # serviceAccount: myServiceAccount
   br:
     cluster: myCluster

manifests/backup/restore-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
     # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
   # backupType: full
+  # useKMS: false
   # serviceAccount: myServiceAccount
   br:
     cluster: myCluster

manifests/crd.yaml

@@ -6954,6 +6954,9 @@ spec:
                     type: string
                 type: object
               type: array
+            useKMS:
+              description: Use KMS to decrypt the secrets
+              type: boolean
           type: object
       type: object
   version: v1alpha1
@@ -7795,6 +7798,9 @@ spec:
                     type: string
                 type: object
               type: array
+            useKMS:
+              description: Use KMS to decrypt the secrets
+              type: boolean
           type: object
       type: object
   version: v1alpha1
@@ -8680,6 +8686,9 @@ spec:
                         type: string
                     type: object
                   type: array
+                useKMS:
+                  description: Use KMS to decrypt the secrets
+                  type: boolean
               type: object
             maxBackups:
               description: MaxBackups is to specify how many backups we want to keep

pkg/apis/pingcap/v1alpha1/types.go

@@ -804,6 +804,8 @@ type BackupSpec struct {
 	// Affinity of backup Pods
 	// +optional
 	Affinity *corev1.Affinity `json:"affinity,omitempty"`
+	// Use KMS to decrypt the secrets
+	UseKMS bool `json:"useKMS,omitempty"`
 	// Specify service account of backup
 	ServiceAccount string `json:"serviceAccount,omitempty"`
 }
@@ -1024,6 +1026,8 @@ type RestoreSpec struct {
 	// Affinity of restore Pods
 	// +optional
 	Affinity *corev1.Affinity `json:"affinity,omitempty"`
+	// Use KMS to decrypt the secrets
+	UseKMS bool `json:"useKMS,omitempty"`
 	// Specify service account of restore
 	ServiceAccount string `json:"serviceAccount,omitempty"`
 }

pkg/backup/backup/backup_cleaner.go

@@ -128,7 +128,6 @@ func (bc *backupCleaner) makeCleanJob(backup *v1alpha1.Backup) (*batchv1.Job, st
 		serviceAccount = backup.Spec.ServiceAccount
 	}
 	backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).CleanJob().Backup(name)
-
 	podSpec := &corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels:      backupLabel.Labels(),

pkg/backup/backup/backup_manager.go

@@ -163,7 +163,7 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s
 	ns := backup.GetNamespace()
 	name := backup.GetName()
 
-	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister)
+	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.secretLister)
 	if err != nil {
 		return nil, reason, err
 	}
@@ -255,7 +255,7 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s
 	ns := backup.GetNamespace()
 	name := backup.GetName()
 
-	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister)
+	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.secretLister)
 	if err != nil {
 		return nil, reason, err
 	}

pkg/backup/constants/constants.go

@@ -55,4 +55,7 @@ const (
 
 	// ServiceAccountCAPath is where is CABundle of serviceaccount locates
 	ServiceAccountCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+	// KMS secret env prefix
+	KMSSecretPrefix = "KMS_ENCRYPTED"
 )

pkg/backup/restore/restore_manager.go

@@ -154,7 +154,7 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job
 	ns := restore.GetNamespace()
 	name := restore.GetName()
 
-	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister)
+	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.secretLister)
 	if err != nil {
 		return nil, reason, err
 	}
@@ -240,7 +240,7 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo
 	ns := restore.GetNamespace()
 	name := restore.GetName()
 
-	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister)
+	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.secretLister)
 	if err != nil {
 		return nil, reason, err
 	}

pkg/backup/util/util.go

@@ -213,8 +213,9 @@ func GenerateStorageCertEnv(ns string, provider v1alpha1.StorageProvider, secret
 }
 
 // GenerateTidbPasswordEnv generate the password EnvVar
-func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, secretLister corelisters.SecretLister) ([]corev1.EnvVar, string, error) {
+func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, useKMS bool, secretLister corelisters.SecretLister) ([]corev1.EnvVar, string, error) {
 	var certEnv []corev1.EnvVar
+	var passwordKey string
 	secret, err := secretLister.Secrets(ns).Get(tidbSecretName)
 	if err != nil {
 		err = fmt.Errorf("backup %s/%s get tidb secret %s failed, err: %v", ns, name, tidbSecretName, err)
@@ -226,9 +227,16 @@ func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, secretLister corel
 		err = fmt.Errorf("backup %s/%s, tidb secret %s missing password key %s", ns, name, tidbSecretName, keyStr)
 		return certEnv, "KeyNotExist", err
 	}
+
+	if useKMS {
+		passwordKey = fmt.Sprintf("%s_%s_%s", constants.KMSSecretPrefix, constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey))
+	} else {
+		passwordKey = fmt.Sprintf("%s_%s", constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey))
+	}
+
 	certEnv = []corev1.EnvVar{
 		{
-			Name: fmt.Sprintf("%s_%s", constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey)),
+			Name: passwordKey,
 			ValueFrom: &corev1.EnvVarSource{
 				SecretKeyRef: &corev1.SecretKeySelector{
 					LocalObjectReference: corev1.LocalObjectReference{Name: tidbSecretName},