backup: TLS Support for Backup&Restore with dumpling&lightning (#3100)

* support tls in dumpling&lightning

cmd/backup-manager/app/cmd/export.go

@@ -45,6 +45,7 @@ func NewExportCommand() *cobra.Command {
 	cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace")
 	cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name")
 	cmd.Flags().StringVar(&bo.Bucket, "bucket", "", "Bucket in which to store the backup data")
+	cmd.Flags().BoolVar(&bo.TLSClient, "client-tls", false, "Whether client tls is enabled")
 	cmd.Flags().StringVar(&bo.StorageType, "storageType", "", "Backend storage type")
 	return cmd
 }

cmd/backup-manager/app/cmd/import.go

@@ -44,6 +44,7 @@ func NewImportCommand() *cobra.Command {
 
 	cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace")
 	cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name")
+	cmd.Flags().BoolVar(&ro.TLSClient, "client-tls", false, "Whether client tls is enabled")
 	cmd.Flags().StringVar(&ro.BackupPath, "backupPath", "", "The location of the backup")
 	return cmd
 }

cmd/backup-manager/app/export/export.go

@@ -16,21 +16,24 @@ package export
 import (
 	"fmt"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/mholt/archiver"
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
-	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/util"
+	backupUtil "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+	"github.com/pingcap/tidb-operator/pkg/util"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog"
 )
 
 // Options contains the input arguments to the backup command
 type Options struct {
-	util.GenericOptions
+	backupUtil.GenericOptions
 	Bucket      string
 	Prefix      string
 	StorageType string
@@ -57,7 +60,7 @@ func (bo *Options) getDestBucketURI(remotePath string) string {
 
 func (bo *Options) dumpTidbClusterData(backup *v1alpha1.Backup) (string, error) {
 	bfPath := bo.getBackupFullPath()
-	err := util.EnsureDirectoryExist(bfPath)
+	err := backupUtil.EnsureDirectoryExist(bfPath)
 	if err != nil {
 		return "", err
 	}
@@ -68,7 +71,12 @@ func (bo *Options) dumpTidbClusterData(backup *v1alpha1.Backup) (string, error)
 		fmt.Sprintf("--user=%s", bo.User),
 		fmt.Sprintf("--password=%s", bo.Password),
 	}
-	args = append(args, util.ConstructDumplingOptionsForBackup(backup)...)
+	args = append(args, backupUtil.ConstructDumplingOptionsForBackup(backup)...)
+	if bo.TLSClient {
+		args = append(args, fmt.Sprintf("--ca=%s", path.Join(util.TiDBClientTLSPath, corev1.ServiceAccountRootCAKey)))
+		args = append(args, fmt.Sprintf("--cert=%s", path.Join(util.TiDBClientTLSPath, corev1.TLSCertKey)))
+		args = append(args, fmt.Sprintf("--key=%s", path.Join(util.TiDBClientTLSPath, corev1.TLSPrivateKeyKey)))
+	}
 
 	klog.Infof("The dump process is ready, command \"/dumpling %s\"", strings.Join(args, " "))
 
@@ -80,9 +88,9 @@ func (bo *Options) dumpTidbClusterData(backup *v1alpha1.Backup) (string, error)
 }
 
 func (bo *Options) backupDataToRemote(source, bucketURI string, opts []string) error {
-	destBucket := util.NormalizeBucketURI(bucketURI)
+	destBucket := backupUtil.NormalizeBucketURI(bucketURI)
 	tmpDestBucket := fmt.Sprintf("%s.tmp", destBucket)
-	args := util.ConstructArgs(constants.RcloneConfigArg, opts, "copyto", source, tmpDestBucket)
+	args := backupUtil.ConstructArgs(constants.RcloneConfigArg, opts, "copyto", source, tmpDestBucket)
 	// TODO: We may need to use exec.CommandContext to control timeouts.
 	output, err := exec.Command("rclone", args...).CombinedOutput()
 	if err != nil {
@@ -93,7 +101,7 @@ func (bo *Options) backupDataToRemote(source, bucketURI string, opts []string) e
 
 	// the backup was a success
 	// remove .tmp extension
-	args = util.ConstructArgs(constants.RcloneConfigArg, opts, "moveto", tmpDestBucket, destBucket)
+	args = backupUtil.ConstructArgs(constants.RcloneConfigArg, opts, "moveto", tmpDestBucket, destBucket)
 	output, err = exec.Command("rclone", args...).CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("cluster %s, execute rclone moveto command failed, output: %s, err: %v", bo, string(output), err)
@@ -104,10 +112,10 @@ func (bo *Options) backupDataToRemote(source, bucketURI string, opts []string) e
 // getBackupSize get the backup data size
 func getBackupSize(backupPath string, opts []string) (int64, error) {
 	var size int64
-	if exist := util.IsFileExist(backupPath); !exist {
+	if exist := backupUtil.IsFileExist(backupPath); !exist {
 		return size, fmt.Errorf("file %s does not exist or is not regular file", backupPath)
 	}
-	args := util.ConstructArgs(constants.RcloneConfigArg, opts, "ls", backupPath, "")
+	args := backupUtil.ConstructArgs(constants.RcloneConfigArg, opts, "ls", backupPath, "")
 	out, err := exec.Command("rclone", args...).CombinedOutput()
 	if err != nil {
 		return size, fmt.Errorf("failed to get backup %s size, err: %v", backupPath, err)
@@ -122,11 +130,11 @@ func getBackupSize(backupPath string, opts []string) (int64, error) {
 
 // archiveBackupData archive backup data by destFile's extension name
 func archiveBackupData(backupDir, destFile string) error {
-	if exist := util.IsDirExist(backupDir); !exist {
+	if exist := backupUtil.IsDirExist(backupDir); !exist {
 		return fmt.Errorf("dir %s does not exist or is not a dir", backupDir)
 	}
 	destDir := filepath.Dir(destFile)
-	if err := util.EnsureDirectoryExist(destDir); err != nil {
+	if err := backupUtil.EnsureDirectoryExist(destDir); err != nil {
 		return err
 	}
 	err := archiver.Archive([]string{backupDir}, destFile)

cmd/backup-manager/app/export/manager.go

@@ -111,8 +111,7 @@ func (bm *BackupManager) ProcessBackup() error {
 	var db *sql.DB
 	var dsn string
 	err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) {
-		// TLS is not currently supported
-		dsn, err = bm.GetDSN(false)
+		dsn, err = bm.GetDSN(bm.TLSClient)
 		if err != nil {
 			klog.Errorf("can't get dsn of tidb cluster %s, err: %s", bm, err)
 			return false, err

cmd/backup-manager/app/import/restore.go → cmd/backup-manager/app/import/import.go

@@ -17,18 +17,21 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
 
 	"github.com/mholt/archiver"
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
-	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/util"
+	backupUtil "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util"
+	"github.com/pingcap/tidb-operator/pkg/util"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog"
 )
 
 // Options contains the input arguments to the restore command
 type Options struct {
-	util.GenericOptions
+	backupUtil.GenericOptions
 	BackupPath string
 }
 
@@ -39,12 +42,12 @@ func (ro *Options) getRestoreDataPath() string {
 }
 
 func (ro *Options) downloadBackupData(localPath string, opts []string) error {
-	if err := util.EnsureDirectoryExist(filepath.Dir(localPath)); err != nil {
+	if err := backupUtil.EnsureDirectoryExist(filepath.Dir(localPath)); err != nil {
 		return err
 	}
 
-	remoteBucket := util.NormalizeBucketURI(ro.BackupPath)
-	args := util.ConstructArgs(constants.RcloneConfigArg, opts, "copyto", remoteBucket, localPath)
+	remoteBucket := backupUtil.NormalizeBucketURI(ro.BackupPath)
+	args := backupUtil.ConstructArgs(constants.RcloneConfigArg, opts, "copyto", remoteBucket, localPath)
 	rcCopy := exec.Command("rclone", args...)
 
 	stdOut, err := rcCopy.StdoutPipe()
@@ -79,7 +82,7 @@ func (ro *Options) downloadBackupData(localPath string, opts []string) error {
 }
 
 func (ro *Options) loadTidbClusterData(restorePath string) error {
-	if exist := util.IsDirExist(restorePath); !exist {
+	if exist := backupUtil.IsDirExist(restorePath); !exist {
 		return fmt.Errorf("dir %s does not exist or is not a dir", restorePath)
 	}
 	// args for restore
@@ -94,6 +97,11 @@ func (ro *Options) loadTidbClusterData(restorePath string) error {
 		fmt.Sprintf("--d=%s", restorePath),
 		fmt.Sprintf("--tidb-port=%d", ro.Port),
 	}
+	if ro.TLSClient {
+		args = append(args, fmt.Sprintf("--ca=%s", path.Join(util.TiDBClientTLSPath, corev1.ServiceAccountRootCAKey)))
+		args = append(args, fmt.Sprintf("--cert=%s", path.Join(util.TiDBClientTLSPath, corev1.TLSCertKey)))
+		args = append(args, fmt.Sprintf("--key=%s", path.Join(util.TiDBClientTLSPath, corev1.TLSPrivateKeyKey)))
+	}
 
 	output, err := exec.Command("/tidb-lightning", args...).CombinedOutput()
 	if err != nil {
@@ -105,7 +113,7 @@ func (ro *Options) loadTidbClusterData(restorePath string) error {
 // unarchiveBackupData unarchive backup data to dest dir
 func unarchiveBackupData(backupFile, destDir string) (string, error) {
 	var unarchiveBackupPath string
-	if err := util.EnsureDirectoryExist(destDir); err != nil {
+	if err := backupUtil.EnsureDirectoryExist(destDir); err != nil {
 		return unarchiveBackupPath, err
 	}
 	backupName := strings.TrimSuffix(filepath.Base(backupFile), constants.DefaultArchiveExtention)

cmd/backup-manager/app/import/manager.go

@@ -91,8 +91,8 @@ func (rm *RestoreManager) ProcessRestore() error {
 	var db *sql.DB
 	var dsn string
 	err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) {
-		// TLS is not currently supported
-		dsn, err = rm.GetDSN(false)
+		// TODO: for local backend mode, lightning will use both pd and mysql client. We should set both tls configurations in that mode
+		dsn, err = rm.GetDSN(rm.TLSClient)
 		if err != nil {
 			klog.Errorf("can't get dsn of tidb cluster %s, err: %s", rm, err)
 			return false, err

pkg/backup/backup/backup_manager.go

@@ -199,6 +199,26 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s
 		fmt.Sprintf("--storageType=%s", backuputil.GetStorageType(backup.Spec.StorageProvider)),
 	}
 
+	volumeMounts := []corev1.VolumeMount{}
+	volumes := []corev1.Volume{}
+	if backup.Spec.From.TLSClientSecretName != nil {
+		args = append(args, "--client-tls=true")
+		clientSecretName := *backup.Spec.From.TLSClientSecretName
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "tidb-client-tls",
+			ReadOnly:  true,
+			MountPath: util.TiDBClientTLSPath,
+		})
+		volumes = append(volumes, corev1.Volume{
+			Name: "tidb-client-tls",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: clientSecretName,
+				},
+			},
+		})
+	}
+
 	serviceAccount := constants.DefaultServiceAccountName
 	if backup.Spec.ServiceAccount != "" {
 		serviceAccount = backup.Spec.ServiceAccount
@@ -218,17 +238,17 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s
 					Image:           controller.TidbBackupManagerImage,
 					Args:            args,
 					ImagePullPolicy: corev1.PullIfNotPresent,
-					VolumeMounts: []corev1.VolumeMount{
+					VolumeMounts: append([]corev1.VolumeMount{
 						{Name: label.BackupJobLabelVal, MountPath: constants.BackupRootPath},
-					},
+					}, volumeMounts...),
 					Env:       util.AppendEnvIfPresent(envVars, "TZ"),
 					Resources: backup.Spec.ResourceRequirements,
 				},
 			},
 			RestartPolicy: corev1.RestartPolicyNever,
 			Affinity:      backup.Spec.Affinity,
 			Tolerations:   backup.Spec.Tolerations,
-			Volumes: []corev1.Volume{
+			Volumes: append([]corev1.Volume{
 				{
 					Name: label.BackupJobLabelVal,
 					VolumeSource: corev1.VolumeSource{
@@ -237,7 +257,7 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s
 						},
 					},
 				},
-			},
+			}, volumes...),
 		},
 	}
 

pkg/backup/restore/restore_manager.go

@@ -184,6 +184,26 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job
 		fmt.Sprintf("--backupPath=%s", backupPath),
 	}
 
+	volumeMounts := []corev1.VolumeMount{}
+	volumes := []corev1.Volume{}
+	if restore.Spec.To.TLSClientSecretName != nil {
+		args = append(args, "--client-tls=true")
+		clientSecretName := *restore.Spec.To.TLSClientSecretName
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "tidb-client-tls",
+			ReadOnly:  true,
+			MountPath: util.TiDBClientTLSPath,
+		})
+		volumes = append(volumes, corev1.Volume{
+			Name: "tidb-client-tls",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: clientSecretName,
+				},
+			},
+		})
+	}
+
 	restoreLabel := label.NewBackup().Instance(restore.GetInstanceName()).RestoreJob().Restore(name)
 	serviceAccount := constants.DefaultServiceAccountName
 	if restore.Spec.ServiceAccount != "" {
@@ -204,17 +224,17 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job
 					Image:           controller.TidbBackupManagerImage,
 					Args:            args,
 					ImagePullPolicy: corev1.PullIfNotPresent,
-					VolumeMounts: []corev1.VolumeMount{
+					VolumeMounts: append([]corev1.VolumeMount{
 						{Name: label.RestoreJobLabelVal, MountPath: constants.BackupRootPath},
-					},
+					}, volumeMounts...),
 					Env:       util.AppendEnvIfPresent(envVars, "TZ"),
 					Resources: restore.Spec.ResourceRequirements,
 				},
 			},
 			RestartPolicy: corev1.RestartPolicyNever,
 			Affinity:      restore.Spec.Affinity,
 			Tolerations:   restore.Spec.Tolerations,
-			Volumes: []corev1.Volume{
+			Volumes: append([]corev1.Volume{
 				{
 					Name: label.RestoreJobLabelVal,
 					VolumeSource: corev1.VolumeSource{
@@ -223,7 +243,7 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job
 						},
 					},
 				},
-			},
+			}, volumes...),
 		},
 	}