From a786dce19ae8ce86fdbda5c7d39f5005c989ee7c Mon Sep 17 00:00:00 2001 From: pingcap-github-bot Date: Tue, 28 Apr 2020 22:51:56 +0800 Subject: [PATCH] support rclone options in backup and restore cr (#2318) (#2326) --- cmd/backup-manager/app/clean/clean.go | 5 +-- cmd/backup-manager/app/clean/manager.go | 4 ++- cmd/backup-manager/app/export/export.go | 13 +++++--- cmd/backup-manager/app/export/manager.go | 5 +-- cmd/backup-manager/app/import/manager.go | 3 +- cmd/backup-manager/app/import/restore.go | 31 +++++++++++++++-- cmd/backup-manager/app/util/util.go | 33 +++++++++++++++++++ docs/api-references/docs.md | 11 +++++++ manifests/crd.yaml | 18 ++++++++++ .../pingcap/v1alpha1/openapi_generated.go | 14 ++++++++ pkg/apis/pingcap/v1alpha1/types.go | 2 ++ .../pingcap/v1alpha1/zz_generated.deepcopy.go | 7 +++- 12 files changed, 131 insertions(+), 15 deletions(-) diff --git a/cmd/backup-manager/app/clean/clean.go b/cmd/backup-manager/app/clean/clean.go index c727319d58..9bf174306e 100644 --- a/cmd/backup-manager/app/clean/clean.go +++ b/cmd/backup-manager/app/clean/clean.go @@ -63,9 +63,10 @@ func (bo *Options) cleanBRRemoteBackupData(backup *v1alpha1.Backup) error { return nil } -func (bo *Options) cleanRemoteBackupData(bucket string) error { +func (bo *Options) cleanRemoteBackupData(bucket string, opts []string) error { destBucket := util.NormalizeBucketURI(bucket) - output, err := exec.Command("rclone", constants.RcloneConfigArg, "deletefile", destBucket).CombinedOutput() + args := util.ConstructArgs(constants.RcloneConfigArg, opts, "deletefile", destBucket, "") + output, err := exec.Command("rclone", args...).CombinedOutput() if err != nil { return fmt.Errorf("cluster %s, execute rclone deletefile command failed, output: %s, err: %v", bo, string(output), err) } diff --git a/cmd/backup-manager/app/clean/manager.go b/cmd/backup-manager/app/clean/manager.go index e2e4061567..c7302ddfa9 100644 --- a/cmd/backup-manager/app/clean/manager.go +++ b/cmd/backup-manager/app/clean/manager.go @@ -19,6 +19,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/klog" + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" @@ -68,7 +69,8 @@ func (bm *Manager) performCleanBackup(backup *v1alpha1.Backup) error { if backup.Spec.BR != nil { err = bm.cleanBRRemoteBackupData(backup) } else { - err = bm.cleanRemoteBackupData(backup.Status.BackupPath) + opts := util.GetOptions(backup.Spec.StorageProvider) + err = bm.cleanRemoteBackupData(backup.Status.BackupPath, opts) } if err != nil { diff --git a/cmd/backup-manager/app/export/export.go b/cmd/backup-manager/app/export/export.go index fe9d37e40b..e1b0814cac 100644 --- a/cmd/backup-manager/app/export/export.go +++ b/cmd/backup-manager/app/export/export.go @@ -74,11 +74,12 @@ func (bo *Options) dumpTidbClusterData() (string, error) { return bfPath, nil } -func (bo *Options) backupDataToRemote(source, bucketURI string) error { +func (bo *Options) backupDataToRemote(source, bucketURI string, opts []string) error { destBucket := util.NormalizeBucketURI(bucketURI) tmpDestBucket := fmt.Sprintf("%s.tmp", destBucket) + args := util.ConstructArgs(constants.RcloneConfigArg, opts, "copyto", source, tmpDestBucket) // TODO: We may need to use exec.CommandContext to control timeouts. - output, err := exec.Command("rclone", constants.RcloneConfigArg, "copyto", source, tmpDestBucket).CombinedOutput() + output, err := exec.Command("rclone", args...).CombinedOutput() if err != nil { return fmt.Errorf("cluster %s, execute rclone copyto command for upload backup data %s failed, output: %s, err: %v", bo, bucketURI, string(output), err) } @@ -87,7 +88,8 @@ func (bo *Options) backupDataToRemote(source, bucketURI string) error { // the backup was a success // remove .tmp extension - output, err = exec.Command("rclone", constants.RcloneConfigArg, "moveto", tmpDestBucket, destBucket).CombinedOutput() + args = util.ConstructArgs(constants.RcloneConfigArg, opts, "moveto", tmpDestBucket, destBucket) + output, err = exec.Command("rclone", args...).CombinedOutput() if err != nil { return fmt.Errorf("cluster %s, execute rclone moveto command failed, output: %s, err: %v", bo, string(output), err) } @@ -134,12 +136,13 @@ func getCommitTsFromMetadata(backupPath string) (string, error) { } // getBackupSize get the backup data size -func getBackupSize(backupPath string) (int64, error) { +func getBackupSize(backupPath string, opts []string) (int64, error) { var size int64 if exist := util.IsFileExist(backupPath); !exist { return size, fmt.Errorf("file %s does not exist or is not regular file", backupPath) } - out, err := exec.Command("rclone", constants.RcloneConfigArg, "ls", backupPath).CombinedOutput() + args := util.ConstructArgs(constants.RcloneConfigArg, opts, "ls", backupPath, "") + out, err := exec.Command("rclone", args...).CombinedOutput() if err != nil { return size, fmt.Errorf("failed to get backup %s size, err: %v", backupPath, err) } diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index 544dd75598..b4259f3fbd 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -229,7 +229,8 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro } klog.Infof("archive cluster %s backup data %s success", bm, archiveBackupPath) - size, err := getBackupSize(archiveBackupPath) + opts := util.GetOptions(backup.Spec.StorageProvider) + size, err := getBackupSize(archiveBackupPath, opts) if err != nil { klog.Errorf("get cluster %s archived backup file %s size %d failed, err: %s", bm, archiveBackupPath, size, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -255,7 +256,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro remotePath := strings.TrimPrefix(archiveBackupPath, constants.BackupRootPath+"/") bucketURI := bm.getDestBucketURI(remotePath) - err = bm.backupDataToRemote(archiveBackupPath, bucketURI) + err = bm.backupDataToRemote(archiveBackupPath, bucketURI, opts) if err != nil { klog.Errorf("backup cluster %s data to %s failed, err: %s", bm, bm.StorageType, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ diff --git a/cmd/backup-manager/app/import/manager.go b/cmd/backup-manager/app/import/manager.go index 624d24e156..2d2d26ab06 100644 --- a/cmd/backup-manager/app/import/manager.go +++ b/cmd/backup-manager/app/import/manager.go @@ -127,7 +127,8 @@ func (rm *RestoreManager) performRestore(restore *v1alpha1.Restore) error { } restoreDataPath := rm.getRestoreDataPath() - if err := rm.downloadBackupData(restoreDataPath); err != nil { + opts := util.GetOptions(restore.Spec.StorageProvider) + if err := rm.downloadBackupData(restoreDataPath, opts); err != nil { klog.Errorf("download cluster %s backup %s data failed, err: %s", rm, rm.BackupPath, err) return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ Type: v1alpha1.RestoreFailed, diff --git a/cmd/backup-manager/app/import/restore.go b/cmd/backup-manager/app/import/restore.go index fc28da5ed3..e76f339387 100644 --- a/cmd/backup-manager/app/import/restore.go +++ b/cmd/backup-manager/app/import/restore.go @@ -15,6 +15,7 @@ package _import import ( "fmt" + "io/ioutil" "os/exec" "path/filepath" "strings" @@ -22,6 +23,7 @@ import ( "github.com/mholt/archiver" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" + "k8s.io/klog" ) // Options contains the input arguments to the restore command @@ -36,18 +38,41 @@ func (ro *Options) getRestoreDataPath() string { return filepath.Join(constants.BackupRootPath, bucketName, backupName) } -func (ro *Options) downloadBackupData(localPath string) error { +func (ro *Options) downloadBackupData(localPath string, opts []string) error { if err := util.EnsureDirectoryExist(filepath.Dir(localPath)); err != nil { return err } remoteBucket := util.NormalizeBucketURI(ro.BackupPath) - rcCopy := exec.Command("rclone", constants.RcloneConfigArg, "copyto", remoteBucket, localPath) + args := util.ConstructArgs(constants.RcloneConfigArg, opts, "copyto", remoteBucket, localPath) + rcCopy := exec.Command("rclone", args...) + + stdOut, err := rcCopy.StdoutPipe() + if err != nil { + return fmt.Errorf("cluster %s, create stdout pipe failed, err: %v", ro, err) + } + stdErr, err := rcCopy.StderrPipe() + if err != nil { + return fmt.Errorf("cluster %s, create stderr pipe failed, err: %v", ro, err) + } + if err := rcCopy.Start(); err != nil { return fmt.Errorf("cluster %s, start rclone copyto command for download backup data %s falied, err: %v", ro, ro.BackupPath, err) } + + var errMsg string + tmpOut, _ := ioutil.ReadAll(stdOut) + if len(tmpOut) > 0 { + klog.Infof(string(tmpOut)) + } + tmpErr, _ := ioutil.ReadAll(stdErr) + if len(tmpErr) > 0 { + klog.Infof(string(tmpErr)) + errMsg = string(tmpErr) + } + if err := rcCopy.Wait(); err != nil { - return fmt.Errorf("cluster %s, execute rclone copyto command for download backup data %s failed, err: %v", ro, ro.BackupPath, err) + return fmt.Errorf("cluster %s, execute rclone copyto command for download backup data %s failed, errMsg: %v, err: %v", ro, ro.BackupPath, errMsg, err) } return nil diff --git a/cmd/backup-manager/app/util/util.go b/cmd/backup-manager/app/util/util.go index 89cffee275..bad36d8b47 100644 --- a/cmd/backup-manager/app/util/util.go +++ b/cmd/backup-manager/app/util/util.go @@ -25,6 +25,7 @@ import ( cmdutil "k8s.io/kubectl/pkg/cmd/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/backup/util" ) var ( @@ -187,3 +188,35 @@ func Suffix(version string) string { } return defaultSuffix } + +// GetOptions gets the rclone options +func GetOptions(provider v1alpha1.StorageProvider) []string { + st := util.GetStorageType(provider) + switch st { + case v1alpha1.BackupStorageTypeS3: + return provider.S3.Options + default: + return nil + } +} + +// ConstructArgs constructs the rclone args +func ConstructArgs(conf string, opts []string, command, source, dest string) []string { + var args []string + if conf != "" { + args = append(args, conf) + } + if len(opts) > 0 { + args = append(args, opts...) + } + if command != "" { + args = append(args, command) + } + if source != "" { + args = append(args, source) + } + if dest != "" { + args = append(args, dest) + } + return args +} diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 754f822d94..feb516e0b0 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -7277,6 +7277,17 @@ string

SSE Sever-Side Encryption.

+ + +options
+ +[]string + + + +

Options Rclone options for backup and restore with mydumper and lightning.

+ +

S3StorageProviderType

diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 44aae164e0..e6781a9032 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -8509,6 +8509,12 @@ spec: endpoint: description: Endpoint of S3 compatible storage service type: string + options: + description: Options Rclone options for backup and restore with + mydumper and lightning. + items: + type: string + type: array path: description: 'Path is the full path where the backup is saved. The format of the path must be: "/"' @@ -9330,6 +9336,12 @@ spec: endpoint: description: Endpoint of S3 compatible storage service type: string + options: + description: Options Rclone options for backup and restore with + mydumper and lightning. + items: + type: string + type: array path: description: 'Path is the full path where the backup is saved. The format of the path must be: "/"' @@ -10242,6 +10254,12 @@ spec: endpoint: description: Endpoint of S3 compatible storage service type: string + options: + description: Options Rclone options for backup and restore with + mydumper and lightning. + items: + type: string + type: array path: description: 'Path is the full path where the backup is saved. The format of the path must be: "/"' diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 5e1411920f..803bb3d5e2 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -3536,6 +3536,20 @@ func schema_pkg_apis_pingcap_v1alpha1_S3StorageProvider(ref common.ReferenceCall Format: "", }, }, + "options": { + SchemaProps: spec.SchemaProps{ + Description: "Options Rclone options for backup and restore with mydumper and lightning.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, }, Required: []string{"provider"}, }, diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 0615667360..5f7ea3d7c1 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -844,6 +844,8 @@ type S3StorageProvider struct { Prefix string `json:"prefix,omitempty"` // SSE Sever-Side Encryption. SSE string `json:"sse,omitempty"` + // Options Rclone options for backup and restore with mydumper and lightning. + Options []string `json:"options,omitempty"` } // +k8s:openapi-gen=true diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index 75ceb70095..366f9d49ad 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -2854,6 +2854,11 @@ func (in *RestoreStatus) DeepCopy() *RestoreStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *S3StorageProvider) DeepCopyInto(out *S3StorageProvider) { *out = *in + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -3082,7 +3087,7 @@ func (in *StorageProvider) DeepCopyInto(out *StorageProvider) { if in.S3 != nil { in, out := &in.S3, &out.S3 *out = new(S3StorageProvider) - **out = **in + (*in).DeepCopyInto(*out) } if in.Gcs != nil { in, out := &in.Gcs, &out.Gcs