From ff4cde81fdad0d3f71baa944df43805e901a9700 Mon Sep 17 00:00:00 2001 From: DanielZhangQD Date: Thu, 16 Apr 2020 15:08:23 +0800 Subject: [PATCH 1/2] fix br log issue and include both 3.1 and 4.0 br in the tidb-backup-manager image --- cmd/backup-manager/app/backup/backup.go | 32 +++++++++++++++------- cmd/backup-manager/app/cmd/backup.go | 1 + cmd/backup-manager/app/cmd/restore.go | 1 + cmd/backup-manager/app/restore/restore.go | 33 ++++++++++++++++------- cmd/backup-manager/app/util/generic.go | 1 + cmd/backup-manager/app/util/util.go | 27 ++++++++++++++++++- go.mod | 1 + go.sum | 1 + images/tidb-backup-manager/Dockerfile | 32 +++++++++++++++------- pkg/backup/backup/backup_manager.go | 15 +++++++++++ pkg/backup/restore/restore_manager.go | 15 +++++++++++ 11 files changed, 128 insertions(+), 31 deletions(-) diff --git a/cmd/backup-manager/app/backup/backup.go b/cmd/backup-manager/app/backup/backup.go index 73c9de56b0..8e3b3748cc 100644 --- a/cmd/backup-manager/app/backup/backup.go +++ b/cmd/backup-manager/app/backup/backup.go @@ -14,9 +14,11 @@ package backup import ( + "bufio" "context" "fmt" "io" + "io/ioutil" "os/exec" "path" "strings" @@ -64,29 +66,39 @@ func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { } fullArgs = append(fullArgs, args...) klog.Infof("Running br command with args: %v", fullArgs) - cmd := exec.Command("br", fullArgs...) - cmd.Stderr = cmd.Stdout + bin := "br" + backupUtil.Suffix(bo.TiKVVersion) + cmd := exec.Command(bin, fullArgs...) + stdOut, err := cmd.StdoutPipe() if err != nil { return remotePath, fmt.Errorf("cluster %s, create stdout pipe failed, err: %v", bo, err) } + stdErr, err := cmd.StderrPipe() + if err != nil { + return remotePath, fmt.Errorf("cluster %s, create stderr pipe failed, err: %v", bo, err) + } err = cmd.Start() if err != nil { return remotePath, fmt.Errorf("cluster %s, execute br command failed, args: %s, err: %v", bo, fullArgs, err) } - var tmpOutput, errMsg string + var errMsg string + reader := bufio.NewReader(stdOut) for { - tmp := make([]byte, 1024) - _, err := stdOut.Read(tmp) - tmpOutput = string(tmp) - if strings.Contains(tmpOutput, "[ERROR]") { - errMsg += tmpOutput + line, err := reader.ReadString('\n') + if strings.Contains(line, "[ERROR]") { + errMsg += line } - klog.Infof(strings.Replace(tmpOutput, "\n", "", -1)) - if err != nil { + + klog.Infof(strings.Replace(line, "\n", "", -1)) + if err != nil || io.EOF == err { break } } + tmpErr, _ := ioutil.ReadAll(stdErr) + if len(tmpErr) > 0 { + klog.Infof(string(tmpErr)) + errMsg += string(tmpErr) + } err = cmd.Wait() if err != nil { return remotePath, fmt.Errorf("cluster %s, wait pipe message failed, errMsg %s, err: %v", bo, errMsg, err) diff --git a/cmd/backup-manager/app/cmd/backup.go b/cmd/backup-manager/app/cmd/backup.go index 57df7bf0d2..5c7bc86b21 100644 --- a/cmd/backup-manager/app/cmd/backup.go +++ b/cmd/backup-manager/app/cmd/backup.go @@ -42,6 +42,7 @@ func NewBackupCommand() *cobra.Command { cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace") cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name") + cmd.Flags().StringVar(&bo.TiKVVersion, "tikvVersion", util.DefaultVersion, "TiKV version") cmd.Flags().BoolVar(&bo.TLSClient, "client-tls", false, "Whether client tls is enabled") cmd.Flags().BoolVar(&bo.TLSCluster, "cluster-tls", false, "Whether cluster tls is enabled") return cmd diff --git a/cmd/backup-manager/app/cmd/restore.go b/cmd/backup-manager/app/cmd/restore.go index dd46b1dce3..fa383c2e5c 100644 --- a/cmd/backup-manager/app/cmd/restore.go +++ b/cmd/backup-manager/app/cmd/restore.go @@ -44,6 +44,7 @@ func NewRestoreCommand() *cobra.Command { cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace") cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name") + cmd.Flags().StringVar(&ro.TiKVVersion, "tikvVersion", util.DefaultVersion, "TiKV version") cmd.Flags().BoolVar(&ro.TLSClient, "client-tls", false, "Whether client tls is enabled") cmd.Flags().BoolVar(&ro.TLSCluster, "cluster-tls", false, "Whether cluster tls is enabled") return cmd diff --git a/cmd/backup-manager/app/restore/restore.go b/cmd/backup-manager/app/restore/restore.go index 2bee909171..ff79167a49 100644 --- a/cmd/backup-manager/app/restore/restore.go +++ b/cmd/backup-manager/app/restore/restore.go @@ -14,7 +14,10 @@ package restore import ( + "bufio" "fmt" + "io" + "io/ioutil" "os/exec" "path" "strings" @@ -58,29 +61,39 @@ func (ro *Options) restoreData(restore *v1alpha1.Restore) error { } fullArgs = append(fullArgs, args...) klog.Infof("Running br command with args: %v", fullArgs) - cmd := exec.Command("br", fullArgs...) - cmd.Stderr = cmd.Stdout + bin := "br" + backupUtil.Suffix(ro.TiKVVersion) + cmd := exec.Command(bin, fullArgs...) + stdOut, err := cmd.StdoutPipe() if err != nil { return fmt.Errorf("cluster %s, create stdout pipe failed, err: %v", ro, err) } + stdErr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("cluster %s, create stderr pipe failed, err: %v", ro, err) + } err = cmd.Start() if err != nil { return fmt.Errorf("cluster %s, execute br command failed, args: %s, err: %v", ro, fullArgs, err) } - var tmpOutput, errMsg string + var errMsg string + reader := bufio.NewReader(stdOut) for { - tmp := make([]byte, 1024) - _, err := stdOut.Read(tmp) - tmpOutput = string(tmp) - if strings.Contains(tmpOutput, "[ERROR]") { - errMsg += tmpOutput + line, err := reader.ReadString('\n') + if strings.Contains(line, "[ERROR]") { + errMsg += line } - klog.Infof(strings.Replace(tmpOutput, "\n", "", -1)) - if err != nil { + klog.Infof(strings.Replace(line, "\n", "", -1)) + if err != nil || io.EOF == err { break } } + tmpErr, _ := ioutil.ReadAll(stdErr) + if len(tmpErr) > 0 { + klog.Infof(string(tmpErr)) + errMsg += string(tmpErr) + } + err = cmd.Wait() if err != nil { return fmt.Errorf("cluster %s, wait pipe message failed, errMsg %s, err: %v", ro, errMsg, err) diff --git a/cmd/backup-manager/app/util/generic.go b/cmd/backup-manager/app/util/generic.go index 41505d88a3..54f49d45bd 100644 --- a/cmd/backup-manager/app/util/generic.go +++ b/cmd/backup-manager/app/util/generic.go @@ -39,6 +39,7 @@ type GenericOptions struct { Port int32 Password string User string + TiKVVersion string } func (bo *GenericOptions) String() string { diff --git a/cmd/backup-manager/app/util/util.go b/cmd/backup-manager/app/util/util.go index ca5723c1d4..804cedb805 100644 --- a/cmd/backup-manager/app/util/util.go +++ b/cmd/backup-manager/app/util/util.go @@ -19,14 +19,22 @@ import ( "os" "strings" + "github.com/Masterminds/semver" "github.com/spf13/pflag" + "k8s.io/klog" cmdutil "k8s.io/kubectl/pkg/cmd/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" ) var ( - cmdHelpMsg string + cmdHelpMsg string + supportedVersions = map[string]struct{}{ + "3.1": struct{}{}, + "4.0": struct{}{}, + } + // DefaultVersion is the default tikv and br version + DefaultVersion = "4.0" ) func validCmdFlagFunc(flag *pflag.Flag) { @@ -162,3 +170,20 @@ func constructBRGlobalOptions(config *v1alpha1.BRConfig) []string { } return args } + +// Suffix parses the major and minor version from the string and return the suffix +func Suffix(version string) string { + numS := strings.Split(DefaultVersion, ".") + defaultSuffix := numS[0] + numS[1] + + v, err := semver.NewVersion(version) + if err != nil { + klog.Errorf("Parse version %s failure, error: %v", version, err) + return defaultSuffix + } + parsed := fmt.Sprintf("%d.%d", v.Major(), v.Minor()) + if _, ok := supportedVersions[parsed]; ok { + return fmt.Sprintf("%d%d", v.Major(), v.Minor()) + } + return defaultSuffix +} diff --git a/go.mod b/go.mod index 8153d820a0..0968f9aa4b 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Azure/go-autorest/autorest/mocks v0.3.0 // indirect github.com/BurntSushi/toml v0.3.1 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect + github.com/Masterminds/semver v1.4.2 github.com/Microsoft/go-winio v0.4.12 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2 diff --git a/go.sum b/go.sum index 397fda71e7..383bfe3b6d 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,7 @@ github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e h1:eb0Pzkt15Bm7f2FFYv7sjY7NPFi3cPkS3tv1CcrFBWA= github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc= diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index f69b0e6573..6b29976a56 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -2,19 +2,15 @@ FROM pingcap/tidb-enterprise-tools:latest ARG VERSION=v1.51.0 ARG SHUSH_VERSION=v1.4.0 ARG TOOLKIT_VERSION=v3.0.12 +ARG TOOLKIT_V31=v3.1.0-rc +ARG TOOLKIT_V40=v4.0.0-rc RUN apk update && apk add ca-certificates RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \ - && unzip rclone-${VERSION}-linux-amd64.zip \ - && mv rclone-${VERSION}-linux-amd64/rclone /usr/local/bin \ - && chmod 755 /usr/local/bin/rclone \ - && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 - -RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ - && tar -xzf br-latest-linux-amd64.tar.gz \ - && mv bin/br /usr/local/bin \ - && chmod 755 /usr/local/bin/br \ - && rm -rf br-latest-linux-amd64.tar.gz + && unzip rclone-${VERSION}-linux-amd64.zip \ + && mv rclone-${VERSION}-linux-amd64/rclone /usr/local/bin \ + && chmod 755 /usr/local/bin/rclone \ + && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \ && mv shush_linux_amd64 /usr/local/bin/shush \ @@ -29,6 +25,22 @@ RUN \ && rm -rf tidb-toolkit-${TOOLKIT_VERSION}-linux-amd64.tar.gz \ && rm -rf tidb-toolkit-${TOOLKIT_VERSION}-linux-amd64 +RUN \ + wget -nv https://download.pingcap.org/tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \ + && tar -xzf tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \ + && mv tidb-toolkit-${TOOLKIT_V31}-linux-amd64/bin/br /usr/local/bin/br31 \ + && chmod 755 /usr/local/bin/br31 \ + && rm -rf tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \ + && rm -rf tidb-toolkit-${TOOLKIT_V31}-linux-amd64 + +RUN \ + wget -nv https://download.pingcap.org/tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \ + && tar -xzf tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \ + && mv tidb-toolkit-${TOOLKIT_V40}-linux-amd64/bin/br /usr/local/bin/br40 \ + && chmod 755 /usr/local/bin/br40 \ + && rm -rf tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \ + && rm -rf tidb-toolkit-${TOOLKIT_V40}-linux-amd64 + COPY bin/tidb-backup-manager /tidb-backup-manager COPY entrypoint.sh /entrypoint.sh diff --git a/pkg/backup/backup/backup_manager.go b/pkg/backup/backup/backup_manager.go index 42dacb7936..56b45ba7f8 100644 --- a/pkg/backup/backup/backup_manager.go +++ b/pkg/backup/backup/backup_manager.go @@ -15,6 +15,7 @@ package backup import ( "fmt" + "strings" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/backup" @@ -269,6 +270,13 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s return nil, fmt.Sprintf("failed to fetch tidbcluster %s/%s", backupNamespace, backup.Spec.BR.Cluster), err } + var tikvVersion string + tikvImage := tc.TiKVImage() + imageVersion := strings.Split(tikvImage, ":") + if len(imageVersion) == 2 { + tikvVersion = imageVersion[1] + } + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.kubeCli) if err != nil { return nil, reason, err @@ -280,12 +288,19 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s } envVars = append(envVars, storageEnv...) + envVars = append(envVars, corev1.EnvVar{ + Name: "BR_LOG_TO_TERM", + Value: string(1), + }) args := []string{ "backup", fmt.Sprintf("--namespace=%s", ns), fmt.Sprintf("--backupName=%s", name), } + if tikvVersion != "" { + args = append(args, fmt.Sprintf("--tikvVersion=%s", tikvVersion)) + } backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).BackupJob().Backup(name) volumeMounts := []corev1.VolumeMount{} diff --git a/pkg/backup/restore/restore_manager.go b/pkg/backup/restore/restore_manager.go index 17018feae2..ceef74cbc6 100644 --- a/pkg/backup/restore/restore_manager.go +++ b/pkg/backup/restore/restore_manager.go @@ -15,6 +15,7 @@ package restore import ( "fmt" + "strings" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/backup" @@ -254,6 +255,13 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo return nil, fmt.Sprintf("failed to fetch tidbcluster %s/%s", restoreNamespace, restore.Spec.BR.Cluster), err } + var tikvVersion string + tikvImage := tc.TiKVImage() + imageVersion := strings.Split(tikvImage, ":") + if len(imageVersion) == 2 { + tikvVersion = imageVersion[1] + } + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.kubeCli) if err != nil { return nil, reason, err @@ -265,11 +273,18 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo } envVars = append(envVars, storageEnv...) + envVars = append(envVars, corev1.EnvVar{ + Name: "BR_LOG_TO_TERM", + Value: string(1), + }) args := []string{ "restore", fmt.Sprintf("--namespace=%s", ns), fmt.Sprintf("--restoreName=%s", name), } + if tikvVersion != "" { + args = append(args, fmt.Sprintf("--tikvVersion=%s", tikvVersion)) + } restoreLabel := label.NewBackup().Instance(restore.GetInstanceName()).RestoreJob().Restore(name) volumeMounts := []corev1.VolumeMount{} From 033cb39e55f74f9a0649272f1b851deb1743c8da Mon Sep 17 00:00:00 2001 From: DanielZhangQD Date: Thu, 16 Apr 2020 15:31:37 +0800 Subject: [PATCH 2/2] fix ci --- cmd/backup-manager/app/util/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/backup-manager/app/util/util.go b/cmd/backup-manager/app/util/util.go index 804cedb805..89cffee275 100644 --- a/cmd/backup-manager/app/util/util.go +++ b/cmd/backup-manager/app/util/util.go @@ -30,8 +30,8 @@ import ( var ( cmdHelpMsg string supportedVersions = map[string]struct{}{ - "3.1": struct{}{}, - "4.0": struct{}{}, + "3.1": {}, + "4.0": {}, } // DefaultVersion is the default tikv and br version DefaultVersion = "4.0"