Merge pull request pingcap#4 from pingcap/rate-limit

backup: support rate limit

cmd/raw.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"github.com/pingcap/errors"
-	"github.com/pingcap/tidb/util/codec"
 	"github.com/spf13/cobra"
 )
 
@@ -15,8 +14,9 @@ func NewBackupCommand() *cobra.Command {
 	bp.AddCommand(
 		newFullBackupCommand(),
 		newTableBackupCommand(),
-		// newRegionCommand(),
 	)
+	bp.PersistentFlags().Uint64P(
+		"ratelimit", "", 0, "The rate limit of the backup task, MB/s per node")
 	return bp
 }
 
@@ -38,67 +38,17 @@ func newFullBackupCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			err = client.BackupRange([]byte(""), []byte(""), u, backupTS)
+			rate, err := command.Flags().GetUint64("ratelimit")
 			if err != nil {
 				return err
 			}
-			return client.SaveBackupMeta(u)
-		},
-	}
-	return command
-}
-
-// newRegionCommand return a backup region subcommand.
-func newRegionCommand() *cobra.Command {
-	command := &cobra.Command{
-		Use:   "region [flags]",
-		Short: "backup specified regions",
-		RunE: func(command *cobra.Command, _ []string) error {
-			client := GetDefaultRawClient()
-			regionID, err := command.Flags().GetUint64("region")
-			if err != nil {
-				return err
-			}
-			u, err := command.Flags().GetString("storage")
-			if err != nil {
-				return err
-			}
-			useRaw, err := command.Flags().GetBool("raw")
-			if err != nil {
-				return err
-			}
-			backer := GetDefaultBacker()
-			region, _, err := backer.GetPDClient().GetRegionByID(backer.Context(), regionID)
-			if err != nil {
-				return err
-			}
-			startKey := region.GetStartKey()
-			endKey := region.GetEndKey()
-			if !useRaw {
-				startKey, _, err = codec.DecodeBytes(startKey, nil)
-				if err != nil {
-					return err
-				}
-				endKey, _, err = codec.DecodeBytes(endKey, nil)
-				if err != nil {
-					return err
-				}
-			}
-			backupTS, err := client.GetTS()
-			if err != nil {
-				return err
-			}
-			err = client.BackupRange(startKey, endKey, u, backupTS)
+			err = client.BackupRange([]byte(""), []byte(""), u, backupTS, rate)
 			if err != nil {
 				return err
 			}
 			return client.SaveBackupMeta(u)
 		},
 	}
-	command.Flags().BoolP("raw", "raw", false,
-		"backup region with decode region keys")
-	command.Flags().Uint64P("region", "r", 0, "backup the specific region")
-	command.MarkFlagRequired("region")
 	return command
 }
 
@@ -135,7 +85,11 @@ func newTableBackupCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			err = client.BackupTable(db, table, u, backupTS)
+			rate, err := command.Flags().GetUint64("ratelimit")
+			if err != nil {
+				return err
+			}
+			err = client.BackupTable(db, table, u, backupTS, rate)
 			if err != nil {
 				return err
 			}

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gogo/protobuf v1.2.1
-	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/google/btree v1.0.0
 	github.com/gorilla/mux v1.7.3 // indirect
@@ -22,11 +21,11 @@ require (
 	github.com/pingcap/errcode v0.3.0 // indirect
 	github.com/pingcap/errors v0.11.4
 	github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81 // indirect
-	github.com/pingcap/kvproto v0.0.0-20190822083357-8a2d1a3ccb10
+	github.com/pingcap/kvproto v0.0.0-20190910074707-5a11ba1a2674
 	github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
-	github.com/pingcap/parser v0.0.0-20190822024127-41d48df05864
+	github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e
 	github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823
-	github.com/pingcap/tidb v0.0.0-20190822044546-9312090275b2
+	github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4
 	github.com/prometheus/client_golang v1.1.0
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
@@ -42,13 +41,6 @@ require (
 	go.etcd.io/etcd v3.3.13+incompatible // indirect
 	go.uber.org/atomic v1.4.0 // indirect
 	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
-	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
-	golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/appengine v1.6.1 // indirect
-	google.golang.org/genproto v0.0.0-20190817000702-55e96fffbd48 // indirect
 	google.golang.org/grpc v1.23.0
 )
-
-// replace github.com/pingcap/kvproto => /home/stn/kvproto