Merge master to release 3.1

.codecov.yml

@@ -0,0 +1,10 @@
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        # Allow the coverage to drop by 3%
+        threshold: 3%
+    patch: off

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.gitignore

@@ -7,3 +7,5 @@ backupmeta
 *.ngo
 *.coverprofile
 coverage.txt
+docker/data/
+docker/logs/

LICENSE.md

@@ -186,7 +186,7 @@ Apache License
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@ Apache License
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

Makefile

@@ -12,25 +12,28 @@ LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S
 LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRGitHash=$(shell git rev-parse HEAD)"
 LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRGitBranch=$(shell git rev-parse --abbrev-ref HEAD)"
 
-all: check test build
+ifeq ("$(WITH_RACE)", "1")
+	RACEFLAG = -race
+endif
 
-release:
-	GO111MODULE=on go build -ldflags '$(LDFLAGS)' -o bin/br
+all: check test build
 
 build:
-	GO111MODULE=on go build -ldflags '$(LDFLAGS)' -race -o bin/br
+	GO111MODULE=on go build -ldflags '$(LDFLAGS)' ${RACEFLAG} -o bin/br
 
 build_for_integration_test:
 	GO111MODULE=on go test -c -cover -covermode=count \
 		-coverpkg=$(BR_PKG)/... \
 		-o bin/br.test
 	# build key locker
-	GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go
+	GO111MODULE=on go build ${RACEFLAG} -o bin/locker tests/br_key_locked/*.go
 	# build gc
-	GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go
+	GO111MODULE=on go build ${RACEFLAG} -o bin/gc tests/br_z_gc_safepoint/*.go
+	# build rawkv client
+	GO111MODULE=on go build ${RACEFLAG} -o bin/rawkv tests/br_rawkv/*.go
 
 test:
-	GO111MODULE=on go test -race -tags leak ./...
+	GO111MODULE=on go test ${RACEFLAG} -tags leak ./...
 
 testcover:
 	GO111MODULE=on retool do overalls \
@@ -46,6 +49,7 @@ integration_test: build build_for_integration_test
 	@which bin/pd-server
 	@which bin/pd-ctl
 	@which bin/go-ycsb
+	@which bin/minio
 	@which bin/br
 	tests/run.sh
 
@@ -71,6 +75,12 @@ static:
 		--disable interfacer \
 		--disable goimports \
 		--disable gofmt \
+		--disable wsl \
+		--disable funlen \
+		--disable whitespace \
+		--disable gocognit \
+		--disable godox \
+		--disable gomnd \
 		$$($(PACKAGE_DIRECTORIES))
 
 lint:

README.md

@@ -36,6 +36,55 @@ Notice BR supports building with Go version `Go >= 1.13`
 
 When BR is built successfully, you can find binary in the `bin` directory.
 
+## Quick start
+
+```sh
+# Start TiDB cluster
+docker-compose -f docker-compose.yaml rm -s -v && \
+docker-compose -f docker-compose.yaml build && \
+docker-compose -f docker-compose.yaml up --remove-orphans
+
+# Attach to control container to run BR
+docker exec -it br_control_1 bash
+
+# Load testing data to TiDB
+go-ycsb load mysql -p workload=core \
+    -p mysql.host=tidb -p mysql.port=4000 -p mysql.user=root \
+    -p recordcount=100000 -p threadcount=100
+
+# How many rows do we get? 100000 rows.
+mysql -uroot -htidb -P4000 -E -e "SELECT COUNT(*) FROM test.usertable"
+
+# Build BR and backup!
+make build && \
+bin/br backup full --pd pd0:2379 --storage "local:///data/backup/full" \
+    --log-file "/logs/br_backup.log"
+
+# Let's drop database.
+mysql -uroot -htidb -P4000 -E -e "DROP DATABASE test; SHOW DATABASES;"
+
+# Restore!
+bin/br restore full --pd pd0:2379 --storage "local:///data/backup/full" \
+    --log-file "/logs/br_restore.log"
+
+# How many rows do we get again? Expected to be 100000 rows.
+mysql -uroot -htidb -P4000 -E -e "SELECT COUNT(*) FROM test.usertable"
+
+# Test S3 compatible storage (MinIO).
+# Create a bucket to save backup by mc (a MinIO Client).
+mc config host add minio $S3_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && \
+mc mb minio/mybucket
+
+# Backup to S3 compatible storage.
+bin/br backup full --pd pd0:2379 --storage "s3://mybucket/full" \
+    --s3.endpoint="$S3_ENDPOINT"
+
+# Drop database and restore!
+mysql -uroot -htidb -P4000 -E -e "DROP DATABASE test; SHOW DATABASES;" && \
+bin/br restore full --pd pd0:2379 --storage "s3://mybucket/full" \
+    --s3.endpoint="$S3_ENDPOINT"
+```
+
 ## Contributing
 
 Contributions are welcomed and greatly appreciated. See [CONTRIBUTING](./CONTRIBUTING.md)

cmd/backup.go

@@ -1,10 +1,13 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
 package cmd
 
 import (
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
 
+	"github.com/pingcap/br/pkg/gluetikv"
 	"github.com/pingcap/br/pkg/summary"
 	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
@@ -13,16 +16,27 @@ import (
 func runBackupCommand(command *cobra.Command, cmdName string) error {
 	cfg := task.BackupConfig{Config: task.Config{LogProgress: HasLogFile()}}
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		command.SilenceUsage = false
 		return err
 	}
-	return task.RunBackup(GetDefaultContext(), cmdName, &cfg)
+	return task.RunBackup(GetDefaultContext(), tidbGlue, cmdName, &cfg)
+}
+
+func runBackupRawCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.RawKvConfig{Config: task.Config{LogProgress: HasLogFile()}}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		command.SilenceUsage = false
+		return err
+	}
+	return task.RunBackupRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg)
 }
 
 // NewBackupCommand return a full backup subcommand.
 func NewBackupCommand() *cobra.Command {
 	command := &cobra.Command{
-		Use:   "backup",
-		Short: "backup a TiDB cluster",
+		Use:          "backup",
+		Short:        "backup a TiDB/TiKV cluster",
+		SilenceUsage: true,
 		PersistentPreRunE: func(c *cobra.Command, args []string) error {
 			if err := Init(c); err != nil {
 				return err
@@ -43,6 +57,7 @@ func NewBackupCommand() *cobra.Command {
 		newFullBackupCommand(),
 		newDbBackupCommand(),
 		newTableBackupCommand(),
+		newRawBackupCommand(),
 	)
 
 	task.DefineBackupFlags(command.PersistentFlags())
@@ -87,3 +102,18 @@ func newTableBackupCommand() *cobra.Command {
 	task.DefineTableFlags(command)
 	return command
 }
+
+// newRawBackupCommand return a raw kv range backup subcommand.
+func newRawBackupCommand() *cobra.Command {
+	// TODO: remove experimental tag if it's stable
+	command := &cobra.Command{
+		Use:   "raw",
+		Short: "(experimental) backup a raw kv range from TiKV cluster",
+		RunE: func(command *cobra.Command, _ []string) error {
+			return runBackupRawCommand(command, "Raw backup")
+		},
+	}
+
+	task.DefineRawBackupFlags(command)
+	return command
+}

cmd/cmd.go

@@ -1,26 +1,35 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
 package cmd
 
 import (
 	"context"
 	"net/http"
 	"net/http/pprof"
+	"os"
+	"path/filepath"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/util/logutil"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
+	"github.com/pingcap/br/pkg/gluetidb"
+	"github.com/pingcap/br/pkg/summary"
 	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
 )
 
 var (
-	initOnce       = sync.Once{}
-	defaultContext context.Context
-	hasLogFile     uint64
+	initOnce        = sync.Once{}
+	defaultContext  context.Context
+	hasLogFile      uint64
+	tidbGlue        = gluetidb.Glue{}
+	envLogToTermKey = "BR_LOG_TO_TERM"
 )
 
 const (
@@ -37,6 +46,10 @@ const (
 	flagVersionShort = "V"
 )
 
+func timestampLogFileName() string {
+	return filepath.Join(os.TempDir(), "br.log."+time.Now().Format(time.RFC3339))
+}
+
 // AddFlags adds flags to the given cmd.
 func AddFlags(cmd *cobra.Command) {
 	cmd.Version = utils.BRInfo()
@@ -45,8 +58,8 @@ func AddFlags(cmd *cobra.Command) {
 
 	cmd.PersistentFlags().StringP(FlagLogLevel, "L", "info",
 		"Set the log level")
-	cmd.PersistentFlags().String(FlagLogFile, "",
-		"Set the log file path. If not set, logs will output to stdout")
+	cmd.PersistentFlags().String(FlagLogFile, timestampLogFileName(),
+		"Set the log file path. If not set, logs will output to temp file")
 	cmd.PersistentFlags().String(FlagStatusAddr, "",
 		"Set the HTTP listening address for the status report service. Set to empty string to disable")
 	task.DefineCommonFlags(cmd.PersistentFlags())
@@ -69,8 +82,15 @@ func Init(cmd *cobra.Command) (err error) {
 		if err != nil {
 			return
 		}
+		_, outputLogToTerm := os.LookupEnv(envLogToTermKey)
+		if outputLogToTerm {
+			// Log to term if env `BR_LOG_TO_TERM` is set.
+			conf.File.Filename = ""
+		}
 		if len(conf.File.Filename) != 0 {
 			atomic.StoreUint64(&hasLogFile, 1)
+			summary.InitCollector(true)
+			cmd.Printf("Detial BR log in %s\n", conf.File.Filename)
 		}
 		lg, p, e := log.InitLogger(conf)
 		if e != nil {
@@ -84,16 +104,20 @@ func Init(cmd *cobra.Command) (err error) {
 			err = e
 			return
 		}
+		tidbLogCfg := logutil.LogConfig{}
 		if len(slowLogFilename) != 0 {
-			slowCfg := logutil.LogConfig{SlowQueryFile: slowLogFilename}
-			e = logutil.InitLogger(&slowCfg)
-			if e != nil {
-				err = e
-				return
-			}
+			tidbLogCfg.SlowQueryFile = slowLogFilename
 		} else {
 			// Hack! Discard slow log by setting log level to PanicLevel
 			logutil.SlowQueryLogger.SetLevel(logrus.PanicLevel)
+			// Disable annoying TiDB Log.
+			// TODO: some error logs outputs randomly, we need to fix them in TiDB.
+			tidbLogCfg.Level = "fatal"
+		}
+		e = logutil.InitLogger(&tidbLogCfg)
+		if e != nil {
+			err = e
+			return
 		}
 
 		// Initialize the pprof server.