Merge branch 'master' into issue#25842

Makefile

@@ -45,8 +45,8 @@ check-static: tools/bin/golangci-lint
 	tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES))
 
 unconvert:tools/bin/unconvert
-	@echo "unconvert check"
-	@GO111MODULE=on tools/bin/unconvert ./...
+	@echo "unconvert check(skip check the genenrated or copied code in lightning)"
+	@GO111MODULE=on tools/bin/unconvert $(UNCONVERT_PACKAGES)
 
 gogenerate:
 	@echo "go generate ./..."
@@ -81,7 +81,9 @@ test: test_part_1 test_part_2
 
 test_part_1: checklist explaintest
 
-test_part_2: gotest gogenerate
+test_part_2: gotest gogenerate br_unit_test
+
+test_part_br: br_unit_test br_integration_test
 
 explaintest: server_check
 	@cd cmd/explaintest && ./run-tests.sh -s ../../bin/tidb-server
@@ -103,7 +105,7 @@ ifeq ("$(TRAVIS_COVERAGE)", "1")
 	@export log_level=info; \
 	$(OVERALLS) -project=github.com/pingcap/tidb \
 			-covermode=count \
-			-ignore='.git,vendor,cmd,docs,tests,LICENSES' \
+			-ignore='.git,br,vendor,cmd,docs,tests,LICENSES' \
 			-concurrency=4 \
 			-- -coverpkg=./... \
 			|| { $(FAILPOINT_DISABLE); exit 1; }
@@ -204,6 +206,10 @@ tools/bin/errdoc-gen: tools/check/go.mod
 tools/bin/golangci-lint:
 	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.41.1
 
+tools/bin/vfsgendev: tools/check/go.mod
+	cd tools/check; \
+	$(GO) build -o ../bin/vfsgendev github.com/shurcooL/vfsgen/cmd/vfsgendev
+
 # Usage:
 #
 # 	$ make vectorized-bench VB_FILE=Time VB_FUNC=builtinCurrentDateSig
@@ -230,3 +236,97 @@ endif
 bench-daily:
 	cd ./session && \
 	go test -run TestBenchDaily --date `git log -n1 --date=unix --pretty=format:%cd` --commit `git log -n1 --pretty=format:%h` --outfile $(TO)
+
+build_tools: build_br build_lightning build_lightning-ctl
+
+br_web:
+	@cd br/web && npm install && npm run build
+
+build_br:
+	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(TOOL_LDFLAGS) $(CHECK_FLAG)' -o $(BR_BIN) br/cmd/br/*.go
+
+build_lightning_for_web:
+	CGO_ENABLED=1 $(GOBUILD) -tags dev $(RACE_FLAG) -ldflags '$(TOOL_LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_BIN) br/cmd/tidb-lightning/main.go
+
+build_lightning:
+	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(TOOL_LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_BIN) br/cmd/tidb-lightning/main.go
+
+build_lightning-ctl:
+	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(TOOL_LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_CTL_BIN) br/cmd/tidb-lightning-ctl/main.go
+
+build_for_br_integration_test:
+	@make failpoint-enable
+	($(GOTEST) -c -cover -covermode=count \
+		-coverpkg=github.com/pingcap/tidb/br/... \
+		-o $(BR_BIN).test \
+		github.com/pingcap/tidb/br/cmd/br && \
+	$(GOTEST) -c -cover -covermode=count \
+		-coverpkg=github.com/pingcap/tidb/br/... \
+		-o $(LIGHTNING_BIN).test \
+		github.com/pingcap/tidb/br/cmd/tidb-lightning && \
+	$(GOTEST) -c -cover -covermode=count \
+		-coverpkg=github.com/pingcap/tidb/br/... \
+		-o $(LIGHTNING_CTL_BIN).test \
+		github.com/pingcap/tidb/br/cmd/tidb-lightning-ctl && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/locker br/tests/br_key_locked/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/gc br/tests/br_z_gc_safepoint/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/oauth br/tests/br_gcs/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/rawkv br/tests/br_rawkv/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/parquet_gen br/tests/lightning_checkpoint_parquet/*.go \
+	) || (make failpoint-disable && exit 1)
+	@make failpoint-disable
+
+br_unit_test: export ARGS=$$($(BR_PACKAGES))
+br_unit_test:
+	@make failpoint-enable
+	$(GOTEST) $(RACE_FLAG) -tags leak $(ARGS) || ( make failpoint-disable && exit 1 )
+	@make failpoint-disable
+
+br_integration_test: br_bins build_br build_for_br_integration_test
+	@cd br && tests/run.sh
+
+br_compatibility_test_prepare:
+	@cd br && tests/run_compatible.sh prepare
+
+br_compatibility_test:
+	@cd br && tests/run_compatible.sh run
+
+# There is no FreeBSD environment for GitHub actions. So cross-compile on Linux
+# but that doesn't work with CGO_ENABLED=1, so disable cgo. The reason to have
+# cgo enabled on regular builds is performance.
+ifeq ("$(GOOS)", "freebsd")
+        GOBUILD  = CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '$(LDFLAGS)'
+endif
+
+br_coverage:
+	tools/bin/gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
+ifeq ("$(JenkinsCI)", "1")
+	tools/bin/goveralls -coverprofile=$(TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
+else
+	go tool cover -html "$(TEST_DIR)/all_cov.out" -o "$(TEST_DIR)/all_cov.html"
+	grep -F '<option' "$(TEST_DIR)/all_cov.html"
+endif
+
+# TODO: adjust bins when br integraion tests reformat.
+br_bins:
+	@which bin/tidb-server
+	@which bin/tikv-server
+	@which bin/pd-server
+	@which bin/pd-ctl
+	@which bin/go-ycsb
+	@which bin/minio
+	@which bin/tiflash
+	@which bin/libtiflash_proxy.so
+	@which bin/cdc
+	@which bin/fake-gcs-server
+	@which bin/tikv-importer
+	if [ ! -d bin/flash_cluster_manager ]; then echo "flash_cluster_manager not exist"; exit 1; fi
+
+%_generated.go: %.rl
+	ragel -Z -G2 -o tmp_parser.go $<
+	@echo '// Code generated by ragel DO NOT EDIT.' | cat - tmp_parser.go | sed 's|//line |//.... |g' > $@
+	@rm tmp_parser.go
+
+data_parsers: tools/bin/vfsgendev br/pkg/lightning/mydump/parser_generated.go br_web
+	PATH="$(GOPATH)/bin":"$(PATH)":"$(TOOLS)" protoc -I. -I"$(GOPATH)/src" br/pkg/lightning/checkpoints/checkpointspb/file_checkpoints.proto --gogofaster_out=.
+	tools/bin/vfsgendev -source='"github.com/pingcap/tidb/br/pkg/lightning/web".Res' && mv res_vfsdata.go br/pkg/lightning/web/

Makefile.common

@@ -43,10 +43,12 @@ endif
 ARCH      := "`uname -s`"
 LINUX     := "Linux"
 MAC       := "Darwin"
-PACKAGE_LIST  := go list ./...| grep -vE "cmd|github.com\/pingcap\/tidb\/tests"
+PACKAGE_LIST  := go list ./...| grep -vE "cmd|github.com\/pingcap\/tidb\/tests|github.com\/pingcap\/tidb\/br"
 PACKAGES  ?= $$($(PACKAGE_LIST))
 PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||'
 FILES     := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go")
+UNCONVERT_PACKAGES_LIST := go list ./...| grep -vE "lightning\/checkpoints|lightning\/manual|lightning\/common"
+UNCONVERT_PACKAGES := $$($(UNCONVERT_PACKAGES_LIST))
 
 FAILPOINT_ENABLE  := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs tools/bin/failpoint-ctl enable)
 FAILPOINT_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs tools/bin/failpoint-ctl disable)
@@ -78,3 +80,31 @@ CHECK_FLAG =
 ifeq ("$(WITH_CHECK)", "1")
 	CHECK_FLAG = $(TEST_LDFLAGS)
 endif
+
+BR_PKG := github.com/pingcap/tidb/br
+BR_PACKAGES       := go list ./...| grep "github.com\/pingcap\/tidb\/br"
+LIGHTNING_BIN     := bin/tidb-lightning
+LIGHTNING_CTL_BIN := bin/tidb-lightning-ctl
+BR_BIN            := bin/br
+TEST_DIR          := /tmp/backup_restore_test
+
+TOOL_RELEASE_VERSION =
+ifeq ($(TOOL_RELEASE_VERSION),)
+        TOOL_RELEASE_VERSION := v4.0.0-dev
+        release_version_regex := ^v4\..*$$
+        release_branch_regex := "^release-[0-9]\.[0-9].*$$|^HEAD$$|^.*/*tags/v[0-9]\.[0-9]\..*$$"
+        ifneq ($(shell git rev-parse --abbrev-ref HEAD | egrep $(release_branch_regex)),)
+                # If we are in release branch, try to use tag version.
+                ifneq ($(shell git describe --tags --dirty | egrep $(release_version_regex)),)
+                        TOOL_RELEASE_VERSION := $(shell git describe --tags --dirty)
+                endif
+        else ifneq ($(shell git status --porcelain),)
+                # Add -dirty if the working tree is dirty for non release branch.
+                TOOL_RELEASE_VERSION := $(TOOL_RELEASE_VERSION)-dirty
+        endif
+endif
+
+TOOL_LDFLAGS += -X "$(BR_PKG)/pkg/version/build.ReleaseVersion=$(TOOL_RELEASE_VERSION)"
+TOOL_LDFLAGS += -X "$(BR_PKG)/pkg/version/build.BuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
+TOOL_LDFLAGS += -X "$(BR_PKG)/pkg/version/build.GitHash=$(shell git rev-parse HEAD)"
+TOOL_LDFLAGS += -X "$(BR_PKG)/pkg/version/build.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)"

bindinfo/bind_test.go

@@ -2039,10 +2039,9 @@ func (s *testSuite) TestExplainShowBindSQL(c *C) {
 		"select * from `test` . `t` SELECT * FROM `test`.`t` USE INDEX (`a`)",
 	))
 
-	tk.MustExec("explain select * from t")
-	tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Using the bindSQL: SELECT * FROM `test`.`t` USE INDEX (`a`)"))
-	tk.MustExec("explain analyze select * from t")
-	tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Using the bindSQL: SELECT * FROM `test`.`t` USE INDEX (`a`)"))
+	tk.MustExec("explain format = 'verbose' select * from t")
+	tk.MustQuery("show warnings").Check(testkit.Rows("Note 1105 Using the bindSQL: SELECT * FROM `test`.`t` USE INDEX (`a`)"))
+	// explain analyze do not support verbose yet.
 }
 
 func (s *testSuite) TestDMLIndexHintBind(c *C) {

bindinfo/handle.go

@@ -699,7 +699,13 @@ func getHintsForSQL(sctx sessionctx.Context, sql string) (string, error) {
 	rs, err := sctx.(sqlexec.SQLExecutor).ExecuteInternal(context.TODO(), fmt.Sprintf("EXPLAIN FORMAT='hint' %s", sql))
 	sctx.GetSessionVars().UsePlanBaselines = origVals
 	if rs != nil {
-		defer terror.Call(rs.Close)
+		defer func() {
+			// Audit log is collected in Close(), set InRestrictedSQL to avoid 'create sql binding' been recorded as 'explain'.
+			origin := sctx.GetSessionVars().InRestrictedSQL
+			sctx.GetSessionVars().InRestrictedSQL = true
+			terror.Call(rs.Close)
+			sctx.GetSessionVars().InRestrictedSQL = origin
+		}()
 	}
 	if err != nil {
 		return "", err

br/.codecov.yml

@@ -0,0 +1,22 @@
+codecov:
+  require_ci_to_pass: yes
+
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: false  # if true: only post the comment if coverage changes
+  require_base: no        # [yes :: must have a base report to post]
+  require_head: yes       # [yes :: must have a head report to post]
+  branches:               # branch names that can post comment
+    - "master"
+
+coverage:
+  status:
+    project:
+      default:
+        # Allow the coverage to drop by 3%
+        target: 85%
+        threshold: 3%
+        branches:
+          - master
+    patch: off

br/.dockerignore

@@ -0,0 +1 @@
+.gitignore

br/.editorconfig

@@ -0,0 +1,10 @@
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+
+# tab_size = 4 spaces
+[*.go]
+indent_style = tab
+indent_size = 4
+trim_trailing_whitespace = true

br/.gitattributes

@@ -0,0 +1 @@
+*_generated.go linguist-generated=true

br/.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,45 @@
+---
+name: "🐛 Bug Report"
+about: Something isn't working as expected
+title: ''
+labels: 'type/bug '
+---
+
+Please answer these questions before submitting your issue. Thanks!
+
+1. What did you do?
+If possible, provide a recipe for reproducing the error.
+
+
+2. What did you expect to see?
+
+
+
+3. What did you see instead?
+
+
+
+4. What version of BR and TiDB/TiKV/PD are you using?
+
+<!--
+br -V
+tidb-lightning -V
+tidb-server -V
+tikv-server -V
+pd-server -V
+-->
+
+5. Operation logs
+   - Please upload `br.log` for BR if possible
+   - Please upload `tidb-lightning.log` for TiDB-Lightning if possible
+   - Please upload `tikv-importer.log` from TiKV-Importer if possible
+   - Other interesting logs
+
+
+6. Configuration of the cluster and the task
+   - `tidb-lightning.toml` for TiDB-Lightning if possible
+   - `tikv-importer.toml` for TiKV-Importer if possible
+   - `topology.yml` if deployed by TiUP
+
+
+7. Screenshot/exported-PDF of Grafana dashboard or metrics' graph in Prometheus if possible

br/.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,19 @@
+---
+name: "🚀 Feature Request"
+about: I have a suggestion
+labels: 'type/feature-request'
+---
+
+## Feature Request
+
+### Describe your feature request related problem:
+<!-- A description of what the problem is. -->
+
+### Describe the feature you'd like:
+<!-- A description of what you want to happen. -->
+
+### Describe alternatives you've considered:
+<!-- A description of any alternative solutions or features you've considered. -->
+
+### Teachability, Documentation, Adoption, Migration Strategy:
+<!-- If you can, explain some scenarios how users might use this, or situations in which it would be helpful. Any API designs, mockups, or diagrams are also helpful. -->

br/.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,24 @@
+---
+name: "\U0001F914 Question"
+labels: "type/question"
+about: Usage question that isn't answered in docs or discussion
+
+---
+
+## Question
+
+
+<!--
+
+Before asking a question, make sure you have:
+
+- Searched existing Stack Overflow questions.
+- Googled your question.
+- Searched open and closed [GitHub issues](https://github.com/pingcap/br/issues?q=is%3Aissue)
+- Read the documentation:
+    * [BR documentation](https://docs.pingcap.com/tidb/stable/backup-and-restore-tool)
+    * [Lightning documentation](https://docs.pingcap.com/tidb/stable/tidb-lightning-overview)
+    * [BR 中文文档](https://docs.pingcap.com/zh/tidb/stable/backup-and-restore-tool)
+    * [Lightning 中文文档](https://docs.pingcap.com/zh/tidb/stable/tidb-lightning-overview)
+
+-->

br/.github/challenge-bot.yml

@@ -0,0 +1 @@
+defaultSigLabel: sig/migrate

br/.github/pull_request_template.md

@@ -0,0 +1,42 @@
+<!--
+Thank you for working on BR! Please read BR's [CONTRIBUTING](https://github.com/pingcap/br/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
+-->
+
+### What problem does this PR solve? <!--add issue link with summary if exists-->
+
+
+### What is changed and how it works?
+
+
+### Check List <!--REMOVE the items that are not applicable-->
+
+Tests <!-- At least one of them must be included. -->
+
+ - Unit test
+ - Integration test
+ - Manual test (add detailed scripts or steps below)
+ - No code
+
+Code changes
+
+ - Has exported function/method change
+ - Has exported variable/fields change
+ - Has interface methods change
+ - Has persistent data change
+
+Side effects
+
+ - Possible performance regression
+ - Increased code complexity
+ - Breaking backward compatibility
+
+Related changes
+
+ - Need to cherry-pick to the release branch
+ - Need to update the documentation
+
+### Release note
+
+ -
+
+<!-- fill in the release note, or just write "No release note" -->