Merge branch 'master' into issue#31022

Makefile

@@ -44,7 +44,7 @@ goword:tools/bin/goword
 	tools/bin/goword $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
 
 check-static: tools/bin/golangci-lint
-	GO111MODULE=on CGO_ENABLED=0 tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES_TIDB_TESTS)) --config .golangci.yml
+	GO111MODULE=on CGO_ENABLED=0 tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES)) --config .golangci.yml
 
 unconvert:tools/bin/unconvert
 	@echo "unconvert check(skip check the genenrated or copied code in lightning)"
@@ -64,7 +64,7 @@ lint:tools/bin/revive
 
 vet:
 	@echo "vet"
-	$(GO) vet -all $(PACKAGES_TIDB_TESTS_WITHOUT_BR) 2>&1 | $(FAIL_ON_STDOUT)
+	$(GO) vet -all $(PACKAGES_TIDB_TESTS) 2>&1 | $(FAIL_ON_STDOUT)
 
 tidy:
 	@echo "go mod tidy"
@@ -122,7 +122,7 @@ devgotest: failpoint-enable
 # - 'ok ' to ignore passed directories
 	@echo "Running in native mode."
 	@export log_level=info; export TZ='Asia/Shanghai'; \
-	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS_WITHOUT_BR) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
+	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 ut: failpoint-enable tools/bin/ut
@@ -132,7 +132,7 @@ ut: failpoint-enable tools/bin/ut
 gotest: failpoint-enable
 	@echo "Running in native mode."
 	@export log_level=info; export TZ='Asia/Shanghai'; \
-	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -timeout 20m -cover $(PACKAGES_TIDB_TESTS_WITHOUT_BR) -coverprofile=coverage.txt -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
+	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -timeout 20m -cover $(PACKAGES_TIDB_TESTS) -coverprofile=coverage.txt -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 gotest_in_verify_ci: failpoint-enable tools/bin/gotestsum
@@ -141,7 +141,7 @@ gotest_in_verify_ci: failpoint-enable tools/bin/gotestsum
 	@export TZ='Asia/Shanghai'; \
 	CGO_ENABLED=1 tools/bin/gotestsum --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" -- -v -p $(P) \
 	-ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -coverprofile="$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" \
-	$(PACKAGES_TIDB_TESTS_WITHOUT_BR) -check.p true || { $(FAILPOINT_DISABLE); exit 1; }
+	$(PACKAGES_TIDB_TESTS) -check.p true || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 race: failpoint-enable

Makefile.common

@@ -46,15 +46,13 @@ LINUX     := "Linux"
 MAC       := "Darwin"
 
 PACKAGE_LIST  := go list ./...
-PACKAGE_LIST_TIDB_TESTS  := go list ./... | grep -vE "github.com\/pingcap\/tidb\/cmd|github.com\/pingcap\/tidb\/dumpling"
+PACKAGE_LIST_TIDB_TESTS  := go list ./... | grep -vE "github.com\/pingcap\/tidb\/br|github.com\/pingcap\/tidb\/cmd|github.com\/pingcap\/tidb\/dumpling"
 PACKAGES  ?= $$($(PACKAGE_LIST))
 PACKAGES_TIDB_TESTS ?= $$($(PACKAGE_LIST_TIDB_TESTS))
-PACKAGES_TIDB_TESTS_WITHOUT_BR ?= $$($(PACKAGE_LIST_TIDB_TESTS) | grep -v "br")
 PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||'
 PACKAGE_DIRECTORIES_TIDB_TESTS := $(PACKAGE_LIST_TIDB_TESTS) | sed 's|github.com/pingcap/$(PROJECT)/||'
-PACKAGE_DIRECTORIES_TIDB_TESTS_WITHOUT_BR := $(PACKAGE_LIST_TIDB_TESTS) | sed 's|github.com/pingcap/$(PROJECT)/||' | grep -v "br"
 FILES     := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go")
-FILES_TIDB_TESTS := $$(find $$($(PACKAGE_DIRECTORIES_TIDB_TESTS_WITHOUT_BR)) -name "*.go")
+FILES_TIDB_TESTS := $$(find $$($(PACKAGE_DIRECTORIES_TIDB_TESTS)) -name "*.go")
 
 UNCONVERT_PACKAGES_LIST := go list ./...| grep -vE "lightning\/checkpoints|lightning\/manual|lightning\/common"
 UNCONVERT_PACKAGES := $$($(UNCONVERT_PACKAGES_LIST))

cmd/explaintest/r/collation_agg_func_enabled.result

@@ -195,7 +195,7 @@ StreamAgg	1.00	root		funcs:min(Column#8)->Column#6
     └─TableFullScan	10000.00	cop[tikv]	table:tt	keep order:false, stats:pseudo
 select min(b) from tt;
 min(b)
-B
+a
 desc format='brief' select min(b collate utf8mb4_bin) from tt;
 id	estRows	task	access object	operator info
 StreamAgg	1.00	root		funcs:min(Column#8)->Column#6
@@ -224,7 +224,7 @@ HashAgg	1.00	root		funcs:min(collation_agg_func.tt.c)->Column#6
   └─TableFullScan	10000.00	cop[tikv]	table:tt	keep order:false, stats:pseudo
 select min(c) from tt;
 min(c)
-B
+a
 desc format='brief' select min(c collate utf8mb4_bin) from tt;
 id	estRows	task	access object	operator info
 HashAgg	1.00	root		funcs:min(Column#7)->Column#6

cmd/explaintest/r/common_collation.result

@@ -13,3 +13,25 @@ coercibility(convert('a' using utf8mb4))
 select coercibility(convert('a' using utf8mb4) collate utf8mb4_general_ci);
 coercibility(convert('a' using utf8mb4) collate utf8mb4_general_ci)
 0
+DROP TABLE IF EXISTS t2;
+CREATE TABLE t2 (
+id INT NOT NULL PRIMARY KEY auto_increment,
+`col_91` char(47) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+`col_92` int(10) unsigned DEFAULT '2478966067',
+`col_97` char(32) COLLATE utf8mb4_bin NOT NULL
+) collate utf8mb4_general_ci;
+INSERT INTO `t2` VALUES (17,'UUtJeaV',497551109,'snRXXCZHBPW');
+SET names utf8mb4 collate utf8mb4_bin;
+SELECT greatest( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;
+expr1
+snRXXCZHBPW
+SELECT least( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;
+expr1
+UUtJeaV
+SET names utf8mb4 collate utf8mb4_general_ci;
+SELECT greatest( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;
+expr1
+snRXXCZHBPW
+SELECT least( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;
+expr1
+UUtJeaV

cmd/explaintest/t/common_collation.test

@@ -6,4 +6,22 @@ insert into t values ('啊', '撒旦');
 select coercibility(concat(a, b)) from t;
 select coercibility(convert(concat(a, b) using utf8mb4) collate utf8mb4_general_ci) from t;
 select coercibility(convert('a' using utf8mb4));
-select coercibility(convert('a' using utf8mb4) collate utf8mb4_general_ci);
+select coercibility(convert('a' using utf8mb4) collate utf8mb4_general_ci);
+
+# test greatest and least function with collation.
+DROP TABLE IF EXISTS t2;
+CREATE TABLE t2 (
+    id INT NOT NULL PRIMARY KEY auto_increment,
+  `col_91` char(47) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+  `col_92` int(10) unsigned DEFAULT '2478966067',
+  `col_97` char(32) COLLATE utf8mb4_bin NOT NULL
+) collate utf8mb4_general_ci;
+
+INSERT INTO `t2` VALUES (17,'UUtJeaV',497551109,'snRXXCZHBPW');
+
+SET names utf8mb4 collate utf8mb4_bin;
+SELECT greatest( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;
+SELECT least( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;
+SET names utf8mb4 collate utf8mb4_general_ci;
+SELECT greatest( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;
+SELECT least( col_91 , col_97 ) as expr1 FROM t2 ORDER BY id;

cmd/importer/db.go

@@ -117,7 +117,7 @@ func genRowDatas(table *table, count int) ([]string, error) {
 }
 
 func genRowData(table *table) (string, error) {
-	var values []byte
+	var values []byte // nolint: prealloc
 	for _, column := range table.columns {
 		data, err := genColumnData(table, column)
 		if err != nil {

cmd/importer/rand.go

@@ -87,7 +87,7 @@ func randDate(col *column) string {
 		log.Warn("parse min date failed", zap.Error(err))
 	}
 	if max == "" {
-		t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour)
+		t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour) // nolint: durationcheck
 		return fmt.Sprintf("%04d-%02d-%02d", t.Year(), t.Month(), t.Day())
 	}
 
@@ -96,7 +96,7 @@ func randDate(col *column) string {
 		log.Warn("parse max date failed", zap.Error(err))
 	}
 	days := int(maxTime.Sub(minTime).Hours() / 24)
-	t := minTime.Add(time.Duration(randInt(0, days)) * 24 * time.Hour)
+	t := minTime.Add(time.Duration(randInt(0, days)) * 24 * time.Hour) // nolint: durationcheck
 	return fmt.Sprintf("%04d-%02d-%02d", t.Year(), t.Month(), t.Day())
 }
 
@@ -145,7 +145,7 @@ func randTimestamp(col *column) string {
 		log.Warn("parse min timestamp failed", zap.Error(err))
 	}
 	if max == "" {
-		t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour)
+		t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour) // nolint: durationcheck
 		return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
 	}
 

config/config.go

@@ -131,7 +131,7 @@ type Config struct {
 	CompatibleKillQuery        bool               `toml:"compatible-kill-query" json:"compatible-kill-query"`
 	Plugin                     Plugin             `toml:"plugin" json:"plugin"`
 	PessimisticTxn             PessimisticTxn     `toml:"pessimistic-txn" json:"pessimistic-txn"`
-	CheckMb4ValueInUTF8        bool               `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
+	CheckMb4ValueInUTF8        AtomicBool         `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
 	MaxIndexLength             int                `toml:"max-index-length" json:"max-index-length"`
 	IndexLimit                 int                `toml:"index-limit" json:"index-limit"`
 	TableColumnCountLimit      uint32             `toml:"table-column-count-limit" json:"table-column-count-limit"`
@@ -629,7 +629,7 @@ var defaultConf = Config{
 	MemQuotaQuery:                1 << 30,
 	EnableStreaming:              false,
 	EnableBatchDML:               false,
-	CheckMb4ValueInUTF8:          true,
+	CheckMb4ValueInUTF8:          *NewAtomicBool(true),
 	MaxIndexLength:               3072,
 	IndexLimit:                   64,
 	TableColumnCountLimit:        1017,