Merge branch 'master' into fix_clustered_index_prefix

bindinfo/bind_test.go

@@ -2061,3 +2061,21 @@ func (s *testSuite) TestExplainTableStmts(c *C) {
 	tk.MustExec("explain table t")
 	tk.MustExec("desc table t")
 }
+
+func (s *testSuite) TestSPMWithoutUseDatabase(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk1 := testkit.NewTestKit(c, s.store)
+	s.cleanBindingEnv(tk)
+	s.cleanBindingEnv(tk1)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, key(a))")
+	tk.MustExec("create global binding for select * from t using select * from t force index(a)")
+
+	err := tk1.ExecToErr("select * from t")
+	c.Assert(err, ErrorMatches, "*No database selected")
+	tk1.MustQuery(`select @@last_plan_from_binding;`).Check(testkit.Rows("0"))
+	c.Assert(tk1.MustUseIndex("select * from test.t", "a"), IsTrue)
+	tk1.MustExec("select * from test.t")
+	tk1.MustQuery(`select @@last_plan_from_binding;`).Check(testkit.Rows("1"))
+}

cmd/explaintest/r/partition_pruning.result

@@ -2276,7 +2276,7 @@ TableReader	250.00	root	partition:dual	data:Selection
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select * from t2 where a > 0xFE AND a <= 0xFF;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:dual	data:Selection
 └─Selection	250.00	cop[tikv]		gt(test.t2.a, 254), le(test.t2.a, 255)
   └─TableFullScan	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
 explain format = 'brief' select * from t1 where a >= 0xFE AND a <= 0xFF;
@@ -2286,7 +2286,7 @@ TableReader	250.00	root	partition:p2	data:Selection
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select * from t2 where a >= 0xFE AND a <= 0xFF;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:p2	data:Selection
 └─Selection	250.00	cop[tikv]		ge(test.t2.a, 254), le(test.t2.a, 255)
   └─TableFullScan	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
 explain format = 'brief' select * from t1 where a < 64 AND a >= 63;
@@ -2296,7 +2296,7 @@ TableReader	250.00	root	partition:p0	data:Selection
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select * from t2 where a < 64 AND a >= 63;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:p0	data:Selection
 └─Selection	250.00	cop[tikv]		ge(test.t2.a, 63), lt(test.t2.a, 64)
   └─TableFullScan	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
 explain format = 'brief' select * from t1 where a <= 64 AND a >= 63;
@@ -2306,7 +2306,7 @@ TableReader	250.00	root	partition:p0,p1	data:Selection
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select * from t2 where a <= 64 AND a >= 63;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:p0,p1	data:Selection
 └─Selection	250.00	cop[tikv]		ge(test.t2.a, 63), le(test.t2.a, 64)
   └─TableFullScan	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
 drop table t1;
@@ -2322,13 +2322,13 @@ insert into t1 values (9),(19),(0xFFFF0000FFFF000-1), (0xFFFF0000FFFFFFF-1);
 explain format = 'brief' select * from t1 where
 a >= 2305561538531885056-10 and a <= 2305561538531885056-8;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:p3	data:Selection
 └─Selection	250.00	cop[tikv]		ge(test.t1.a, 2305561538531885046), le(test.t1.a, 2305561538531885048)
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select * from t1 where
 a > 0xFFFFFFFFFFFFFFEC and a < 0xFFFFFFFFFFFFFFEE;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:dual	data:Selection
 └─Selection	250.00	cop[tikv]		gt(test.t1.a, 18446744073709551596), lt(test.t1.a, 18446744073709551598)
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select * from t1 where a>=0 and a <= 0xFFFFFFFFFFFFFFFF;
@@ -2346,7 +2346,7 @@ partition p4 values less than (1000)
 insert into t1 values (-15),(-5),(5),(15),(-15),(-5),(5),(15);
 explain format = 'brief' select * from t1 where a>-2 and a <=0;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:p3	data:Selection
 └─Selection	250.00	cop[tikv]		gt(test.t1.a, -2), le(test.t1.a, 0)
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 drop table t1;
@@ -2391,12 +2391,12 @@ partition p2 values less than (255)
 insert into t1 select A.a + 10*B.a from t0 A, t0 B;
 explain format = 'brief' select * from t1 where a between 10 and 13;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:p0	data:Selection
 └─Selection	250.00	cop[tikv]		ge(test.t1.a, 10), le(test.t1.a, 13)
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select * from t1 where a between 10 and 10+33;
 id	estRows	task	access object	operator info
-TableReader	250.00	root	partition:all	data:Selection
+TableReader	250.00	root	partition:p0	data:Selection
 └─Selection	250.00	cop[tikv]		ge(test.t1.a, 10), le(test.t1.a, 43)
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 drop table t0, t1;

config/config.go

@@ -617,7 +617,7 @@ var defaultConf = Config{
 		HeaderTimeout: 5,
 	},
 	PreparedPlanCache: PreparedPlanCache{
-		Enabled:          true,
+		Enabled:          false,
 		Capacity:         100,
 		MemoryGuardRatio: 0.1,
 	},
@@ -699,6 +699,7 @@ var deprecatedConfig = map[string]struct{}{
 	"max-txn-time-use":               {},
 	"experimental.allow-auto-random": {},
 	"enable-redact-log":              {}, // use variable tidb_redact_log instead
+	"tikv-client.copr-cache.enable":  {},
 }
 
 func isAllDeprecatedConfigItems(items []string) bool {

config/config.toml.example

@@ -312,7 +312,7 @@ networks = ""
 header-timeout = 5
 
 [prepared-plan-cache]
-enabled = true
+enabled = false
 capacity = 100
 memory-guard-ratio = 0.1
 
@@ -418,20 +418,9 @@ store-liveness-timeout = "5s"
 ttl-refreshed-txn-size = 33554432
 
 [tikv-client.copr-cache]
-# Whether to enable the copr cache. The copr cache saves the result from TiKV Coprocessor in the memory and
-# reuses the result when corresponding data in TiKV is unchanged, on a region basis.
-enable = true
-
-# The capacity in MB of the cache.
+# The capacity in MB of the cache. Zero means disable coprocessor cache.
 capacity-mb = 1000.0
 
-# Only cache small requests. Zero means no limits for requests.
-admission-max-ranges = 500
-# Only cache requests whose result set is small.
-admission-max-result-mb = 10.0
-# Only cache requests takes notable time to process.
-admission-min-process-ms = 5
-
 [binlog]
 # enable to write binlog.
 # NOTE: If binlog is enabled with Kafka (e.g. arbiter cluster),

ddl/db_integration_test.go

@@ -2225,7 +2225,7 @@ func (s *testIntegrationSuite7) TestAddExpressionIndex(c *C) {
 }
 
 func (s *testIntegrationSuite7) TestCreateExpressionIndexError(c *C) {
-	defer config.RestoreFunc()
+	defer config.RestoreFunc()()
 	config.UpdateGlobal(func(conf *config.Config) {
 		conf.Experimental.AllowsExpressionIndex = true
 		conf.AlterPrimaryKey = true
@@ -2640,6 +2640,7 @@ func (s *testIntegrationSuite7) TestDuplicateErrorMessage(c *C) {
 			restoreConfig := config.RestoreFunc()
 			config.UpdateGlobal(func(conf *config.Config) {
 				conf.EnableGlobalIndex = globalIndex
+				conf.AlterPrimaryKey = false
 			})
 			for _, clusteredIndex := range []bool{false, true} {
 				tk.Se.GetSessionVars().EnableClusteredIndex = clusteredIndex

ddl/db_test.go

@@ -58,7 +58,6 @@ import (
 	"github.com/pingcap/tidb/util/domainutil"
 	"github.com/pingcap/tidb/util/israce"
 	"github.com/pingcap/tidb/util/mock"
-	"github.com/pingcap/tidb/util/rowcodec"
 	"github.com/pingcap/tidb/util/testkit"
 	"github.com/pingcap/tidb/util/testutil"
 )
@@ -1141,7 +1140,11 @@ func (s *testDBSuite4) TestAddIndex4(c *C) {
 			      partition p4 values less than maxvalue)`, "")
 }
 
-func (s *testDBSuite5) TestAddIndex5(c *C) {
+func (s *testSerialDBSuite) TestAddIndex5(c *C) {
+	defer config.RestoreFunc()()
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.AlterPrimaryKey = false
+	})
 	testAddIndex(c, s.store, s.lease, testClusteredIndex,
 		`create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c2, c3))`, "")
 }
@@ -1303,7 +1306,7 @@ LOOP:
 
 		c.Assert(err, IsNil)
 		_, ok := handles.Get(h)
-		c.Assert(ok, IsTrue)
+		c.Assert(ok, IsTrue, Commentf("handle: %v", h.String()))
 		handles.Delete(h)
 	}
 	c.Assert(handles.Len(), Equals, 0)
@@ -1903,15 +1906,6 @@ func checkGlobalIndexRow(c *C, ctx sessionctx.Context, tblInfo *model.TableInfo,
 	for _, col := range tblInfo.Columns {
 		tblColMap[col.ID] = &col.FieldType
 	}
-	idxColInfos := make([]rowcodec.ColInfo, 0, len(indexInfo.Columns))
-	for _, idxCol := range indexInfo.Columns {
-		col := tblInfo.Columns[idxCol.Offset]
-		idxColInfos = append(idxColInfos, rowcodec.ColInfo{
-			ID:         col.ID,
-			IsPKHandle: tblInfo.PKIsHandle && mysql.HasPriKeyFlag(col.Flag),
-			Ft:         rowcodec.FieldTypeFromModelColumn(col),
-		})
-	}
 
 	// Check local index entry does not exist.
 	localPrefix := tablecodec.EncodeTableIndexPrefix(pid, indexInfo.ID)
@@ -1928,6 +1922,7 @@ func checkGlobalIndexRow(c *C, ctx sessionctx.Context, tblInfo *model.TableInfo,
 	c.Assert(err, IsNil)
 	value, err := txn.Get(context.Background(), key)
 	c.Assert(err, IsNil)
+	idxColInfos := tables.BuildRowcodecColInfoForIndexColumns(indexInfo, tblInfo)
 	colVals, err := tablecodec.DecodeIndexKV(key, value, len(indexInfo.Columns), tablecodec.HandleDefault, idxColInfos)
 	c.Assert(err, IsNil)
 	c.Assert(colVals, HasLen, len(idxVals)+2)
@@ -1955,8 +1950,10 @@ func checkGlobalIndexRow(c *C, ctx sessionctx.Context, tblInfo *model.TableInfo,
 }
 
 func (s *testSerialDBSuite) TestAddGlobalIndex(c *C) {
+	defer config.RestoreFunc()()
 	config.UpdateGlobal(func(conf *config.Config) {
 		conf.EnableGlobalIndex = true
+		conf.AlterPrimaryKey = true
 	})
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test_db")
@@ -2024,9 +2021,6 @@ func (s *testSerialDBSuite) TestAddGlobalIndex(c *C) {
 
 	err = txn.Commit(context.Background())
 	c.Assert(err, IsNil)
-	config.UpdateGlobal(func(conf *config.Config) {
-		conf.EnableGlobalIndex = false
-	})
 }
 
 func (s *testDBSuite) showColumns(tk *testkit.TestKit, c *C, tableName string) [][]interface{} {
@@ -2678,6 +2672,10 @@ func (s *testSerialDBSuite) TestRepairTable(c *C) {
 	defer func() {
 		c.Assert(failpoint.Disable("github.com/pingcap/tidb/infoschema/repairFetchCreateTable"), IsNil)
 	}()
+	defer config.RestoreFunc()()
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.AlterPrimaryKey = true
+	})
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists t, other_table, origin")

ddl/ddl_worker_test.go

@@ -102,6 +102,11 @@ func (s *testDDLSuite) TestNotifyDDLJob(c *C) {
 	)
 	defer d.Stop()
 	getFirstNotificationAfterStartDDL(d)
+	// Ensure that the notification is not handled in workers `start` function.
+	d.cancel()
+	for _, worker := range d.workers {
+		worker.close()
+	}
 
 	job := &model.Job{
 		SchemaID:   1,
@@ -127,6 +132,7 @@ func (s *testDDLSuite) TestNotifyDDLJob(c *C) {
 	default:
 		c.Fatal("do not get the add index job notification")
 	}
+
 	// Test the notification mechanism that the owner and the server receiving the DDL request are not on the same TiDB.
 	// And the etcd client is nil.
 	d1 := testNewDDLAndStart(
@@ -135,8 +141,13 @@ func (s *testDDLSuite) TestNotifyDDLJob(c *C) {
 		WithStore(store),
 		WithLease(testLease),
 	)
-	getFirstNotificationAfterStartDDL(d1)
 	defer d1.Stop()
+	getFirstNotificationAfterStartDDL(d1)
+	// Ensure that the notification is not handled by worker's "start".
+	d1.cancel()
+	for _, worker := range d1.workers {
+		worker.close()
+	}
 	d1.ownerManager.RetireOwner()
 	d1.asyncNotifyWorker(job)
 	job.Type = model.ActionCreateTable

ddl/index.go

@@ -14,7 +14,6 @@
 package ddl
 
 import (
-	"bytes"
 	"context"
 	"strings"
 	"sync/atomic"
@@ -40,10 +39,8 @@ import (
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util"
-	"github.com/pingcap/tidb/util/codec"
 	"github.com/pingcap/tidb/util/logutil"
 	decoder "github.com/pingcap/tidb/util/rowDecoder"
-	"github.com/pingcap/tidb/util/rowcodec"
 	"github.com/pingcap/tidb/util/timeutil"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
@@ -1026,68 +1023,34 @@ func (w *addIndexWorker) initBatchCheckBufs(batchCount int) {
 func (w *addIndexWorker) checkHandleExists(key kv.Key, value []byte, handle kv.Handle) error {
 	idxInfo := w.index.Meta()
 	tblInfo := w.table.Meta()
-	name := w.index.Meta().Name.String()
-
-	colInfo := make([]rowcodec.ColInfo, 0, len(idxInfo.Columns))
-	for _, idxCol := range idxInfo.Columns {
-		col := tblInfo.Columns[idxCol.Offset]
-		colInfo = append(colInfo, rowcodec.ColInfo{
-			ID:         col.ID,
-			IsPKHandle: tblInfo.PKIsHandle && mysql.HasPriKeyFlag(col.Flag),
-			Ft:         rowcodec.FieldTypeFromModelColumn(col),
-		})
+	idxColLen := len(idxInfo.Columns)
+	h, err := tablecodec.DecodeIndexHandle(key, value, idxColLen)
+	if err != nil {
+		return errors.Trace(err)
 	}
-
-	values, err := tablecodec.DecodeIndexKV(key, value, len(idxInfo.Columns), tablecodec.HandleDefault, colInfo)
+	hasBeenBackFilled := h.Equal(handle)
+	if hasBeenBackFilled {
+		return nil
+	}
+	colInfos := tables.BuildRowcodecColInfoForIndexColumns(idxInfo, tblInfo)
+	values, err := tablecodec.DecodeIndexKV(key, value, idxColLen, tablecodec.HandleNotNeeded, colInfos)
 	if err != nil {
 		return err
 	}
-
-	if !w.table.Meta().IsCommonHandle {
-		_, d, err := codec.DecodeOne(values[len(colInfo)])
-		if err != nil {
-			return errors.Trace(err)
-		}
-		if d.GetInt64() == handle.IntValue() {
-			return nil
-		}
-	} else {
-		// We expect the two handle have the same number of columns, because they come from a same table.
-		// But we still need to check it explicitly, otherwise we will encounter undesired index out of range panic,
-		// or undefined behavior if someone change the format of the value returned by tablecodec.DecodeIndexKV.
-		colsOfHandle := len(values) - len(colInfo)
-		if w.index.Meta().Global {
-			colsOfHandle--
-		}
-		if colsOfHandle != handle.NumCols() {
-			// We can claim these two handle are different, because they have different length.
-			// But we'd better report an error at here to detect compatibility problem introduced in other package during tests.
-			return errors.New("number of columns in two handle is different")
-		}
-
-		for i := 0; i < handle.NumCols(); i++ {
-			if bytes.Equal(values[i+len(colInfo)], handle.EncodedCol(i)) {
-				colsOfHandle--
-			}
-		}
-		if colsOfHandle == 0 {
-			return nil
-		}
-	}
-
-	valueStr := make([]string, 0, len(colInfo))
-	for i, val := range values[:len(colInfo)] {
-		d, err := tablecodec.DecodeColumnValue(val, colInfo[i].Ft, time.Local)
+	indexName := w.index.Meta().Name.String()
+	valueStr := make([]string, 0, idxColLen)
+	for i, val := range values[:idxColLen] {
+		d, err := tablecodec.DecodeColumnValue(val, colInfos[i].Ft, time.Local)
 		if err != nil {
-			return kv.ErrKeyExists.FastGenByArgs(key.String(), name)
+			return kv.ErrKeyExists.FastGenByArgs(key.String(), indexName)
 		}
 		str, err := d.ToString()
 		if err != nil {
 			str = string(val)
 		}
 		valueStr = append(valueStr, str)
 	}
-	return kv.ErrKeyExists.FastGenByArgs(strings.Join(valueStr, "-"), name)
+	return kv.ErrKeyExists.FastGenByArgs(strings.Join(valueStr, "-"), indexName)
 }
 
 func (w *addIndexWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*indexRecord) error {

distsql/select_result.go

@@ -455,7 +455,7 @@ func (s *selectResultRuntimeStats) String() string {
 			buf.WriteString(", rpc_time: ")
 			buf.WriteString(execdetails.FormatDuration(time.Duration(copRPC.Consume)))
 		}
-		if config.GetGlobalConfig().TiKVClient.CoprCache.Enable {
+		if config.GetGlobalConfig().TiKVClient.CoprCache.CapacityMB > 0 {
 			buf.WriteString(fmt.Sprintf(", copr_cache_hit_ratio: %v",
 				strconv.FormatFloat(float64(s.CoprCacheHitNum)/float64(len(s.copRespTime)), 'f', 2, 64)))
 		} else {