From 8469dd638bf6712dc31d8aa1edbed775c67b09d3 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Thu, 29 Sep 2022 21:19:17 +0800 Subject: [PATCH 1/4] refine index join range display for clustered index --- planner/core/exhaust_physical_plans.go | 23 +++++++++++++++---- planner/core/explain.go | 15 ++++-------- planner/core/integration_test.go | 2 ++ planner/core/physical_plans.go | 8 +++---- .../core/testdata/integration_suite_in.json | 4 +++- .../core/testdata/integration_suite_out.json | 16 +++++++++++-- 6 files changed, 45 insertions(+), 23 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index ba3dd81329204..0ea00464768a1 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -18,6 +18,7 @@ import ( "bytes" "fmt" "math" + "strings" "unsafe" "github.com/pingcap/errors" @@ -804,13 +805,14 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( if helper == nil { return nil } - innerTask = p.constructInnerTableScanTask(ds, helper.chosenRanges.Range(), outerJoinKeys, us, false, false, avgInnerRowCnt) + rangeInfo := helper.buildRangeDecidedByInformation(helper.chosenPath.IdxCols, outerJoinKeys) + innerTask = p.constructInnerTableScanTask(ds, helper.chosenRanges.Range(), outerJoinKeys, us, rangeInfo, false, false, avgInnerRowCnt) // The index merge join's inner plan is different from index join, so we // should construct another inner plan for it. // Because we can't keep order for union scan, if there is a union scan in inner task, // we can't construct index merge join. if us == nil { - innerTask2 = p.constructInnerTableScanTask(ds, helper.chosenRanges.Range(), outerJoinKeys, us, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) + innerTask2 = p.constructInnerTableScanTask(ds, helper.chosenRanges.Range(), outerJoinKeys, us, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) } ranges = helper.chosenRanges } else { @@ -834,13 +836,23 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( return nil } ranges := ranger.FullIntRange(mysql.HasUnsignedFlag(pkCol.RetType.GetFlag())) - innerTask = p.constructInnerTableScanTask(ds, ranges, outerJoinKeys, us, false, false, avgInnerRowCnt) + var buffer strings.Builder + buffer.WriteString("[") + for i, key := range outerJoinKeys { + if i != 0 { + buffer.WriteString(" ") + } + buffer.WriteString(fmt.Sprintf("eq(%v, %v)", pkCol, key)) + } + buffer.WriteString("]") + rangeInfo := buffer.String() + innerTask = p.constructInnerTableScanTask(ds, ranges, outerJoinKeys, us, rangeInfo, false, false, avgInnerRowCnt) // The index merge join's inner plan is different from index join, so we // should construct another inner plan for it. // Because we can't keep order for union scan, if there is a union scan in inner task, // we can't construct index merge join. if us == nil { - innerTask2 = p.constructInnerTableScanTask(ds, ranges, outerJoinKeys, us, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) + innerTask2 = p.constructInnerTableScanTask(ds, ranges, outerJoinKeys, us, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) } } var ( @@ -960,6 +972,7 @@ func (p *LogicalJoin) constructInnerTableScanTask( ranges ranger.Ranges, outerJoinKeys []*expression.Column, us *LogicalUnionScan, + rangeInfo string, keepOrder bool, desc bool, rowCount float64, @@ -977,7 +990,7 @@ func (p *LogicalJoin) constructInnerTableScanTask( DBName: ds.DBName, filterCondition: ds.pushedDownConds, Ranges: ranges, - rangeDecidedBy: outerJoinKeys, + rangeInfo: rangeInfo, KeepOrder: keepOrder, Desc: desc, physicalTableID: ds.physicalTableID, diff --git a/planner/core/explain.go b/planner/core/explain.go index 706f96c7a047b..2dd4c5802ea4c 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -171,15 +171,10 @@ func (p *PhysicalTableScan) ExplainNormalizedInfo() string { // OperatorInfo implements dataAccesser interface. func (p *PhysicalTableScan) OperatorInfo(normalized bool) string { var buffer strings.Builder - if len(p.rangeDecidedBy) > 0 { - buffer.WriteString("range: decided by [") - for i, rangeDecidedBy := range p.rangeDecidedBy { - if i != 0 { - buffer.WriteString(" ") - } - buffer.WriteString(rangeDecidedBy.String()) - } - buffer.WriteString("], ") + if len(p.rangeInfo) > 0 { + buffer.WriteString("range: decided by ") + buffer.WriteString(p.rangeInfo) + buffer.WriteString(", ") } else if p.haveCorCol() { if normalized { buffer.WriteString("range: decided by ") @@ -230,7 +225,7 @@ func (p *PhysicalTableScan) haveCorCol() bool { } func (p *PhysicalTableScan) isFullScan() bool { - if len(p.rangeDecidedBy) > 0 || p.haveCorCol() { + if len(p.rangeInfo) > 0 || p.haveCorCol() { return false } var unsignedIntHandle bool diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 17d265be0490b..0d97f20bdb2d8 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1730,6 +1730,8 @@ func TestIndexJoinTableRange(t *testing.T) { tk.MustExec("drop table if exists t1, t2") tk.MustExec("create table t1(a int, b int, primary key (a), key idx_t1_b (b))") tk.MustExec("create table t2(a int, b int, primary key (a), key idx_t1_b (b))") + tk.MustExec("create table t3(a int)") + tk.MustExec("create table t4(a int, b int, c int, primary key (a, b) clustered)") var input []string var output []struct { diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index b42d6855332df..a4419b2ab5caf 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -788,7 +788,7 @@ type PhysicalTableScan struct { physicalTableID int64 - rangeDecidedBy []*expression.Column + rangeInfo string // HandleIdx is the index of handle, which is only used for admin check table. HandleIdx []int @@ -839,7 +839,7 @@ func (ts *PhysicalTableScan) Clone() (PhysicalPlan, error) { if ts.Hist != nil { clonedScan.Hist = ts.Hist.Copy() } - clonedScan.rangeDecidedBy = cloneCols(ts.rangeDecidedBy) + clonedScan.rangeInfo = ts.rangeInfo return clonedScan, nil } @@ -961,9 +961,7 @@ func (ts *PhysicalTableScan) MemoryUsage() (sum int64) { for _, rang := range ts.Ranges { sum += rang.MemUsage() } - for _, col := range ts.rangeDecidedBy { - sum += col.MemoryUsage() - } + sum += int64(len(ts.rangeInfo)) for _, col := range ts.tblCols { sum += col.MemoryUsage() } diff --git a/planner/core/testdata/integration_suite_in.json b/planner/core/testdata/integration_suite_in.json index 1235ecc3535cb..63a5ed8b9c36d 100644 --- a/planner/core/testdata/integration_suite_in.json +++ b/planner/core/testdata/integration_suite_in.json @@ -107,7 +107,9 @@ "name": "TestIndexJoinTableRange", "cases": [ "desc format = 'brief' select /*+ TIDB_INLJ(t2)*/ * from t1, t2 where t1.a = t2.a and t1.b = t2.b", - "desc format = 'brief' select /*+ TIDB_INLJ(t2)*/ * from t1, t2 where t1.a = t2.a and t1.b = t2.a and t1.b = t2.b" + "desc format = 'brief' select /*+ TIDB_INLJ(t2)*/ * from t1, t2 where t1.a = t2.a and t1.b = t2.a and t1.b = t2.b", + "desc format = 'brief' select /*+ INL_JOIN(t4) */ * from t3 join t4 on t3.a = t4.a where t4.b = 1", + "desc format = 'brief' select /*+ INL_JOIN(t4) */ * from t3 join t4 on t3.b = t4.b where t4.a = 1" ] }, { diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index df6dd3f068f59..6d9bb72f44ef6 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -456,7 +456,7 @@ "│ └─IndexFullScan 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", "└─TableReader(Probe) 1.00 root data:Selection", " └─Selection 1.00 cop[tikv] not(isnull(test.t2.b))", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t1.a], keep order:false, stats:pseudo" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t2.a, test.t1.a)], keep order:false, stats:pseudo" ] }, { @@ -467,7 +467,19 @@ "│ └─IndexFullScan 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", "└─TableReader(Probe) 1.00 root data:Selection", " └─Selection 1.00 cop[tikv] not(isnull(test.t2.b))", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t1.a test.t1.b], keep order:false, stats:pseudo" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t2.a, test.t1.a) eq(test.t2.a, test.t1.b)], keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ INL_JOIN(t4) */ * from t3 join t4 on t3.a = t4.a where t4.b = 1", + "Plan": [ + + ] + }, + { + "SQL": "desc format = 'brief' select /*+ INL_JOIN(t4) */ * from t3 join t4 on t3.b = t4.b where t4.a = 1", + "Plan": [ + ] } ] From 286ea2be9edbd67133feb7e49a650f099ad89a69 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Thu, 29 Sep 2022 21:23:20 +0800 Subject: [PATCH 2/4] add test --- planner/core/integration_test.go | 2 +- planner/core/testdata/integration_suite_out.json | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 0d97f20bdb2d8..156898906b887 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1730,7 +1730,7 @@ func TestIndexJoinTableRange(t *testing.T) { tk.MustExec("drop table if exists t1, t2") tk.MustExec("create table t1(a int, b int, primary key (a), key idx_t1_b (b))") tk.MustExec("create table t2(a int, b int, primary key (a), key idx_t1_b (b))") - tk.MustExec("create table t3(a int)") + tk.MustExec("create table t3(a int, b int, c int)") tk.MustExec("create table t4(a int, b int, c int, primary key (a, b) clustered)") var input []string diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 6d9bb72f44ef6..5b0112e0ce21e 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -473,13 +473,25 @@ { "SQL": "desc format = 'brief' select /*+ INL_JOIN(t4) */ * from t3 join t4 on t3.a = t4.a where t4.b = 1", "Plan": [ - + "IndexJoin 12.50 root inner join, inner:TableReader, outer key:test.t3.a, inner key:test.t4.a, equal cond:eq(test.t3.a, test.t4.a)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test.t3.a))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo", + "└─TableReader(Probe) 0.00 root data:Selection", + " └─Selection 0.00 cop[tikv] eq(test.t4.b, 1)", + " └─TableRangeScan 1.00 cop[tikv] table:t4 range: decided by [eq(test.t4.a, test.t3.a) eq(test.t4.b, 1)], keep order:false, stats:pseudo" ] }, { "SQL": "desc format = 'brief' select /*+ INL_JOIN(t4) */ * from t3 join t4 on t3.b = t4.b where t4.a = 1", "Plan": [ - + "IndexJoin 12.50 root inner join, inner:TableReader, outer key:test.t3.b, inner key:test.t4.b, equal cond:eq(test.t3.b, test.t4.b)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test.t3.b))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo", + "└─TableReader(Probe) 0.00 root data:Selection", + " └─Selection 0.00 cop[tikv] eq(test.t4.a, 1)", + " └─TableRangeScan 1.00 cop[tikv] table:t4 range: decided by [eq(test.t4.b, test.t3.b) eq(test.t4.a, 1)], keep order:false, stats:pseudo" ] } ] From 50031d22d43259ce699a2605c6378ffad28aaeec Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 30 Sep 2022 10:52:17 +0800 Subject: [PATCH 3/4] fix --- planner/core/exhaust_physical_plans.go | 2 +- planner/core/explain.go | 1 + planner/core/integration_test.go | 2 +- planner/core/testdata/integration_suite_out.json | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 0ea00464768a1..7fecffb556e0d 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -842,7 +842,7 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( if i != 0 { buffer.WriteString(" ") } - buffer.WriteString(fmt.Sprintf("eq(%v, %v)", pkCol, key)) + buffer.WriteString(key.String()) } buffer.WriteString("]") rangeInfo := buffer.String() diff --git a/planner/core/explain.go b/planner/core/explain.go index 2dd4c5802ea4c..d73d45116955d 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -172,6 +172,7 @@ func (p *PhysicalTableScan) ExplainNormalizedInfo() string { func (p *PhysicalTableScan) OperatorInfo(normalized bool) string { var buffer strings.Builder if len(p.rangeInfo) > 0 { + // TODO: deal with normalized case buffer.WriteString("range: decided by ") buffer.WriteString(p.rangeInfo) buffer.WriteString(", ") diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 156898906b887..f7eaa1c29d592 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1721,7 +1721,7 @@ func TestInvisibleIndex(t *testing.T) { tk.MustExec("admin check index t i_a") } -// for issue #14822 +// for issue #14822 and #38258 func TestIndexJoinTableRange(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 5b0112e0ce21e..3b8f0c4021d1f 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -456,7 +456,7 @@ "│ └─IndexFullScan 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", "└─TableReader(Probe) 1.00 root data:Selection", " └─Selection 1.00 cop[tikv] not(isnull(test.t2.b))", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t2.a, test.t1.a)], keep order:false, stats:pseudo" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t1.a], keep order:false, stats:pseudo" ] }, { @@ -467,7 +467,7 @@ "│ └─IndexFullScan 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", "└─TableReader(Probe) 1.00 root data:Selection", " └─Selection 1.00 cop[tikv] not(isnull(test.t2.b))", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t2.a, test.t1.a) eq(test.t2.a, test.t1.b)], keep order:false, stats:pseudo" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t1.a test.t1.b], keep order:false, stats:pseudo" ] }, { From 124f38e6e8829f7844531da18a53e8498b1108c5 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 30 Sep 2022 12:03:36 +0800 Subject: [PATCH 4/4] fix ut --- executor/explainfor_test.go | 4 ++-- executor/index_lookup_join_test.go | 2 +- planner/core/testdata/integration_suite_out.json | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/executor/explainfor_test.go b/executor/explainfor_test.go index 77b15c292747a..21617c95aa1b7 100644 --- a/executor/explainfor_test.go +++ b/executor/explainfor_test.go @@ -466,7 +466,7 @@ func TestPointGetUserVarPlanCache(t *testing.T) { ` │ └─Point_Get_43 1.00 root table:t2, index:idx_a(a) `, ` └─TableReader_13(Probe) 0.00 root data:Selection_12`, ` └─Selection_12 0.00 cop[tikv] eq(test.t1.a, 1)`, - ` └─TableRangeScan_11 1.00 cop[tikv] table:t1 range: decided by [test.t2.a], keep order:false, stats:pseudo`)) + ` └─TableRangeScan_11 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo`)) tk.MustExec("set @a=2") tk.MustQuery("execute stmt using @a").Check(testkit.Rows( @@ -482,7 +482,7 @@ func TestPointGetUserVarPlanCache(t *testing.T) { ` │ └─Point_Get_43 1.00 root table:t2, index:idx_a(a) `, ` └─TableReader_13(Probe) 0.00 root data:Selection_12`, ` └─Selection_12 0.00 cop[tikv] eq(test.t1.a, 2)`, - ` └─TableRangeScan_11 1.00 cop[tikv] table:t1 range: decided by [test.t2.a], keep order:false, stats:pseudo`)) + ` └─TableRangeScan_11 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo`)) tk.MustQuery("execute stmt using @a").Check(testkit.Rows( "2 4 2 2", )) diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index 24168058353ec..ef8b582fb151f 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -434,7 +434,7 @@ PARTITIONS 1`) "[│ └─Selection_24 10.00 cop[tikv] eq(test.t2.postfiller, 1)]\n" + "[│ └─TableFullScan_23 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo]\n" + "[└─TableReader_14(Probe) 1.00 root partition:all data:TableRangeScan_13]\n" + - "[ └─TableRangeScan_13 1.00 cop[tikv] table:t1 range: decided by [test.t2.prefiller], keep order:false, stats:pseudo")) + "[ └─TableRangeScan_13 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo")) tk.MustQuery("show warnings").Check(testkit.Rows()) // without fix it fails with: "runtime error: index out of range [0] with length 0" tk.MustQuery("select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows()) diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 3b8f0c4021d1f..4a2c2b7cdddaf 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -1129,7 +1129,7 @@ "├─TableReader(Build) 3.00 root data:TableFullScan", "│ └─TableFullScan 3.00 cop[tikv] table:t1 keep order:false", "└─TableReader(Probe) 1.00 root data:TableRangeScan", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t.a], keep order:false" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t.a, test.t.a)], keep order:false" ], "Res": [ "1 111 1.1000000000 11 1 111 1.1000000000 11", @@ -1144,7 +1144,7 @@ "├─TableReader(Build) 3.00 root data:TableFullScan", "│ └─TableFullScan 3.00 cop[tikv] table:t1 keep order:false", "└─TableReader(Probe) 1.00 root data:TableRangeScan", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t.a], keep order:true" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t.a, test.t.a)], keep order:true" ], "Res": [ "1 111 1.1000000000 11 1 111 1.1000000000 11", @@ -1159,7 +1159,7 @@ "├─TableReader(Build) 3.00 root data:TableFullScan", "│ └─TableFullScan 3.00 cop[tikv] table:t1 keep order:false", "└─TableReader(Probe) 1.00 root data:TableRangeScan", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t.a], keep order:false" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t.a, test.t.a)], keep order:false" ], "Res": [ "1 111 1.1000000000 11 1 111 1.1000000000 11", @@ -1174,7 +1174,7 @@ "├─TableReader(Build) 3.00 root data:TableFullScan", "│ └─TableFullScan 3.00 cop[tikv] table:t1 keep order:false", "└─TableReader(Probe) 1.00 root data:TableRangeScan", - " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [test.t.a test.t.b], keep order:false" + " └─TableRangeScan 1.00 cop[tikv] table:t2 range: decided by [eq(test.t.a, test.t.a) eq(test.t.b, test.t.b)], keep order:false" ], "Res": [ "1 111 1.1000000000 11 1 111 1.1000000000 11",