Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner: fix index merge doesn't take effect when using prefix key #20425

Merged
merged 16 commits into from
Nov 12, 2020
Merged

planner: fix index merge doesn't take effect when using prefix key #20425

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7d9c61b
(WIP)planner: fix index merge doesn't take effect when using prefix key
tangwz Oct 13, 2020
df60685
add unittest.
tangwz Oct 13, 2020
c96e021
gofmt
tangwz Oct 13, 2020
87425d2
modify comment.
tangwz Oct 13, 2020
4a84545
add unittest.
tangwz Oct 14, 2020
35420c1
WIP
qw4990 Nov 3, 2020
06f0918
address comments
qw4990 Nov 3, 2020
d08bebc
fix CI
qw4990 Nov 3, 2020
13ed976
fixup
qw4990 Nov 5, 2020
c4caeef
fix CI
qw4990 Nov 6, 2020
97caf45
Merge remote-tracking branch 'upstream/master' into fix_index_merge
qw4990 Nov 6, 2020
9b6b596
fixup
qw4990 Nov 6, 2020
f3de03a
address comments
qw4990 Nov 6, 2020
e858c24
Merge remote-tracking branch 'upstream/master' into fix_index_merge
qw4990 Nov 6, 2020
c3cd7a8
Merge branch 'master' into fix_index_merge
qw4990 Nov 10, 2020
fedf89e
Merge branch 'master' into fix_index_merge
ti-srebot Nov 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 67 additions & 4 deletions planner/core/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1235,14 +1235,77 @@ func (s *testIntegrationSerialSuite) TestIssue16837(c *C) {
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int,b int,c int,d int,e int,unique key idx_ab(a,b),unique key(c),unique key(d))")
tk.MustQuery("explain select /*+ use_index_merge(t,c,idx_ab) */ * from t where a = 1 or (e = 1 and c = 1)").Check(testkit.Rows(
"TableReader_7 10.00 root data:Selection_6",
"└─Selection_6 10.00 cop[tikv] or(eq(test.t.a, 1), and(eq(test.t.e, 1), eq(test.t.c, 1)))",
" └─TableFullScan_5 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"))
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled"))
"Projection_4 10.00 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e",
"└─IndexMerge_9 0.01 root ",
" ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t, index:idx_ab(a, b) range:[1,1], keep order:false, stats:pseudo",
" ├─IndexRangeScan_6(Build) 1.00 cop[tikv] table:t, index:c(c) range:[1,1], keep order:false, stats:pseudo",
" └─Selection_8(Probe) 0.01 cop[tikv] eq(test.t.e, 1)",
" └─TableRowIDScan_7 11.00 cop[tikv] table:t keep order:false, stats:pseudo"))
tk.MustQuery("show warnings").Check(testkit.Rows())
tk.MustExec("insert into t values (2, 1, 1, 1, 2)")
tk.MustQuery("select /*+ use_index_merge(t,c,idx_ab) */ * from t where a = 1 or (e = 1 and c = 1)").Check(testkit.Rows())
}

func (s *testIntegrationSerialSuite) TestIndexMerge(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (a int, b int, unique key(a), unique key(b))")
tk.MustQuery("desc select /*+ use_index_merge(t) */ * from t where a =1 or (b=1 and b+2>1)").Check(testkit.Rows(
"Projection_4 1.80 root test.t.a, test.t.b",
"└─IndexMerge_9 2.00 root ",
" ├─IndexRangeScan_5(Build) 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false, stats:pseudo",
" ├─Selection_7(Build) 0.80 cop[tikv] gt(plus(test.t.b, 2), 1)",
" │ └─IndexRangeScan_6 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false, stats:pseudo",
" └─TableRowIDScan_8(Probe) 2.00 cop[tikv] table:t keep order:false, stats:pseudo"))
tk.MustQuery("show warnings").Check(testkit.Rows())

tk.MustQuery("desc select /*+ use_index_merge(t) */ * from t where a =1 or (b=1 and length(b)=1)").Check(testkit.Rows(
"Projection_4 1.80 root test.t.a, test.t.b",
"└─IndexMerge_9 2.00 root ",
" ├─IndexRangeScan_5(Build) 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false, stats:pseudo",
" ├─Selection_7(Build) 0.80 cop[tikv] eq(length(cast(test.t.b)), 1)",
" │ └─IndexRangeScan_6 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false, stats:pseudo",
" └─TableRowIDScan_8(Probe) 2.00 cop[tikv] table:t keep order:false, stats:pseudo"))
tk.MustQuery("show warnings").Check(testkit.Rows())

tk.MustQuery("desc select /*+ use_index_merge(t) */ * from t where (a=1 and length(a)=1) or (b=1 and length(b)=1)").Check(testkit.Rows(
"Projection_4 1.60 root test.t.a, test.t.b",
"└─IndexMerge_10 2.00 root ",
" ├─Selection_6(Build) 0.80 cop[tikv] eq(length(cast(test.t.a)), 1)",
" │ └─IndexRangeScan_5 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false, stats:pseudo",
" ├─Selection_8(Build) 0.80 cop[tikv] eq(length(cast(test.t.b)), 1)",
" │ └─IndexRangeScan_7 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false, stats:pseudo",
" └─TableRowIDScan_9(Probe) 2.00 cop[tikv] table:t keep order:false, stats:pseudo"))
tk.MustQuery("show warnings").Check(testkit.Rows())

tk.MustQuery("desc select /*+ use_index_merge(t) */ * from t where (a=1 and length(b)=1) or (b=1 and length(a)=1)").Check(testkit.Rows(
"Projection_4 1.60 root test.t.a, test.t.b",
"└─IndexMerge_9 1.60 root ",
" ├─IndexRangeScan_5(Build) 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false, stats:pseudo",
" ├─IndexRangeScan_6(Build) 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false, stats:pseudo",
" └─Selection_8(Probe) 1.60 cop[tikv] eq(length(cast(test.t.a)), 1), eq(length(cast(test.t.b)), 1)",
" └─TableRowIDScan_7 2.00 cop[tikv] table:t keep order:false, stats:pseudo"))
tk.MustQuery("show warnings").Check(testkit.Rows())
}

func (s *testIntegrationSerialSuite) TestIssue16407(c *C) {
qw4990 marked this conversation as resolved.
Show resolved Hide resolved
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int,b char(100),key(a),key(b(10)))")
tk.MustQuery("explain select /*+ use_index_merge(t) */ * from t where a=10 or b='x'").Check(testkit.Rows(
"Projection_4 19.99 root test.t.a, test.t.b",
"└─IndexMerge_9 0.02 root ",
" ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t, index:a(a) range:[10,10], keep order:false, stats:pseudo",
" ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t, index:b(b) range:[\"x\",\"x\"], keep order:false, stats:pseudo",
" └─Selection_8(Probe) 0.02 cop[tikv] eq(test.t.b, \"x\")",
" └─TableRowIDScan_7 19.99 cop[tikv] table:t keep order:false, stats:pseudo"))
tk.MustQuery("show warnings").Check(testkit.Rows())
tk.MustExec("insert into t values (1, 'xx')")
tk.MustQuery("select /*+ use_index_merge(t) */ * from t where a=10 or b='x'").Check(testkit.Rows())
}

func (s *testIntegrationSuite) TestStreamAggProp(c *C) {
tk := testkit.NewTestKit(c, s.store)

Expand Down
29 changes: 16 additions & 13 deletions planner/core/stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,13 @@ func (ds *DataSource) generateIndexMergeOrPaths() {
}
if len(partialPaths) > 1 {
possiblePath := ds.buildIndexMergeOrPath(partialPaths, i)
sel, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, []expression.Expression{sf}, nil)

accessConds := make([]expression.Expression, 0, len(partialPaths))
for _, p := range partialPaths {
accessConds = append(accessConds, p.AccessConds...)
}
accessDNF := expression.ComposeDNFCondition(ds.ctx, accessConds...)
sel, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, []expression.Expression{accessDNF}, nil)
if err != nil {
logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err))
sel = SelectionFactor
Expand Down Expand Up @@ -473,12 +479,8 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err))
continue
}
if len(path.TableFilters) > 0 || len(path.AccessConds) == 0 {
// If AccessConds is empty or tableFilter is not empty, we ignore the access path.
// Now these conditions are too strict.
// For example, a sql `select * from t where a > 1 or (b < 2 and c > 3)` and table `t` with indexes
// on a and b separately. we can generate a `IndexMergePath` with table filter `a > 1 or (b < 2 and c > 3)`.
// TODO: solve the above case
if len(path.AccessConds) == 0 {
// If AccessConds is empty, we ignore the access path.
continue
}
// If we have point or empty range, just remove other possible paths.
Expand All @@ -502,12 +504,8 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
continue
}
noIntervalRanges := ds.deriveIndexPathStats(path, conditions, true)
if len(path.TableFilters) > 0 || len(path.AccessConds) == 0 {
// If AccessConds is empty or tableFilter is not empty, we ignore the access path.
// Now these conditions are too strict.
// For example, a sql `select * from t where a > 1 or (b < 2 and c > 3)` and table `t` with indexes
// on a and b separately. we can generate a `IndexMergePath` with table filter `a > 1 or (b < 2 and c > 3)`.
// TODO: solve the above case
if len(path.AccessConds) == 0 {
// If AccessConds is empty, we ignore the access path.
continue
}
// If we have empty range, or point range on unique index, just remove other possible paths.
Expand Down Expand Up @@ -554,6 +552,11 @@ func (ds *DataSource) buildIndexMergeOrPath(partialPaths []*util.AccessPath, cur
indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths}
indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[:current]...)
indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[current+1:]...)
for _, path := range partialPaths {
if len(path.TableFilters) > 0 {
indexMergePath.TableFilters = append(indexMergePath.TableFilters, path.TableFilters...)
}
}
qw4990 marked this conversation as resolved.
Show resolved Hide resolved
return indexMergePath
}

Expand Down
4 changes: 2 additions & 2 deletions planner/core/testdata/integration_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,10 +994,10 @@
{
"SQL": "select /*+ use_index_merge(t1 primary, c) */ * from t1 where t1.a = 1 and t1.b = '111' or t1.c = 3.3",
"Plan": [
"IndexMerge_8 1.67 root ",
"IndexMerge_8 2.11 root ",
"├─TableRangeScan_5(Build) 1.00 cop[tikv] table:t1 range:[1 \"111\",1 \"111\"], keep order:false",
"├─IndexRangeScan_6(Build) 1.00 cop[tikv] table:t1, index:c(c) range:[3.3000000000,3.3000000000], keep order:false",
"└─TableRowIDScan_7(Probe) 1.67 cop[tikv] table:t1 keep order:false"
"└─TableRowIDScan_7(Probe) 2.11 cop[tikv] table:t1 keep order:false"
],
"Res": [
"1 111 1.1000000000 11",
Expand Down