From df258800e5c7acd5ab765e29dcd25bb0f4ffee37 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Wed, 22 Feb 2023 15:27:38 +0800 Subject: [PATCH 1/3] don't merge consecutive ranges when building ranges for cnf item --- util/ranger/detacher.go | 26 ++++++++++++++---- util/ranger/ranger_test.go | 31 +++++++++++++++++++++ util/ranger/testdata/ranger_suite_in.json | 7 +++++ util/ranger/testdata/ranger_suite_out.json | 32 ++++++++++++++++++++++ 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/util/ranger/detacher.go b/util/ranger/detacher.go index 3f9f32e2de4d4..f825f9363b53d 100644 --- a/util/ranger/detacher.go +++ b/util/ranger/detacher.go @@ -204,7 +204,14 @@ func extractIndexPointRangesForCNF(sctx sessionctx.Context, conds []expression.E if colSets.Len() == 0 { continue } - res, err := DetachCondAndBuildRangeForIndex(sctx, tmpConds, cols, lengths, rangeMaxSize) + // When we build ranges for the CNF item, we choose not to merge consecutive ranges because we hope to get point + // ranges here. See https://github.com/pingcap/tidb/issues/41572 for more details. + // + // Here is an example. Assume that the index is `idx(a,b,c)` and the condition is `((a,b) in ((1,1),(1,2)) and c = 1`. + // We build ranges for `(a,b) in ((1,1),(1,2))` and get `[1 1, 1 1] [1 2, 1 2]`, which are point ranges and we can + // append `c = 1` to the point ranges. However, if we choose to merge consecutive ranges here, we get `[1 1, 1 2]`, + // which are not point ranges, and we cannot append `c = 1` anymore. + res, err := detachCondAndBuildRangeWithoutMerging(sctx, tmpConds, cols, lengths, rangeMaxSize) if err != nil { return nil, -1, nil, err } @@ -821,11 +828,9 @@ func DetachCondAndBuildRangeForIndex(sctx sessionctx.Context, conditions []expre return d.detachCondAndBuildRangeForCols() } -// DetachCondAndBuildRangeForPartition will detach the index filters from table filters. -// rangeMaxSize is the max memory limit for ranges. O indicates no memory limit. If you ask that all conditions must be used -// for building ranges, set rangeMemQuota to 0 to avoid range fallback. -// The returned values are encapsulated into a struct DetachRangeResult, see its comments for explanation. -func DetachCondAndBuildRangeForPartition(sctx sessionctx.Context, conditions []expression.Expression, cols []*expression.Column, +// detachCondAndBuildRangeWithoutMerging detaches the index filters from table filters and uses them to build ranges. +// When building ranges, it doesn't merge consecutive ranges. +func detachCondAndBuildRangeWithoutMerging(sctx sessionctx.Context, conditions []expression.Expression, cols []*expression.Column, lengths []int, rangeMaxSize int64) (*DetachRangeResult, error) { d := &rangeDetacher{ sctx: sctx, @@ -838,6 +843,15 @@ func DetachCondAndBuildRangeForPartition(sctx sessionctx.Context, conditions []e return d.detachCondAndBuildRangeForCols() } +// DetachCondAndBuildRangeForPartition will detach the index filters from table filters. +// rangeMaxSize is the max memory limit for ranges. O indicates no memory limit. If you ask that all conditions must be used +// for building ranges, set rangeMemQuota to 0 to avoid range fallback. +// The returned values are encapsulated into a struct DetachRangeResult, see its comments for explanation. +func DetachCondAndBuildRangeForPartition(sctx sessionctx.Context, conditions []expression.Expression, cols []*expression.Column, + lengths []int, rangeMaxSize int64) (*DetachRangeResult, error) { + return detachCondAndBuildRangeWithoutMerging(sctx, conditions, cols, lengths, rangeMaxSize) +} + type rangeDetacher struct { sctx sessionctx.Context allConds []expression.Expression diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 244de8e903981..750af759dbaa5 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1000,6 +1000,33 @@ func TestCompIndexMultiColDNF2(t *testing.T) { } } +func TestCompIndexMultiColDNF3(t *testing.T) { + store := testkit.CreateMockStore(t) + + testKit := testkit.NewTestKit(t, store) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists t") + testKit.MustExec("create table t(a varchar(100), b int, c int, d int, index idx(a, b, c))") + testKit.MustExec("insert into t values ('t',1,1,1),('t',1,3,3),('t',2,1,3),('t',2,3,1),('w',0,3,3),('z',0,1,1)") + + var input []string + var output []struct { + SQL string + Plan []string + Result []string + } + rangerSuiteData.LoadTestCases(t, &input, &output) + for i, tt := range input { + testdata.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = testdata.ConvertRowsToStrings(testKit.MustQuery("explain " + tt).Rows()) + output[i].Result = testdata.ConvertRowsToStrings(testKit.MustQuery(tt).Rows()) + }) + testKit.MustQuery("explain " + tt).Check(testkit.Rows(output[i].Plan...)) + testKit.MustQuery(tt).Check(testkit.Rows(output[i].Result...)) + } +} + func TestPrefixIndexMultiColDNF(t *testing.T) { store := testkit.CreateMockStore(t) @@ -2565,3 +2592,7 @@ create table t( require.Equal(t, tt.resultStr, got, fmt.Sprintf("different for expr %s", tt.exprStr)) } } + +func TestConsiderDNFPath(t *testing.T) { + +} diff --git a/util/ranger/testdata/ranger_suite_in.json b/util/ranger/testdata/ranger_suite_in.json index c1641c0f251f7..81e1cfb72ef58 100644 --- a/util/ranger/testdata/ranger_suite_in.json +++ b/util/ranger/testdata/ranger_suite_in.json @@ -59,6 +59,13 @@ "select * from t where ((a = 1 and b = 1) or (a = 2 and b = 2)) and c > 2;" ] }, + { + "name": "TestCompIndexMultiColDNF3", + "cases": [ + "select * from t use index (idx) where ((a = 't' and b = 1) or (a = 't' and b = 2) or (a = 'w' and b = 0)) and c > 2", + "select * from t use index (idx) where ((a = 't' and b = 1) or (a = 't' and b = 2) or (a = 'w' and b = 0)) and d > 2" + ] + }, { "name": "TestPrefixIndexMultiColDNF", "cases": [ diff --git a/util/ranger/testdata/ranger_suite_out.json b/util/ranger/testdata/ranger_suite_out.json index 4a4f0c85e2682..0834f4be40ece 100644 --- a/util/ranger/testdata/ranger_suite_out.json +++ b/util/ranger/testdata/ranger_suite_out.json @@ -333,6 +333,38 @@ } ] }, + { + "Name": "TestCompIndexMultiColDNF3", + "Cases": [ + { + "SQL": "select * from t use index (idx) where ((a = 't' and b = 1) or (a = 't' and b = 2) or (a = 'w' and b = 0)) and c > 2", + "Plan": [ + "IndexLookUp_7 1.00 root ", + "├─IndexRangeScan_5(Build) 1.00 cop[tikv] table:t, index:idx(a, b, c) range:(\"t\" 1 2,\"t\" 1 +inf], (\"t\" 2 2,\"t\" 2 +inf], (\"w\" 0 2,\"w\" 0 +inf], keep order:false, stats:pseudo", + "└─TableRowIDScan_6(Probe) 1.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Result": [ + "t 1 3 3", + "t 2 3 1", + "w 0 3 3" + ] + }, + { + "SQL": "select * from t use index (idx) where ((a = 't' and b = 1) or (a = 't' and b = 2) or (a = 'w' and b = 0)) and d > 2", + "Plan": [ + "IndexLookUp_8 0.10 root ", + "├─IndexRangeScan_5(Build) 0.30 cop[tikv] table:t, index:idx(a, b, c) range:[\"t\" 1,\"t\" 1], [\"t\" 2,\"t\" 2], [\"w\" 0,\"w\" 0], keep order:false, stats:pseudo", + "└─Selection_7(Probe) 0.10 cop[tikv] gt(test.t.d, 2)", + " └─TableRowIDScan_6 0.30 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Result": [ + "t 1 3 3", + "t 2 1 3", + "w 0 3 3" + ] + } + ] + }, { "Name": "TestPrefixIndexMultiColDNF", "Cases": [ From e3c7c51e9505f347537c28e56cbe8100e3e7afff Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 2 Jun 2023 11:00:29 +0800 Subject: [PATCH 2/3] upd --- util/ranger/ranger_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 750af759dbaa5..8e0f55a051c55 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -2592,7 +2592,3 @@ create table t( require.Equal(t, tt.resultStr, got, fmt.Sprintf("different for expr %s", tt.exprStr)) } } - -func TestConsiderDNFPath(t *testing.T) { - -} From 1e1fab29546513dc1608e29e16a85ab2b9b505e2 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 2 Jun 2023 13:34:22 +0800 Subject: [PATCH 3/3] address comment --- util/ranger/ranger_test.go | 4 ++-- util/ranger/testdata/ranger_suite_in.json | 2 +- util/ranger/testdata/ranger_suite_out.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 8e0f55a051c55..63d99ac5c3f9b 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1000,7 +1000,7 @@ func TestCompIndexMultiColDNF2(t *testing.T) { } } -func TestCompIndexMultiColDNF3(t *testing.T) { +func TestIssue41572(t *testing.T) { store := testkit.CreateMockStore(t) testKit := testkit.NewTestKit(t, store) @@ -1023,7 +1023,7 @@ func TestCompIndexMultiColDNF3(t *testing.T) { output[i].Result = testdata.ConvertRowsToStrings(testKit.MustQuery(tt).Rows()) }) testKit.MustQuery("explain " + tt).Check(testkit.Rows(output[i].Plan...)) - testKit.MustQuery(tt).Check(testkit.Rows(output[i].Result...)) + testKit.MustQuery(tt).Sort().Check(testkit.Rows(output[i].Result...)) } } diff --git a/util/ranger/testdata/ranger_suite_in.json b/util/ranger/testdata/ranger_suite_in.json index 81e1cfb72ef58..fbfc86083e661 100644 --- a/util/ranger/testdata/ranger_suite_in.json +++ b/util/ranger/testdata/ranger_suite_in.json @@ -60,7 +60,7 @@ ] }, { - "name": "TestCompIndexMultiColDNF3", + "name": "TestIssue41572", "cases": [ "select * from t use index (idx) where ((a = 't' and b = 1) or (a = 't' and b = 2) or (a = 'w' and b = 0)) and c > 2", "select * from t use index (idx) where ((a = 't' and b = 1) or (a = 't' and b = 2) or (a = 'w' and b = 0)) and d > 2" diff --git a/util/ranger/testdata/ranger_suite_out.json b/util/ranger/testdata/ranger_suite_out.json index 0834f4be40ece..6a86b7b19c4c4 100644 --- a/util/ranger/testdata/ranger_suite_out.json +++ b/util/ranger/testdata/ranger_suite_out.json @@ -334,7 +334,7 @@ ] }, { - "Name": "TestCompIndexMultiColDNF3", + "Name": "TestIssue41572", "Cases": [ { "SQL": "select * from t use index (idx) where ((a = 't' and b = 1) or (a = 't' and b = 2) or (a = 'w' and b = 0)) and c > 2",