Skip to content

Commit

Permalink
util: skip always false DNF (#51901) (#53413)
Browse files Browse the repository at this point in the history 
close #40997
  • Loading branch information
@ti-chi-bot
ti-chi-bot authored May 21, 2024
1 parent 6cd207e commit 2f7f8e8
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 18 deletions.
12 changes: 6 additions & 6 deletions pkg/executor/explainfor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,15 +546,15 @@ func TestIssue28259(t *testing.T) {
ps = []*util.ProcessInfo{tkProcess}
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
res = tk.MustQuery("explain for connection " + strconv.FormatUint(tkProcess.ID, 10))
require.Len(t, res.Rows(), 3)
require.Regexp(t, ".*Selection.*", res.Rows()[1][0])
require.Regexp(t, ".*IndexFullScan.*", res.Rows()[2][0])
require.Len(t, res.Rows(), 2)
require.Regexp(t, ".*IndexReader.*", res.Rows()[0][0])
require.Regexp(t, ".*IndexRangeScan.*", res.Rows()[1][0])

res = tk.MustQuery("explain format = 'brief' select col1 from UK_GCOL_VIRTUAL_18588 use index(UK_COL1) " +
"where col1 between -1696020282760139948 and -2619168038882941276 or col1 < -4004648990067362699;")
require.Len(t, res.Rows(), 3)
require.Regexp(t, ".*Selection.*", res.Rows()[1][0])
require.Regexp(t, ".*IndexFullScan.*", res.Rows()[2][0])
require.Len(t, res.Rows(), 2)
require.Regexp(t, ".*IndexReader.*", res.Rows()[0][0])
require.Regexp(t, ".*IndexRangeScan.*", res.Rows()[1][0])
res = tk.MustQuery("explain format = 'brief' select col1 from UK_GCOL_VIRTUAL_18588 use index(UK_COL1) " +
"where col1 between 5516958330762833919 and 8551969118506051323 or col1 < 2887622822023883594;")
require.Len(t, res.Rows(), 2)
Expand Down
17 changes: 8 additions & 9 deletions pkg/executor/test/indexmergereadtest/index_merge_reader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1331,19 +1331,18 @@ func TestIndexMergeIssue49605(t *testing.T) {
testkit.Rows("Projection 2666.67 root istrue(or(not(ge(test.t.i, j筧8)), not(eq(test.t.i, 暈lH忧ll6))))->Column#11, Column#10, quote(test.t.i)->Column#12",
"└─StreamAgg 2666.67 root group by:test.t.i, funcs:max(test.t.e)->Column#10, funcs:firstrow(test.t.i)->test.t.i",
" └─Sort 3333.33 root test.t.i",
" └─IndexMerge 3333.33 root type: union",
" ├─TableRangeScan(Build) 3333.33 cop[tikv] table:t, partition:p0 range:(240817,+inf], keep order:false, stats:pseudo",
" ├─IndexFullScan(Build) 0.00 cop[tikv] table:t, partition:p0, index:idx_25(h, i, e) keep order:false, stats:pseudo",
" └─TableRowIDScan(Probe) 3333.33 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo"))
" └─TableReader 3333.33 root data:Selection",
" └─Selection 3333.33 cop[tikv] or(gt(test.t.h, 240817), and(ge(test.t.i, \"WVz\"), le(test.t.i, \"G#駧褉ZC領*lov\")))",
" └─TableFullScan 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo"))
tk.MustQuery("select count(*) from (SELECT /*+ AGG_TO_COP() STREAM_AGG()*/ (NOT (`t`.`i`>=_UTF8MB4'j筧8') OR NOT (`t`.`i`=_UTF8MB4'暈lH忧ll6')) IS TRUE,MAX(`t`.`e`) AS `r0`,QUOTE(`t`.`i`) AS `r1` FROM `t` WHERE `t`.`h`>240817 OR `t`.`i` BETWEEN _UTF8MB4'WVz' AND _UTF8MB4'G#駧褉ZC領*lov' GROUP BY `t`.`i`) derived;").Check(
testkit.Rows("16"))
tk.MustQuery("explain format='brief' SELECT /*+ AGG_TO_COP() */ (NOT (`t`.`i`>=_UTF8MB4'j筧8') OR NOT (`t`.`i`=_UTF8MB4'暈lH忧ll6')) IS TRUE,MAX(`t`.`e`) AS `r0`,QUOTE(`t`.`i`) AS `r1` FROM `t` WHERE `t`.`h`>240817 OR `t`.`i` BETWEEN _UTF8MB4'WVz' AND _UTF8MB4'G#駧褉ZC領*lov' GROUP BY `t`.`i`;").Check(
testkit.Rows("Projection 2666.67 root istrue(or(not(ge(test.t.i, j筧8)), not(eq(test.t.i, 暈lH忧ll6))))->Column#11, Column#10, quote(test.t.i)->Column#12",
"└─HashAgg 2666.67 root group by:test.t.i, funcs:max(test.t.e)->Column#10, funcs:firstrow(test.t.i)->test.t.i",
" └─IndexMerge 3333.33 root type: union",
" ├─TableRangeScan(Build) 3333.33 cop[tikv] table:t, partition:p0 range:(240817,+inf], keep order:false, stats:pseudo",
" ├─IndexFullScan(Build) 0.00 cop[tikv] table:t, partition:p0, index:idx_25(h, i, e) keep order:false, stats:pseudo",
" └─TableRowIDScan(Probe) 3333.33 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo"))
"└─HashAgg 2666.67 root group by:test.t.i, funcs:max(Column#15)->Column#10, funcs:firstrow(test.t.i)->test.t.i",
" └─TableReader 2666.67 root data:HashAgg",
" └─HashAgg 2666.67 cop[tikv] group by:test.t.i, funcs:max(test.t.e)->Column#15",
" └─Selection 3333.33 cop[tikv] or(gt(test.t.h, 240817), and(ge(test.t.i, \"WVz\"), le(test.t.i, \"G#駧褉ZC領*lov\")))",
" └─TableFullScan 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo"))
tk.MustQuery("select count(*) from (SELECT /*+ AGG_TO_COP() */ (NOT (`t`.`i`>=_UTF8MB4'j筧8') OR NOT (`t`.`i`=_UTF8MB4'暈lH忧ll6')) IS TRUE,MAX(`t`.`e`) AS `r0`,QUOTE(`t`.`i`) AS `r1` FROM `t` WHERE `t`.`h`>240817 OR `t`.`i` BETWEEN _UTF8MB4'WVz' AND _UTF8MB4'G#駧褉ZC領*lov' GROUP BY `t`.`i`) derived;").Check(
testkit.Rows("16"))
}
12 changes: 9 additions & 3 deletions pkg/util/ranger/detacher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -711,7 +711,7 @@ func ExtractEqAndInCondition(sctx sessionctx.Context, conditions []expression.Ex
return accesses, filters, newConditions, columnValues, false
}

// detachDNFCondAndBuildRangeForIndex will detach the index filters from table filters when it's a DNF.
// detachDNFCondAndBuildRangeForIndex will detach the index filters from table filters when it's a DNF(Disjunctive Normal Form).
// We will detach the conditions of every DNF items, then compose them to a DNF.
func (d *rangeDetacher) detachDNFCondAndBuildRangeForIndex(condition *expression.ScalarFunction, newTpSlice []*types.FieldType) (Ranges, []expression.Expression, []*valueInfo, bool, error) {
firstColumnChecker := &conditionChecker{
Expand All @@ -723,8 +723,10 @@ func (d *rangeDetacher) detachDNFCondAndBuildRangeForIndex(condition *expression
rb := builder{sctx: d.sctx}
dnfItems := expression.FlattenDNFConditions(condition)
newAccessItems := make([]expression.Expression, 0, len(dnfItems))
var totalRanges Ranges
var totalRangesMemUsage int64
var (
totalRanges Ranges
totalRangesMemUsage int64
)
columnValues := make([]*valueInfo, len(d.cols))
hasResidual := false
for i, item := range dnfItems {
Expand All @@ -736,6 +738,10 @@ func (d *rangeDetacher) detachDNFCondAndBuildRangeForIndex(condition *expression
return nil, nil, nil, false, err
}
ranges := res.Ranges
// If DNF item always false, we can return ignore this DNF item.
if len(ranges) == 0 {
continue
}
accesses = res.AccessConds
filters = res.RemainedConds
if len(accesses) == 0 {
Expand Down
40 changes: 40 additions & 0 deletions pkg/util/ranger/ranger_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2274,3 +2274,43 @@ create table t(
require.Equal(t, tt.resultStr, got, fmt.Sprintf("different for expr %s", tt.exprStr))
}
}

func TestIssue40997(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
err := tk.ExecToErr(`
CREATE TABLE t71706696 (
dt char(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
db_id bigint(20) NOT NULL,
tbl_id bigint(20) NOT NULL,
db_name varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
UNIQUE KEY dt_2 (dt, db_id, tbl_id)
);
`)
require.NoError(t, err)
tk.MustQuery(`
EXPLAIN
SELECT *
FROM t71706696 FORCE INDEX(dt_2)
WHERE (
(
dt = '20210112'
AND db_id = '62812'
AND tbl_id > '228892694'
) OR (
dt = '20210112'
AND db_id = '62813'
AND tbl_id <= '226785696'
) OR (
dt = '20210112'
AND db_id > '62812'
AND db_id < '62813'
)
)
`).Check(testkit.Rows(
"IndexLookUp_7 0.67 root ",
"├─IndexRangeScan_5(Build) 0.67 cop[tikv] table:t71706696, index:dt_2(dt, db_id, tbl_id) range:(\"20210112\" 62812 228892694,\"20210112\" 62812 +inf], [\"20210112\" 62813 -inf,\"20210112\" 62813 226785696], keep order:false, stats:pseudo",
"└─TableRowIDScan_6(Probe) 0.67 cop[tikv] table:t71706696 keep order:false, stats:pseudo",
))
}

0 comments on commit 2f7f8e8

Please sign in to comment.