Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

expression: pushdown substring/char_length to tikv #30191

Merged
merged 17 commits into from
Dec 30, 2021
Merged

expression: pushdown substring/char_length to tikv #30191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
4d3bd77
expression: pushdown substring
guo-shaoge Nov 27, 2021
b9409b3
Merge branch 'master' of https://github.com/pingcap/tidb into pushdow…
guo-shaoge Nov 27, 2021
316c119
fix case
guo-shaoge Nov 27, 2021
86ef769
fix case
guo-shaoge Nov 27, 2021
3f4bdd7
fix case
guo-shaoge Nov 27, 2021
550e22e
fix comment
guo-shaoge Dec 29, 2021
e5bd47a
Merge branch 'master' into pushdown_substring
guo-shaoge Dec 29, 2021
423b4cc
Merge branch 'master' of https://github.com/pingcap/tidb into pushdow…
guo-shaoge Dec 29, 2021
0933c56
fix case
guo-shaoge Dec 29, 2021
18fbd2e
Merge branch 'pushdown_substring' of github.com:guo-shaoge/tidb into …
guo-shaoge Dec 29, 2021
0a51af1
fix fmt
guo-shaoge Dec 29, 2021
82b1c7a
fix case
guo-shaoge Dec 29, 2021
0cc2440
Merge branch 'master' into pushdown_substring
guo-shaoge Dec 30, 2021
43203b7
Merge branch 'master' into pushdown_substring
guo-shaoge Dec 30, 2021
d1cb8e2
Merge branch 'master' into pushdown_substring
ti-chi-bot Dec 30, 2021
5710767
Merge branch 'master' into pushdown_substring
ti-chi-bot Dec 30, 2021
773814e
Merge branch 'master' into pushdown_substring
ti-chi-bot Dec 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cmd/explaintest/r/index_merge.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -722,11 +722,11 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
└─Selection_12 4433.77 root eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1")
└─IndexMerge_11 5542.21 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─IndexMerge_12 4433.77 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1")
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1;
c1 c2 c3 c4 c5
1 1 1 1 1
Expand Down
15 changes: 7 additions & 8 deletions cmd/explaintest/r/tpch.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1294,14 +1294,13 @@ cntrycode
order by
cntrycode;
id estRows task access object operator info
Sort 1.00 root Column#27
└─Projection 1.00 root Column#27, Column#28, Column#29
└─HashAgg 1.00 root group by:Column#33, funcs:count(1)->Column#28, funcs:sum(Column#31)->Column#29, funcs:firstrow(Column#32)->Column#27
└─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#32, substring(tpch.customer.c_phone, 1, 2)->Column#33
Sort 1.00 root Column#31
└─Projection 1.00 root Column#31, Column#32, Column#33
└─HashAgg 1.00 root group by:Column#37, funcs:count(1)->Column#32, funcs:sum(Column#35)->Column#33, funcs:firstrow(Column#36)->Column#31
└─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#36, substring(tpch.customer.c_phone, 1, 2)->Column#37
└─HashJoin 0.00 root anti semi join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)]
├─TableReader(Build) 75000000.00 root data:TableFullScan
│ └─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false
└─Selection(Probe) 0.00 root in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21")
└─TableReader 0.00 root data:Selection
└─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL)
└─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false
└─TableReader(Probe) 0.00 root data:Selection
└─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL), in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21")
└─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false
23 changes: 22 additions & 1 deletion expression/expr_to_pb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1078,14 +1078,15 @@ func TestExprPushDownToTiKV(t *testing.T) {
exprs := make([]Expression, 0)

//jsonColumn := genColumn(mysql.TypeJSON, 1)
//intColumn := genColumn(mysql.TypeLonglong, 2)
intColumn := genColumn(mysql.TypeLonglong, 2)
//realColumn := genColumn(mysql.TypeDouble, 3)
//decimalColumn := genColumn(mysql.TypeNewDecimal, 4)
stringColumn := genColumn(mysql.TypeString, 5)
//datetimeColumn := genColumn(mysql.TypeDatetime, 6)
binaryStringColumn := genColumn(mysql.TypeString, 7)
binaryStringColumn.RetType.Collate = charset.CollationBin

// Test exprs that cannot be pushed.
function, err := NewFunction(mock.NewContext(), ast.InetAton, types.NewFieldType(mysql.TypeString), stringColumn)
require.NoError(t, err)
exprs = append(exprs, function)
Expand Down Expand Up @@ -1121,6 +1122,26 @@ func TestExprPushDownToTiKV(t *testing.T) {
pushed, remained := PushDownExprs(sc, exprs, client, kv.TiKV)
require.Len(t, pushed, 0)
require.Len(t, remained, len(exprs))

// Test exprs that can be pushed.
exprs = exprs[:0]
pushed = pushed[:0]
remained = remained[:0]

substringRelated := []string{ast.Substr, ast.Substring, ast.Mid}
for _, exprName := range substringRelated {
function, err = NewFunction(mock.NewContext(), exprName, types.NewFieldType(mysql.TypeString), stringColumn, intColumn, intColumn)
require.NoError(t, err)
exprs = append(exprs, function)
}

function, err = NewFunction(mock.NewContext(), ast.CharLength, types.NewFieldType(mysql.TypeString), stringColumn)
require.NoError(t, err)
exprs = append(exprs, function)

pushed, remained = PushDownExprs(sc, exprs, client, kv.TiKV)
require.Len(t, pushed, len(exprs))
require.Len(t, remained, 0)
}

func TestExprOnlyPushDownToTiKV(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion expression/expression.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -967,7 +967,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool {
// string functions.
ast.Length, ast.BitLength, ast.Concat, ast.ConcatWS /*ast.Locate,*/, ast.Replace, ast.ASCII, ast.Hex,
ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field,
InternalFuncFromBinary, InternalFuncToBinary,
InternalFuncFromBinary, InternalFuncToBinary, ast.Mid, ast.Substring, ast.Substr, ast.CharLength,

// json functions.
ast.JSONType, ast.JSONExtract, ast.JSONObject, ast.JSONArray, ast.JSONMerge, ast.JSONSet,
Expand Down
32 changes: 16 additions & 16 deletions expression/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3472,36 +3472,36 @@ func TestExprPushdown(t *testing.T) {
"(4,'511111','611',7,8,9),(5,'611111','711',8,9,10)")

// case 1, index scan without double read, some filters can not be pushed to cop task
rows := tk.MustQuery("explain format = 'brief' select col2, col1 from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Rows()
rows := tk.MustQuery("explain format = 'brief' select col2, col1 from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows()
require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2]))
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

from_base64 and to_base64 has no plan to pushdown for now...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this looks so dirty...

require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\")", fmt.Sprintf("%v", rows[1][4]))
require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\")", fmt.Sprintf("%v", rows[1][4]))
require.Equal(t, "cop[tikv]", fmt.Sprintf("%v", rows[3][2]))
require.Equal(t, "like(test.t.col2, \"5%\", 92)", fmt.Sprintf("%v", rows[3][4]))
tk.MustQuery("select col2, col1 from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("511 411111"))
tk.MustQuery("select count(col2) from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1"))
tk.MustQuery("select col2, col1 from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("511 411111"))
tk.MustQuery("select count(col2) from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1"))

// case 2, index scan without double read, none of the filters can be pushed to cop task
rows = tk.MustQuery("explain format = 'brief' select col1, col2 from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Rows()
rows = tk.MustQuery("explain format = 'brief' select col1, col2 from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows()
require.Equal(t, "root", fmt.Sprintf("%v", rows[0][2]))
require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\"), eq(substr(test.t.col2, 1, 1), \"5\")", fmt.Sprintf("%v", rows[0][4]))
tk.MustQuery("select col1, col2 from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("411111 511"))
tk.MustQuery("select count(col1) from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1"))
require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\"), eq(from_base64(to_base64(substr(test.t.col2, 1, 1))), \"5\")", fmt.Sprintf("%v", rows[0][4]))
tk.MustQuery("select col1, col2 from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("411111 511"))
tk.MustQuery("select count(col1) from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1"))

// case 3, index scan with double read, some filters can not be pushed to cop task
rows = tk.MustQuery("explain format = 'brief' select id from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Rows()
rows = tk.MustQuery("explain format = 'brief' select id from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows()
require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2]))
require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\")", fmt.Sprintf("%v", rows[1][4]))
require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\")", fmt.Sprintf("%v", rows[1][4]))
require.Equal(t, "cop[tikv]", fmt.Sprintf("%v", rows[3][2]))
require.Equal(t, "like(test.t.col2, \"5%\", 92)", fmt.Sprintf("%v", rows[3][4]))
tk.MustQuery("select id from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("3"))
tk.MustQuery("select count(id) from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1"))
tk.MustQuery("select id from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("3"))
tk.MustQuery("select count(id) from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1"))

// case 4, index scan with double read, none of the filters can be pushed to cop task
rows = tk.MustQuery("explain format = 'brief' select id from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Rows()
rows = tk.MustQuery("explain format = 'brief' select id from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows()
require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2]))
require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\"), eq(substr(test.t.col2, 1, 1), \"5\")", fmt.Sprintf("%v", rows[1][4]))
tk.MustQuery("select id from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("3"))
tk.MustQuery("select count(id) from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1"))
require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\"), eq(from_base64(to_base64(substr(test.t.col2, 1, 1))), \"5\")", fmt.Sprintf("%v", rows[1][4]))
tk.MustQuery("select id from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("3"))
tk.MustQuery("select count(id) from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1"))
}
func TestIssue16973(t *testing.T) {
store, clean := testkit.CreateMockStore(t)
Expand Down
30 changes: 14 additions & 16 deletions planner/core/testdata/integration_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2155,14 +2155,13 @@
" ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─StreamAgg(Probe) 1.00 root funcs:min(test.t1.c1)->Column#8, funcs:sum(0)->Column#9, funcs:count(1)->Column#10",
" └─Selection 0.01 root substring(cast(test.t1.c3, var_string(20)), 10)",
" └─IndexMerge 0.01 root ",
" ├─Selection(Build) 1.00 cop[tikv] eq(10, test.t2.c3)",
" │ └─TableRangeScan 1.00 cop[tikv] table:t1 range:[10,10], keep order:false, stats:pseudo",
" ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)",
" │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo",
" └─Selection(Probe) 0.01 cop[tikv] or(and(eq(test.t1.c1, 10), eq(10, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3)))",
" └─TableRowIDScan 9.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
" └─IndexMerge 0.01 root ",
" ├─Selection(Build) 1.00 cop[tikv] eq(10, test.t2.c3)",
" │ └─TableRangeScan 1.00 cop[tikv] table:t1 range:[10,10], keep order:false, stats:pseudo",
" ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)",
" │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo",
" └─Selection(Probe) 0.01 cop[tikv] or(and(eq(test.t1.c1, 10), eq(10, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3))), substring(cast(test.t1.c3, var_string(20)), 10)",
" └─TableRowIDScan 9.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Res": [
"1 1 1",
Expand Down Expand Up @@ -2199,14 +2198,13 @@
" ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─StreamAgg(Probe) 1.00 root funcs:min(test.t1.c1)->Column#8, funcs:sum(0)->Column#9, funcs:count(1)->Column#10",
" └─Selection 3.03 root substring(cast(test.t1.c3, var_string(20)), 10)",
" └─IndexMerge 3.78 root ",
" ├─Selection(Build) 3.33 cop[tikv] eq(test.t1.c1, test.t2.c3)",
" │ └─TableRangeScan 3333.33 cop[tikv] table:t1 range:[10,+inf], keep order:false, stats:pseudo",
" ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)",
" │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo",
" └─Selection(Probe) 3.78 cop[tikv] or(and(ge(test.t1.c1, 10), eq(test.t1.c1, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3)))",
" └─TableRowIDScan 3338.67 cop[tikv] table:t1 keep order:false, stats:pseudo"
" └─IndexMerge 3.03 root ",
" ├─Selection(Build) 3.33 cop[tikv] eq(test.t1.c1, test.t2.c3)",
" │ └─TableRangeScan 3333.33 cop[tikv] table:t1 range:[10,+inf], keep order:false, stats:pseudo",
" ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)",
" │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo",
" └─Selection(Probe) 3.03 cop[tikv] or(and(ge(test.t1.c1, 10), eq(test.t1.c1, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3))), substring(cast(test.t1.c3, var_string(20)), 10)",
" └─TableRowIDScan 3338.67 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Res": [
"1 1 1",
Expand Down