diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index f4d6571fd8fe3..ddce511db13c8 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -722,11 +722,11 @@ c1 c2 c3 c4 c5 explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; id estRows task access object operator info Sort_5 4433.77 root test.t1.c1 -└─Selection_12 4433.77 root eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") - └─IndexMerge_11 5542.21 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─IndexMerge_12 4433.77 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─Selection_11(Probe) 4433.77 cop[tikv] eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") + └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; c1 c2 c3 c4 c5 1 1 1 1 1 diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index c9edd22189514..32afcf2ab4fad 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -1294,14 +1294,13 @@ cntrycode order by cntrycode; id estRows task access object operator info -Sort 1.00 root Column#27 -└─Projection 1.00 root Column#27, Column#28, Column#29 - └─HashAgg 1.00 root group by:Column#33, funcs:count(1)->Column#28, funcs:sum(Column#31)->Column#29, funcs:firstrow(Column#32)->Column#27 - └─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#32, substring(tpch.customer.c_phone, 1, 2)->Column#33 +Sort 1.00 root Column#31 +└─Projection 1.00 root Column#31, Column#32, Column#33 + └─HashAgg 1.00 root group by:Column#37, funcs:count(1)->Column#32, funcs:sum(Column#35)->Column#33, funcs:firstrow(Column#36)->Column#31 + └─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#36, substring(tpch.customer.c_phone, 1, 2)->Column#37 └─HashJoin 0.00 root anti semi join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] ├─TableReader(Build) 75000000.00 root data:TableFullScan │ └─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false - └─Selection(Probe) 0.00 root in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21") - └─TableReader 0.00 root data:Selection - └─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL) - └─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false + └─TableReader(Probe) 0.00 root data:Selection + └─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL), in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21") + └─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 10ad46d6efd06..e02e1676d890b 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1078,7 +1078,7 @@ func TestExprPushDownToTiKV(t *testing.T) { exprs := make([]Expression, 0) //jsonColumn := genColumn(mysql.TypeJSON, 1) - //intColumn := genColumn(mysql.TypeLonglong, 2) + intColumn := genColumn(mysql.TypeLonglong, 2) //realColumn := genColumn(mysql.TypeDouble, 3) //decimalColumn := genColumn(mysql.TypeNewDecimal, 4) stringColumn := genColumn(mysql.TypeString, 5) @@ -1086,6 +1086,7 @@ func TestExprPushDownToTiKV(t *testing.T) { binaryStringColumn := genColumn(mysql.TypeString, 7) binaryStringColumn.RetType.Collate = charset.CollationBin + // Test exprs that cannot be pushed. function, err := NewFunction(mock.NewContext(), ast.InetAton, types.NewFieldType(mysql.TypeString), stringColumn) require.NoError(t, err) exprs = append(exprs, function) @@ -1121,6 +1122,26 @@ func TestExprPushDownToTiKV(t *testing.T) { pushed, remained := PushDownExprs(sc, exprs, client, kv.TiKV) require.Len(t, pushed, 0) require.Len(t, remained, len(exprs)) + + // Test exprs that can be pushed. + exprs = exprs[:0] + pushed = pushed[:0] + remained = remained[:0] + + substringRelated := []string{ast.Substr, ast.Substring, ast.Mid} + for _, exprName := range substringRelated { + function, err = NewFunction(mock.NewContext(), exprName, types.NewFieldType(mysql.TypeString), stringColumn, intColumn, intColumn) + require.NoError(t, err) + exprs = append(exprs, function) + } + + function, err = NewFunction(mock.NewContext(), ast.CharLength, types.NewFieldType(mysql.TypeString), stringColumn) + require.NoError(t, err) + exprs = append(exprs, function) + + pushed, remained = PushDownExprs(sc, exprs, client, kv.TiKV) + require.Len(t, pushed, len(exprs)) + require.Len(t, remained, 0) } func TestExprOnlyPushDownToTiKV(t *testing.T) { diff --git a/expression/expression.go b/expression/expression.go index d27ccb48a07a9..82c2ea4511d94 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -967,7 +967,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool { // string functions. ast.Length, ast.BitLength, ast.Concat, ast.ConcatWS /*ast.Locate,*/, ast.Replace, ast.ASCII, ast.Hex, ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field, - InternalFuncFromBinary, InternalFuncToBinary, + InternalFuncFromBinary, InternalFuncToBinary, ast.Mid, ast.Substring, ast.Substr, ast.CharLength, // json functions. ast.JSONType, ast.JSONExtract, ast.JSONObject, ast.JSONArray, ast.JSONMerge, ast.JSONSet, diff --git a/expression/integration_test.go b/expression/integration_test.go index 9477c5524d0c6..48284410b94ca 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -3472,36 +3472,36 @@ func TestExprPushdown(t *testing.T) { "(4,'511111','611',7,8,9),(5,'611111','711',8,9,10)") // case 1, index scan without double read, some filters can not be pushed to cop task - rows := tk.MustQuery("explain format = 'brief' select col2, col1 from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Rows() + rows := tk.MustQuery("explain format = 'brief' select col2, col1 from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\")", fmt.Sprintf("%v", rows[1][4])) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\")", fmt.Sprintf("%v", rows[1][4])) require.Equal(t, "cop[tikv]", fmt.Sprintf("%v", rows[3][2])) require.Equal(t, "like(test.t.col2, \"5%\", 92)", fmt.Sprintf("%v", rows[3][4])) - tk.MustQuery("select col2, col1 from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("511 411111")) - tk.MustQuery("select count(col2) from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + tk.MustQuery("select col2, col1 from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("511 411111")) + tk.MustQuery("select count(col2) from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) // case 2, index scan without double read, none of the filters can be pushed to cop task - rows = tk.MustQuery("explain format = 'brief' select col1, col2 from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Rows() + rows = tk.MustQuery("explain format = 'brief' select col1, col2 from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[0][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\"), eq(substr(test.t.col2, 1, 1), \"5\")", fmt.Sprintf("%v", rows[0][4])) - tk.MustQuery("select col1, col2 from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("411111 511")) - tk.MustQuery("select count(col1) from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\"), eq(from_base64(to_base64(substr(test.t.col2, 1, 1))), \"5\")", fmt.Sprintf("%v", rows[0][4])) + tk.MustQuery("select col1, col2 from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("411111 511")) + tk.MustQuery("select count(col1) from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) // case 3, index scan with double read, some filters can not be pushed to cop task - rows = tk.MustQuery("explain format = 'brief' select id from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Rows() + rows = tk.MustQuery("explain format = 'brief' select id from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\")", fmt.Sprintf("%v", rows[1][4])) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\")", fmt.Sprintf("%v", rows[1][4])) require.Equal(t, "cop[tikv]", fmt.Sprintf("%v", rows[3][2])) require.Equal(t, "like(test.t.col2, \"5%\", 92)", fmt.Sprintf("%v", rows[3][4])) - tk.MustQuery("select id from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("3")) - tk.MustQuery("select count(id) from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + tk.MustQuery("select id from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("3")) + tk.MustQuery("select count(id) from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) // case 4, index scan with double read, none of the filters can be pushed to cop task - rows = tk.MustQuery("explain format = 'brief' select id from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Rows() + rows = tk.MustQuery("explain format = 'brief' select id from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\"), eq(substr(test.t.col2, 1, 1), \"5\")", fmt.Sprintf("%v", rows[1][4])) - tk.MustQuery("select id from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("3")) - tk.MustQuery("select count(id) from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\"), eq(from_base64(to_base64(substr(test.t.col2, 1, 1))), \"5\")", fmt.Sprintf("%v", rows[1][4])) + tk.MustQuery("select id from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("3")) + tk.MustQuery("select count(id) from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) } func TestIssue16973(t *testing.T) { store, clean := testkit.CreateMockStore(t) diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 262949b2df84e..5efa0586a2cd9 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -2155,14 +2155,13 @@ " ├─TableReader(Build) 10000.00 root data:TableFullScan", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─StreamAgg(Probe) 1.00 root funcs:min(test.t1.c1)->Column#8, funcs:sum(0)->Column#9, funcs:count(1)->Column#10", - " └─Selection 0.01 root substring(cast(test.t1.c3, var_string(20)), 10)", - " └─IndexMerge 0.01 root ", - " ├─Selection(Build) 1.00 cop[tikv] eq(10, test.t2.c3)", - " │ └─TableRangeScan 1.00 cop[tikv] table:t1 range:[10,10], keep order:false, stats:pseudo", - " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", - " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", - " └─Selection(Probe) 0.01 cop[tikv] or(and(eq(test.t1.c1, 10), eq(10, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3)))", - " └─TableRowIDScan 9.00 cop[tikv] table:t1 keep order:false, stats:pseudo" + " └─IndexMerge 0.01 root ", + " ├─Selection(Build) 1.00 cop[tikv] eq(10, test.t2.c3)", + " │ └─TableRangeScan 1.00 cop[tikv] table:t1 range:[10,10], keep order:false, stats:pseudo", + " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", + " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", + " └─Selection(Probe) 0.01 cop[tikv] or(and(eq(test.t1.c1, 10), eq(10, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3))), substring(cast(test.t1.c3, var_string(20)), 10)", + " └─TableRowIDScan 9.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "Res": [ "1 1 1", @@ -2199,14 +2198,13 @@ " ├─TableReader(Build) 10000.00 root data:TableFullScan", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─StreamAgg(Probe) 1.00 root funcs:min(test.t1.c1)->Column#8, funcs:sum(0)->Column#9, funcs:count(1)->Column#10", - " └─Selection 3.03 root substring(cast(test.t1.c3, var_string(20)), 10)", - " └─IndexMerge 3.78 root ", - " ├─Selection(Build) 3.33 cop[tikv] eq(test.t1.c1, test.t2.c3)", - " │ └─TableRangeScan 3333.33 cop[tikv] table:t1 range:[10,+inf], keep order:false, stats:pseudo", - " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", - " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", - " └─Selection(Probe) 3.78 cop[tikv] or(and(ge(test.t1.c1, 10), eq(test.t1.c1, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3)))", - " └─TableRowIDScan 3338.67 cop[tikv] table:t1 keep order:false, stats:pseudo" + " └─IndexMerge 3.03 root ", + " ├─Selection(Build) 3.33 cop[tikv] eq(test.t1.c1, test.t2.c3)", + " │ └─TableRangeScan 3333.33 cop[tikv] table:t1 range:[10,+inf], keep order:false, stats:pseudo", + " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", + " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", + " └─Selection(Probe) 3.03 cop[tikv] or(and(ge(test.t1.c1, 10), eq(test.t1.c1, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3))), substring(cast(test.t1.c3, var_string(20)), 10)", + " └─TableRowIDScan 3338.67 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "Res": [ "1 1 1",