Skip to content

Commit

Permalink
*: hotfix support use like to build range for new collation columns (#…
Browse files Browse the repository at this point in the history 
…49191)

ref #48181
  • Loading branch information
@time-and-fate
time-and-fate authored Dec 11, 2023
1 parent ada6ff2 commit 7a9dad0
Show file tree
Hide file tree
Showing 18 changed files with 1,821 additions and 259 deletions.
4 changes: 2 additions & 2 deletions cmd/explaintest/r/explain_easy.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -797,14 +797,14 @@ create table t(a binary(16) not null, b varchar(2) default null, c varchar(100)
explain format = 'brief' select * from t where a=x'FA34E1093CB428485734E3917F000000' and b='xb';
id estRows task access object operator info
IndexLookUp 0.10 root
├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:a(a, b) range:[0xFA34E1093CB428485734E3917F000000 "xb",0xFA34E1093CB428485734E3917F000000 "xb"], keep order:false, stats:pseudo
├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:a(a, b) range:["\xfa4\xe1\t<\xb4(HW4\xe3\x91\x7f\x00\x00\x00" "xb","\xfa4\xe1\t<\xb4(HW4\xe3\x91\x7f\x00\x00\x00" "xb"], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.10 cop[tikv] table:t keep order:false, stats:pseudo
explain format = 'brief' update t set c = 'ssss' where a=x'FA34E1093CB428485734E3917F000000' and b='xb';
id estRows task access object operator info
Update N/A root N/A
└─SelectLock 0.10 root for update 0
└─IndexLookUp 0.10 root
├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:a(a, b) range:[0xFA34E1093CB428485734E3917F000000 "xb",0xFA34E1093CB428485734E3917F000000 "xb"], keep order:false, stats:pseudo
├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:a(a, b) range:["\xfa4\xe1\t<\xb4(HW4\xe3\x91\x7f\x00\x00\x00" "xb","\xfa4\xe1\t<\xb4(HW4\xe3\x91\x7f\x00\x00\x00" "xb"], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.10 cop[tikv] table:t keep order:false, stats:pseudo
drop table if exists t;
create table t(a int, b int);
Expand Down
3 changes: 2 additions & 1 deletion cmd/explaintest/r/explain_generate_column_substitute.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,8 @@ id estRows task access object operator info
StreamAgg 1.00 root funcs:count(Column#6)->Column#4
└─IndexReader 1.00 root index:StreamAgg
└─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#6
└─IndexRangeScan 250.00 cop[tikv] table:tbl1, index:expression_index(md5(`s`)) range:["02e74f10e0327ad868d138f2b4fdd6f","02e74f10e0327ad868d138f2b4fdd6g"), keep order:false, stats:pseudo
└─Selection 250.00 cop[tikv] like(md5(cast(test.tbl1.s, var_string(20))), "02e74f10e0327ad868d138f2b4fdd6f%", 92)
└─IndexRangeScan 250.00 cop[tikv] table:tbl1, index:expression_index(md5(`s`)) range:["02e74f10e0327ad868d138f2b4fdd6f","02e74f10e0327ad868d138f2b4fdd6g"), keep order:false, stats:pseudo
select count(*) from tbl1 use index() where md5(s) like '02e74f10e0327ad868d138f2b4fdd6f%';
count(*)
64
Expand Down
1,085 changes: 1,085 additions & 0 deletions cmd/explaintest/r/range_scan_for_like.result
View file

Large diffs are not rendered by default.

248 changes: 248 additions & 0 deletions cmd/explaintest/t/range_scan_for_like.test
View file

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions executor/point_get.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,11 +540,13 @@ func EncodeUniqueIndexValuesForKey(ctx sessionctx.Context, tblInfo *model.TableI
colInfo := tblInfo.Columns[idxInfo.Columns[i].Offset]
// table.CastValue will append 0x0 if the string value's length is smaller than the BINARY column's length.
// So we don't use CastValue for string value for now.
// TODO: merge two if branch.
// TODO: The first if branch should have been removed, because the functionality of set the collation of the datum
// have been moved to util/ranger (normal path) and getNameValuePairs/getPointGetValue (fast path). But this change
// will be cherry-picked to a hotfix, so we choose to be a bit conservative and keep this for now.
if colInfo.GetType() == mysql.TypeString || colInfo.GetType() == mysql.TypeVarString || colInfo.GetType() == mysql.TypeVarchar {
var str string
str, err = idxVals[i].ToString()
idxVals[i].SetString(str, colInfo.FieldType.GetCollate())
idxVals[i].SetString(str, idxVals[i].Collation())
} else if colInfo.GetType() == mysql.TypeEnum && (idxVals[i].Kind() == types.KindString || idxVals[i].Kind() == types.KindBytes || idxVals[i].Kind() == types.KindBinaryLiteral) {
var str string
var e types.Enum
Expand Down
2 changes: 1 addition & 1 deletion planner/core/casetest/testdata/derive_topn_from_window_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,7 @@
" └─TableReader 1.11 root data:TopN",
" └─TopN 1.11 cop[tikv] partition by test.customer.primary_key, test.customer.secondary_key order by test.customer.c_timestamp, offset:0, count:10",
" └─Selection 1.11 cop[tikv] ge(test.customer.c_timestamp, 1661883508511000000)",
" └─TableRangeScan 33.33 cop[tikv] table:customer range:[0x0002 0x0001,0x0002 +inf], keep order:false, stats:pseudo"
" └─TableRangeScan 33.33 cop[tikv] table:customer range:[\"\\x00\\x02\" \"\\x00\\x01\",\"\\x00\\x02\" +inf], keep order:false, stats:pseudo"
],
"Res": null
},
Expand Down
66 changes: 34 additions & 32 deletions planner/core/casetest/testdata/plan_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2787,11 +2787,11 @@
},
{
"SQL": "select a from t where c_str like ''",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"\",\"\"]])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"\",\"\"]]->Sel([like(test.t.c_str, , 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc'",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc\",\"abc\"]])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc\",\"abc\"]]->Sel([like(test.t.c_str, abc, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str not like 'abc'",
Expand All @@ -2807,43 +2807,43 @@
},
{
"SQL": "select a from t where c_str like 'abc%'",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc\",\"abd\")])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc\",\"abd\")]->Sel([like(test.t.c_str, abc%, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc_'",
"Best": "IndexReader(Index(t.c_d_e_str)[(\"abc\",\"abd\")]->Sel([like(test.t.c_str, abc_, 92)]))->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc\",\"abd\")]->Sel([like(test.t.c_str, abc_, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc%af'",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc\",\"abd\")]->Sel([like(test.t.c_str, abc%af, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc\\_' escape ''",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc_\"]])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc_\"]]->Sel([like(test.t.c_str, abc\\_, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc\\_'",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc_\"]])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc_\"]]->Sel([like(test.t.c_str, abc\\_, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc\\\\_'",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc_\"]])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc_\"]]->Sel([like(test.t.c_str, abc\\_, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc\\_%'",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc`\")])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc`\")]->Sel([like(test.t.c_str, abc\\_%, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc=_%' escape '='",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc`\")])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc`\")]->Sel([like(test.t.c_str, abc=_%, 61)]))->Projection"
},
{
"SQL": "select a from t where c_str like 'abc\\__'",
"Best": "IndexReader(Index(t.c_d_e_str)[(\"abc_\",\"abc`\")]->Sel([like(test.t.c_str, abc\\__, 92)]))->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"abc_\",\"abc`\")]->Sel([like(test.t.c_str, abc\\__, 92)]))->Projection"
},
{
"SQL": "select a from t where c_str like 123",
"Best": "IndexReader(Index(t.c_d_e_str)[[\"123\",\"123\"]])->Projection"
"Best": "IndexReader(Index(t.c_d_e_str)[[\"123\",\"123\"]]->Sel([like(test.t.c_str, 123, 92)]))->Projection"
},
{
"SQL": "select a from t where c = 1.9 and d > 3",
Expand Down Expand Up @@ -8006,21 +8006,21 @@
" │ └─StreamAgg 249.75 root funcs:max(test.tc.id)->Column#14",
" │ └─TopN 62.38 root test.tc.id:desc, offset:0, count:1",
" │ └─IndexLookUp 62.38 root ",
" │ ├─Selection(Build) 62.44 cop[tikv] eq(test.ta.name, test.tc.name)",
" │ ├─Selection(Build) 62.38 cop[tikv] eq(test.ta.name, test.tc.name), like(test.tc.name, \"chad99%\", 92)",
" │ │ └─IndexRangeScan 62437.50 cop[tikv] table:tc, index:idx_tc_name(name) range:[\"chad99\",\"chad9:\"), keep order:false, stats:pseudo",
" │ └─TopN(Probe) 62.38 cop[tikv] test.tc.id:desc, offset:0, count:1",
" │ └─Selection 62.38 cop[tikv] not(isnull(test.tc.id))",
" │ └─TableRowIDScan 62.44 cop[tikv] table:tc keep order:false, stats:pseudo",
" │ └─TableRowIDScan 62.38 cop[tikv] table:tc keep order:false, stats:pseudo",
" └─Selection(Probe) 199.80 root gt(Column#19, 100)",
" └─MaxOneRow 249.75 root ",
" └─StreamAgg 249.75 root funcs:max(test.td.id)->Column#19",
" └─Limit 62.38 root offset:0, count:1",
" └─Projection 62.38 root test.td.id, test.td.name",
" └─IndexLookUp 62.38 root ",
" ├─Selection(Build) 2495.00 cop[tikv] eq(test.ta.id, test.td.id)",
" │ └─IndexFullScan 2495002.50 cop[tikv] table:td, index:idx_tc_id(id) keep order:true, desc, stats:pseudo",
" └─Selection(Probe) 62.38 cop[tikv] like(test.td.name, \"chad999%\", 92)",
" └─TableRowIDScan 2495.00 cop[tikv] table:td keep order:false, stats:pseudo"
" └─TopN 62.38 root test.td.id:desc, offset:0, count:1",
" └─IndexLookUp 62.38 root ",
" ├─Selection(Build) 1560.94 cop[tikv] like(test.td.name, \"chad999%\", 92)",
" │ └─IndexRangeScan 62437.50 cop[tikv] table:td, index:idx_tc_name(name) range:[\"chad999\",\"chad99:\"), keep order:false, stats:pseudo",
" └─TopN(Probe) 62.38 cop[tikv] test.td.id:desc, offset:0, count:1",
" └─Selection 62.38 cop[tikv] eq(test.ta.id, test.td.id), not(isnull(test.td.id))",
" └─TableRowIDScan 1560.94 cop[tikv] table:td keep order:false, stats:pseudo"
],
"Result": null,
"Warning": null
Expand All @@ -8034,29 +8034,31 @@
" │ ├─Apply(Build) 10000.00 root CARTESIAN semi join",
" │ │ ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ │ │ └─TableFullScan 10000.00 cop[tikv] table:ta keep order:false, stats:pseudo",
" │ │ └─TableReader(Probe) 2500.00 root data:Selection",
" │ │ └─Selection 2500.00 cop[tikv] eq(test.ta.code, test.tb.code), like(test.tb.name, \"chad9%\", 92)",
" │ │ └─TableFullScan 100000000.00 cop[tikv] table:tb keep order:false, stats:pseudo",
" │ │ └─IndexLookUp(Probe) 2500.00 root ",
" │ │ ├─Selection(Build) 62500.00 cop[tikv] like(test.tb.name, \"chad9%\", 92)",
" │ │ │ └─IndexRangeScan 2500000.00 cop[tikv] table:tb, index:idx_tb_name(name) range:[\"chad9\",\"chad:\"), keep order:false, stats:pseudo",
" │ │ └─Selection(Probe) 2500.00 cop[tikv] eq(test.ta.code, test.tb.code)",
" │ │ └─TableRowIDScan 62500.00 cop[tikv] table:tb keep order:false, stats:pseudo",
" │ └─Selection(Probe) 8000.00 root gt(Column#14, 100)",
" │ └─MaxOneRow 10000.00 root ",
" │ └─StreamAgg 10000.00 root funcs:max(test.tc.id)->Column#14",
" │ └─TopN 2497.50 root test.tc.id:desc, offset:0, count:1",
" │ └─IndexLookUp 2497.50 root ",
" │ ├─Selection(Build) 2500.00 cop[tikv] eq(test.ta.name, test.tc.name)",
" │ ├─Selection(Build) 2497.50 cop[tikv] eq(test.ta.name, test.tc.name), like(test.tc.name, \"chad99%\", 92)",
" │ │ └─IndexRangeScan 2500000.00 cop[tikv] table:tc, index:idx_tc_name(name) range:[\"chad99\",\"chad9:\"), keep order:false, stats:pseudo",
" │ └─TopN(Probe) 2497.50 cop[tikv] test.tc.id:desc, offset:0, count:1",
" │ └─Selection 2497.50 cop[tikv] not(isnull(test.tc.id))",
" │ └─TableRowIDScan 2500.00 cop[tikv] table:tc keep order:false, stats:pseudo",
" │ └─TableRowIDScan 2497.50 cop[tikv] table:tc keep order:false, stats:pseudo",
" └─Selection(Probe) 8000.00 root gt(Column#19, 100)",
" └─MaxOneRow 10000.00 root ",
" └─StreamAgg 10000.00 root funcs:max(test.td.id)->Column#19",
" └─Limit 2497.50 root offset:0, count:1",
" └─Projection 2497.50 root test.td.id, test.td.name",
" └─IndexLookUp 2497.50 root ",
" ├─Selection(Build) 99900.00 cop[tikv] eq(test.ta.id, test.td.id)",
" │ └─IndexFullScan 99900000.00 cop[tikv] table:td, index:idx_tc_id(id) keep order:true, desc, stats:pseudo",
" └─Selection(Probe) 2497.50 cop[tikv] like(test.td.name, \"chad999%\", 92)",
" └─TableRowIDScan 99900.00 cop[tikv] table:td keep order:false, stats:pseudo"
" └─TopN 2497.50 root test.td.id:desc, offset:0, count:1",
" └─IndexLookUp 2497.50 root ",
" ├─Selection(Build) 62500.00 cop[tikv] like(test.td.name, \"chad999%\", 92)",
" │ └─IndexRangeScan 2500000.00 cop[tikv] table:td, index:idx_tc_name(name) range:[\"chad999\",\"chad99:\"), keep order:false, stats:pseudo",
" └─TopN(Probe) 2497.50 cop[tikv] test.td.id:desc, offset:0, count:1",
" └─Selection 2497.50 cop[tikv] eq(test.ta.id, test.td.id), not(isnull(test.td.id))",
" └─TableRowIDScan 62500.00 cop[tikv] table:td keep order:false, stats:pseudo"
],
"Result": null,
"Warning": null
Expand Down
14 changes: 7 additions & 7 deletions planner/core/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3535,13 +3535,13 @@ func TestIssues29711(t *testing.T) {
"`col_251` enum('Alice','Bob','Charlie','David') COLLATE utf8_unicode_ci NOT NULL DEFAULT 'Charlie'," +
"PRIMARY KEY (`col_251`,`col_250`(1)) NONCLUSTERED);")
tk.MustQuery("explain format=brief " +
"select col_250,col_251 from tbl_29711 where col_251 between 'Bob' and 'David' order by col_250,col_251 limit 6;").
"select col_250,col_251 from tbl_29711 use index (primary) where col_251 between 'Bob' and 'David' order by col_250,col_251 limit 6;").
Check(testkit.Rows(
"TopN 6.00 root test.tbl_29711.col_250, test.tbl_29711.col_251, offset:0, count:6",
"└─IndexLookUp 6.00 root ",
" ├─IndexRangeScan(Build) 30.00 cop[tikv] table:tbl_29711, index:PRIMARY(col_251, col_250) range:[\"Bob\",\"Bob\"], [\"Charlie\",\"Charlie\"], [\"David\",\"David\"], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 250.00 cop[tikv] table:tbl_29711, index:PRIMARY(col_251, col_250) range:[\"Bob\",\"David\"], keep order:false, stats:pseudo",
" └─TopN(Probe) 6.00 cop[tikv] test.tbl_29711.col_250, test.tbl_29711.col_251, offset:0, count:6",
" └─TableRowIDScan 30.00 cop[tikv] table:tbl_29711 keep order:false, stats:pseudo",
" └─TableRowIDScan 250.00 cop[tikv] table:tbl_29711 keep order:false, stats:pseudo",
))

tk.MustExec("drop table if exists t29711")
Expand Down Expand Up @@ -5015,14 +5015,14 @@ func TestPlanCacheForIndexJoinRangeFallback(t *testing.T) {
tk.MustExec("drop table if exists t1, t2")
tk.MustExec("create table t1(a int, b varchar(10), c varchar(10), index idx_a_b(a, b))")
tk.MustExec("create table t2(d int)")
tk.MustExec("set @@tidb_opt_range_max_size=1275")
// 1275 is enough for [? a,? a], [? b,? b], [? c,? c] but is not enough for [? aaaaaa,? aaaaaa], [? bbbbbb,? bbbbbb], [? cccccc,? cccccc].
tk.MustExec("set @@tidb_opt_range_max_size=1260")
// 1260 is enough for [? a,? a], [? b,? b], [? c,? c] but is not enough for [? aaaaaa,? aaaaaa], [? bbbbbb,? bbbbbb], [? cccccc,? cccccc].
rows := tk.MustQuery("explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in ('a', 'b', 'c')").Rows()
require.True(t, strings.Contains(rows[6][4].(string), "range: decided by [eq(test.t1.a, test.t2.d) in(test.t1.b, a, b, c)]"))
tk.MustQuery("show warnings").Check(testkit.Rows())
rows = tk.MustQuery("explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in ('aaaaaa', 'bbbbbb', 'cccccc');").Rows()
require.True(t, strings.Contains(rows[6][4].(string), "range: decided by [eq(test.t1.a, test.t2.d)]"))
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1275 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen"))
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1260 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen"))

tk.MustExec("prepare stmt1 from 'select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in (?, ?, ?)'")
tk.MustExec("set @a='a', @b='b', @c='c'")
Expand All @@ -5043,7 +5043,7 @@ func TestPlanCacheForIndexJoinRangeFallback(t *testing.T) {
tk.MustExec("prepare stmt2 from 'select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in (?, ?, ?, ?, ?)'")
tk.MustExec("set @a='a', @b='b', @c='c', @d='d', @e='e'")
tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e")
tk.MustQuery("show warnings").Sort().Check(testkit.Rows("Warning 1105 Memory capacity of 1275 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen",
tk.MustQuery("show warnings").Sort().Check(testkit.Rows("Warning 1105 Memory capacity of 1260 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen",
"Warning 1105 skip prepared plan-cache: in-list is too long"))
tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e")
tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0"))
Expand Down
5 changes: 3 additions & 2 deletions planner/core/plan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -605,8 +605,9 @@ func TestIssue25729(t *testing.T) {
for i := 0; i < 10; i++ {
tk.MustQuery("explain format='brief' select * from t1 where concat(a, b) like \"aadwa\" and a = \"a\";").Check(testkit.Rows(
"Projection 0.10 root test.t1.a, test.t1.b",
"└─IndexReader 0.10 root index:IndexRangeScan",
" └─IndexRangeScan 0.10 cop[tikv] table:t1, index:idx2(a, concat(`a`, `b`), b) range:[\"a\" \"aadwa\",\"a\" \"aadwa\"], keep order:false, stats:pseudo"))
"└─IndexReader 0.10 root index:Selection",
" └─Selection 0.10 cop[tikv] like(concat(test.t1.a, test.t1.b), \"aadwa\", 92)",
" └─IndexRangeScan 0.10 cop[tikv] table:t1, index:idx2(a, concat(`a`, `b`), b) range:[\"a\" \"aadwa\",\"a\" \"aadwa\"], keep order:false, stats:pseudo"))

tk.MustQuery("explain format='brief' select b from t1 where concat(a, b) >= \"aa\" and a = \"b\";").Check(testkit.Rows(
"Projection 33.33 root test.t1.b",
Expand Down
Loading

0 comments on commit 7a9dad0

Please sign in to comment.