Skip to content

Commit

Permalink
planner, CTE: Fix default inline CTE which contains agg or window fun…
Browse files Browse the repository at this point in the history 
…ction and refactor inline CTE strategy (#48437)

close #47711
  • Loading branch information
@ti-chi-bot
ti-chi-bot authored Nov 9, 2023
1 parent a0d7242 commit cf0a91c
Show file tree
Hide file tree
Showing 13 changed files with 208 additions and 148 deletions.
75 changes: 5 additions & 70 deletions pkg/planner/core/casetest/flatplan/testdata/flat_plan_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,8 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ │ ",
Expand All @@ -232,80 +232,15 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
],
"CTEs": [
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ ",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
],
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ ",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
]
]
"CTEs": null
},
{
"SQL": "WITH RECURSIVE cte (n) AS( SELECT 1 UNION ALL SELECT n + 1 FROM cte WHERE n < 5)SELECT * FROM cte;",
Expand Down
23 changes: 10 additions & 13 deletions pkg/planner/core/casetest/hint/testdata/integration_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -798,24 +798,21 @@
{
"SQL": "explain format = 'brief' select /*+ qb_name(qb_v8, v8), merge(@qb_v8) */ * from v8;",
"Plan": [
"HashAgg 16000.00 root group by:Column#21, funcs:firstrow(Column#21)->Column#21",
"HashAgg 16000.00 root group by:Column#41, funcs:firstrow(Column#41)->Column#41",
"└─Union 1000000010000.00 root ",
" ├─HashJoin 1000000000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" │ └─CTEFullScan(Probe) 100000000.00 root CTE:cte2 data:CTE_1",
" │ └─Projection(Probe) 100000000.00 root 1->Column#55",
" │ └─HashJoin 100000000.00 root CARTESIAN inner join",
" │ ├─Projection(Build) 10000.00 root 1->Column#54",
" │ │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ │ └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
" │ └─Projection(Probe) 10000.00 root 1->Column#53",
" │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo",
" └─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"CTE_1 100000000.00 root Non-Recursive CTE",
"└─HashJoin(Seed Part) 100000000.00 root CARTESIAN inner join",
" ├─CTEFullScan(Build) 10000.00 root CTE:cte4 data:CTE_3",
" └─CTEFullScan(Probe) 10000.00 root CTE:cte3 data:CTE_2",
"CTE_3 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
"CTE_2 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo"
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Warn": null
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/planner/core/casetest/physicalplantest/physical_plan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1206,6 +1206,8 @@ func TestSingleConsumerCTE(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("drop table if exists t;")
tk.MustExec("CREATE TABLE `t` (`a` int(11));")
tk.MustExec("create table t1 (c1 int primary key, c2 int, index c2 (c2));")
tk.MustExec("create table t2 (c1 int unique, c2 int);")
tk.MustExec("insert into t values (1), (5), (10), (15), (20), (30), (50);")

var (
Expand Down
9 changes: 8 additions & 1 deletion pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,10 @@
{
"name": "TestSingleConsumerCTE",
"cases": [
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
"with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
"with cte as (select 1) select * from cte; -- inline cte",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union select * from cte2; -- inline cte1, cte2",
Expand All @@ -564,7 +568,10 @@
"set tidb_opt_force_inline_cte=1; -- enable force inline CTE",
"with cte as (select 1) select * from cte union select * from cte; -- force inline cte while multi-consumer",
"set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined"
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined",
"with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c"
]
},
{
Expand Down
123 changes: 118 additions & 5 deletions pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2774,7 +2774,7 @@
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive CTE can not be inlined."
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
Expand Down Expand Up @@ -3023,7 +3023,9 @@
" └─Selection 0.80 root lt(Column#3, 100)",
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "with cte1 as (with cte2 as (select * from t) select * from cte2) select * from cte1; -- non-recursive 'cte2' definition inside another non-recursive 'cte1'",
Expand All @@ -3044,7 +3046,9 @@
" └─Selection 8000.00 root lt(test.t.a, 100)",
" └─CTETable 10000.00 root Scan on CTE_0"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "with cte1 as (with recursive cte2(c1) as (select 1 union select c1 + 1 c1 from cte2 where c1 < 100) select * from cte2) select * from cte1; -- recursive 'cte2' inside non-recursive 'cte1'",
Expand All @@ -3057,7 +3061,10 @@
" └─Selection 0.80 root lt(Column#10, 100)",
" └─CTETable 1.00 root Scan on CTE_2"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte.",
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
Expand Down Expand Up @@ -3092,7 +3099,7 @@
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive CTE can not be inlined."
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
Expand Down Expand Up @@ -3134,6 +3141,55 @@
{
"Name": "TestSingleConsumerCTE",
"Cases": [
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"Limit 1.00 root offset:0, count:1",
"└─HashAgg 1.00 root group by:Column#18, funcs:firstrow(Column#18)->Column#18",
" └─Union 20000.00 root ",
" ├─TableReader 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"Limit 100.00 root offset:100, count:100",
"└─HashAgg 200.00 root group by:Column#18, funcs:firstrow(Column#18)->Column#18",
" └─Union 20000.00 root ",
" ├─TableReader 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"TableDual 0.00 root rows:0"
],
"Warning": null
},
{
"SQL": "with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
"Plan": [
"HashAgg 3.00 root group by:Column#9, funcs:firstrow(Column#9)->Column#9",
"└─Union 3.00 root ",
" ├─Projection 1.00 root 1->Column#9",
" │ └─TableDual 1.00 root rows:1",
" └─Union 2.00 root ",
" ├─CTEFullScan 1.00 root CTE:cte2 data:CTE_2",
" └─CTEFullScan 1.00 root CTE:cte2 data:CTE_2",
"CTE_2 1.00 root Non-Recursive CTE",
"└─Projection(Seed Part) 1.00 root 3->Column#5",
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
"Plan": [
Expand Down Expand Up @@ -3287,6 +3343,63 @@
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
"Plan": [
"CTEFullScan 1.64 root CTE:temp data:CTE_4",
"CTE_4 1.64 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 1->Column#14",
"│ └─TableDual 1.00 root rows:1",
"└─Projection(Recursive Part) 0.64 root cast(plus(Column#15, 1), bigint(1) BINARY)->Column#18",
" └─HashJoin 0.64 root CARTESIAN inner join, other cond:lt(Column#15, Column#16)",
" ├─Selection(Build) 0.80 root not(isnull(Column#16))",
" │ └─CTEFullScan 1.00 root CTE:maxa data:CTE_1",
" └─Selection(Probe) 0.80 root not(isnull(Column#15))",
" └─CTETable 1.00 root Scan on CTE_4",
"CTE_1 1.00 root Non-Recursive CTE",
"└─StreamAgg(Seed Part) 1.00 root funcs:max(Column#3)->Column#4",
" └─Projection 1.00 root 8->Column#3",
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
"Plan": [
"CTEFullScan 20000.00 root CTE:tmp data:CTE_4",
"CTE_4 20000.00 root Recursive CTE",
"├─Projection(Seed Part) 10000.00 root 1->Column#26",
"│ └─TableReader 10000.00 root data:TableFullScan",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─Projection(Recursive Part) 10000.00 root cast(plus(Column#27, 1), bigint(1) BINARY)->Column#30",
" └─HashJoin 10000.00 root CARTESIAN inner join, other cond:lt(plus(Column#27, 1), Column#28)",
" ├─CTEFullScan(Build) 1.00 root CTE:b data:CTE_1",
" └─CTETable(Probe) 10000.00 root Scan on CTE_4",
"CTE_1 1.00 root Non-Recursive CTE",
"└─Projection(Seed Part) 1.00 root 2->Column#8",
" └─HashAgg 1.00 root funcs:count(Column#37)->Column#35",
" └─TableReader 1.00 root data:HashAgg",
" └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#37",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c",
"Plan": [
"CTEFullScan 1.80 root CTE:tmp data:CTE_4",
"CTE_4 1.80 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 2->Column#37",
"│ └─HashAgg 1.00 root funcs:count(Column#46)->Column#44",
"│ └─TableReader 1.00 root data:HashAgg",
"│ └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#46",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─Projection(Recursive Part) 0.80 root cast(plus(Column#38, 1), bigint(1) BINARY)->Column#40",
" └─Selection 0.80 root lt(plus(Column#38, 1), 10)",
" └─CTETable 1.00 root Scan on CTE_4"
],
"Warning": null
}
]
},
Expand Down
8 changes: 4 additions & 4 deletions pkg/planner/core/casetest/planstats/plan_stats_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,13 @@ func TestPlanStatsLoad(t *testing.T) {
{ // CTE
sql: "with cte(x, y) as (select d + 1, b from t where c > 1) select * from cte where x < 3",
check: func(p plannercore.Plan, tableInfo *model.TableInfo) {
ps, ok := p.(*plannercore.PhysicalSelection)
ps, ok := p.(*plannercore.PhysicalProjection)
require.True(t, ok)
pc, ok := ps.Children()[0].(*plannercore.PhysicalCTE)
pc, ok := ps.Children()[0].(*plannercore.PhysicalTableReader)
require.True(t, ok)
pp, ok := pc.SeedPlan.(*plannercore.PhysicalProjection)
pp, ok := pc.GetTablePlan().(*plannercore.PhysicalSelection)
require.True(t, ok)
reader, ok := pp.Children()[0].(*plannercore.PhysicalTableReader)
reader, ok := pp.Children()[0].(*plannercore.PhysicalTableScan)
require.True(t, ok)
require.Greater(t, countFullStats(reader.StatsInfo().HistColl, tableInfo.Columns[2].ID), 0)
},
Expand Down
Loading

0 comments on commit cf0a91c

Please sign in to comment.