Skip to content

Commit

Permalink
preprocessor, planner: inline processing by default for single-consum…
Browse files Browse the repository at this point in the history 
…er cte (#37800)
  • Loading branch information
@elsa0520
elsa0520 authored Sep 28, 2022
1 parent 8378bc7 commit 150813d
Show file tree
Hide file tree
Showing 14 changed files with 577 additions and 542 deletions.
4 changes: 2 additions & 2 deletions bindinfo/bind_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,8 +352,8 @@ func TestBindCTEMerge(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("drop table if exists t1")
tk.MustExec("create table t1(id int)")
require.True(t, tk.HasPlan("with cte as (select * from t1) select * from cte", "CTEFullScan"))
require.False(t, tk.HasPlan("with cte as (select /*+ MERGE() */ * from t1) select * from cte", "CTEFullScan"))
require.True(t, tk.HasPlan("with cte as (select * from t1) select * from cte a, cte b", "CTEFullScan"))
require.False(t, tk.HasPlan("with cte as (select /*+ MERGE() */ * from t1) select * from cte a, cte b", "CTEFullScan"))
tk.MustExec(`
create global binding for
with cte as (select * from t1) select * from cte
Expand Down
4 changes: 2 additions & 2 deletions bindinfo/capture_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -563,9 +563,9 @@ func TestIssue25505(t *testing.T) {
spmMap["with recursive `cte` ( `a` ) as ( select ? union select `a` + ? from `test` . `t1` where `a` > ? ) select * from `cte`"] =
"WITH RECURSIVE `cte` (`a`) AS (SELECT 2 UNION SELECT `a` + 1 FROM `test`.`t1` WHERE `a` > 5) SELECT /*+ hash_agg(@`sel_1`), use_index(@`sel_3` `test`.`t1` `idx_b`)*/ * FROM `cte`"
spmMap["with `cte` as ( with `cte1` as ( select * from `test` . `t2` where `a` > ? and `b` > ? ) select * from `cte1` ) select * from `cte` join `test` . `t1` on `t1` . `a` = `cte` . `a`"] =
"WITH `cte` AS (WITH `cte1` AS (SELECT * FROM `test`.`t2` WHERE `a` > 1 AND `b` > 1) SELECT * FROM `cte1`) SELECT /*+ inl_join(@`sel_1` `test`.`t1`), use_index(@`sel_1` `test`.`t1` `idx_ab`), use_index(@`sel_3` `test`.`t2` `idx_ab`)*/ * FROM `cte` JOIN `test`.`t1` ON `t1`.`a` = `cte`.`a`"
"WITH `cte` AS (WITH `cte1` AS (SELECT * FROM `test`.`t2` WHERE `a` > 1 AND `b` > 1) SELECT * FROM `cte1`) SELECT /*+ use_index(@`sel_3` `test`.`t2` `idx_ab`), use_index(@`sel_1` `test`.`t1` `idx_ab`)*/ * FROM `cte` JOIN `test`.`t1` ON `t1`.`a` = `cte`.`a`"
spmMap["with `cte` as ( with `cte1` as ( select * from `test` . `t2` where `a` = ? and `b` = ? ) select * from `cte1` ) select * from `cte` join `test` . `t1` on `t1` . `a` = `cte` . `a`"] =
"WITH `cte` AS (WITH `cte1` AS (SELECT * FROM `test`.`t2` WHERE `a` = 1 AND `b` = 1) SELECT * FROM `cte1`) SELECT /*+ inl_join(@`sel_1` `test`.`t1`), use_index(@`sel_1` `test`.`t1` `idx_a`), use_index(@`sel_3` `test`.`t2` `idx_a`)*/ * FROM `cte` JOIN `test`.`t1` ON `t1`.`a` = `cte`.`a`"
"WITH `cte` AS (WITH `cte1` AS (SELECT * FROM `test`.`t2` WHERE `a` = 1 AND `b` = 1) SELECT * FROM `cte1`) SELECT /*+ use_index(@`sel_3` `test`.`t2` `idx_a`), use_index(@`sel_1` `test`.`t1` `idx_a`)*/ * FROM `cte` JOIN `test`.`t1` ON `t1`.`a` = `cte`.`a`"

tk.MustExec("with cte as (with cte1 as (select /*+use_index(t2 idx_a)*/ * from t2 where a = 1 and b = 1) select * from cte1) select /*+use_index(t1 idx_a)*/ * from cte join t1 on t1.a=cte.a;")
tk.MustExec("with cte as (with cte1 as (select /*+use_index(t2 idx_a)*/ * from t2 where a = 1 and b = 1) select * from cte1) select /*+use_index(t1 idx_a)*/ * from cte join t1 on t1.a=cte.a;")
Expand Down
190 changes: 75 additions & 115 deletions cmd/explaintest/r/cte.result
View file

Large diffs are not rendered by default.

146 changes: 61 additions & 85 deletions cmd/explaintest/r/explain_cte.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,19 @@ insert into t1 values(1, 0), (2, 1);
insert into t2 values(1, 0), (2, 1);
explain with cte(a) as (select 1) select * from cte;
id estRows task access object operator info
CTEFullScan_8 1.00 root CTE:cte data:CTE_0
CTE_0 1.00 root Non-Recursive CTE
└─Projection_5(Seed Part) 1.00 root 1->Column#1
└─TableDual_6 1.00 root rows:1
Projection_7 1.00 root 1->Column#3
└─TableDual_8 1.00 root rows:1
explain with cte(a) as (select c1 from t1) select * from cte;
id estRows task access object operator info
CTEFullScan_11 10000.00 root CTE:cte data:CTE_0
CTE_0 10000.00 root Non-Recursive CTE
└─TableReader_7(Seed Part) 10000.00 root data:TableFullScan_6
└─TableFullScan_6 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
TableReader_9 10000.00 root data:TableFullScan_8
└─TableFullScan_8 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
explain with cte(a,b,c,d) as (select * from t1, t2) select * from cte;
id estRows task access object operator info
CTEFullScan_18 100000000.00 root CTE:cte data:CTE_0
CTE_0 100000000.00 root Non-Recursive CTE
└─HashJoin_9(Seed Part) 100000000.00 root CARTESIAN inner join
├─TableReader_16(Build) 10000.00 root data:TableFullScan_15
│ └─TableFullScan_15 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_12(Probe) 10000.00 root data:TableFullScan_11
└─TableFullScan_11 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
HashJoin_13 100000000.00 root CARTESIAN inner join
├─TableReader_20(Build) 10000.00 root data:TableFullScan_19
│ └─TableFullScan_19 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_16(Probe) 10000.00 root data:TableFullScan_15
└─TableFullScan_15 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte;
id estRows task access object operator info
CTEFullScan_17 2.00 root CTE:cte data:CTE_0
Expand All @@ -45,15 +39,13 @@ CTE_0 8001.00 root Recursive CTE
└─CTETable_18 10000.00 root Scan on CTE_0
explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte;
id estRows task access object operator info
CTEFullScan_21 2.00 root CTE:cte data:CTE_0
CTE_0 2.00 root Non-Recursive CTE
└─CTEFullScan_19(Seed Part) 2.00 root CTE:cte1 data:CTE_1
CTE_1 2.00 root Recursive CTE
├─Projection_13(Seed Part) 1.00 root 1->Column#2
│ └─TableDual_14 1.00 root rows:1
└─Projection_15(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5
└─Selection_16 0.80 root lt(Column#3, 10)
└─CTETable_17 1.00 root Scan on CTE_1
CTEFullScan_29 2.00 root CTE:cte1 data:CTE_2
CTE_2 2.00 root Recursive CTE
├─Projection_23(Seed Part) 1.00 root 1->Column#9
│ └─TableDual_24 1.00 root rows:1
└─Projection_25(Recursive Part) 0.80 root cast(plus(Column#10, 1), bigint(1) BINARY)->Column#12
└─Selection_26 0.80 root lt(Column#10, 10)
└─CTETable_27 1.00 root Scan on CTE_2
explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte t1, cte t2;
id estRows task access object operator info
HashJoin_20 4.00 root CARTESIAN inner join
Expand Down Expand Up @@ -466,19 +458,14 @@ from version1 v1, version2 v2
where v1.bench_type =v2.bench_type;
id estRows task access object operator info
HashJoin 8000.00 root inner join, equal:[eq(test.t1.bench_type, test.t1.bench_type)]
├─Selection(Build) 6400.00 root not(isnull(test.t1.bench_type))
│ └─CTEFullScan 8000.00 root CTE:version2 AS v2 data:CTE_2
└─Selection(Probe) 6400.00 root not(isnull(test.t1.bench_type))
└─CTEFullScan 8000.00 root CTE:version1 AS v1 data:CTE_1
CTE_2 8000.00 root Non-Recursive CTE
└─Selection(Seed Part) 8000.00 root eq(test.t1.version, "6.0.0"), not(isnull(test.t1.bench_type))
└─CTEFullScan 10000.00 root CTE:all_data data:CTE_0
CTE_1 8000.00 root Non-Recursive CTE
└─Selection(Seed Part) 8000.00 root eq(test.t1.version, "5.4.0"), not(isnull(test.t1.bench_type))
└─CTEFullScan 10000.00 root CTE:all_data data:CTE_0
CTE_0 10000.00 root Non-Recursive CTE
└─TableReader(Seed Part) 10000.00 root data:TableFullScan
└─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
├─Selection(Build) 6400.00 root eq(test.t1.version, "6.0.0"), not(isnull(test.t1.bench_type))
│ └─CTEFullScan 8000.00 root CTE:all_data data:CTE_0
└─Selection(Probe) 6400.00 root eq(test.t1.version, "5.4.0"), not(isnull(test.t1.bench_type))
└─CTEFullScan 8000.00 root CTE:all_data data:CTE_0
CTE_0 8000.00 root Non-Recursive CTE
└─Selection(Seed Part) 8000.00 root or(and(eq(test.t1.version, "5.4.0"), not(isnull(test.t1.bench_type))), and(eq(test.t1.version, "6.0.0"), not(isnull(test.t1.bench_type))))
└─TableReader 10000.00 root data:TableFullScan
└─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
drop table if exists tbl;
create table tbl (id int);
explain with t1 as (select id from tbl), t2 as (select a.id from t1 a join t1 b on a.id = b.id) select * from t2 where id in (select id from t2);
Expand Down Expand Up @@ -507,68 +494,57 @@ insert into t2 values(1,1);
insert into t3 values(1,1234);
explain update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321;
id estRows task access object operator info
Update_14 N/A root N/A
└─HashJoin_25 12.49 root inner join, equal:[eq(test.t1.a, test.t2.c)]
├─HashJoin_33(Build) 9.99 root inner join, equal:[eq(test.t3.e, test.t2.d)]
│ ├─Selection_35(Build) 7.99 root not(isnull(test.t3.e))
│ │ └─CTEFullScan_36 9.99 root CTE:temp data:CTE_0
│ └─TableReader_39(Probe) 9980.01 root data:Selection_38
│ └─Selection_38 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d))
│ └─TableFullScan_37 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_29(Probe) 9990.00 root data:Selection_28
└─Selection_28 9990.00 cop[tikv] not(isnull(test.t1.a))
└─TableFullScan_27 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
CTE_0 9.99 root Non-Recursive CTE
└─Projection_17(Seed Part) 9.99 root test.t3.e
└─TableReader_20 9.99 root data:Selection_19
└─Selection_19 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e))
└─TableFullScan_18 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo
Update_17 N/A root N/A
└─HashJoin_22 15.61 root inner join, equal:[eq(test.t1.a, test.t2.c)]
├─HashJoin_30(Build) 12.49 root inner join, equal:[eq(test.t3.e, test.t2.d)]
│ ├─Projection_32(Build) 9.99 root test.t3.e
│ │ └─TableReader_35 9.99 root data:Selection_34
│ │ └─Selection_34 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e))
│ │ └─TableFullScan_33 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo
│ └─TableReader_38(Probe) 9980.01 root data:Selection_37
│ └─Selection_37 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d))
│ └─TableFullScan_36 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_26(Probe) 9990.00 root data:Selection_25
└─Selection_25 9990.00 cop[tikv] not(isnull(test.t1.a))
└─TableFullScan_24 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321;
select * from t1;
a b
1 4321
explain insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
id estRows task access object operator info
Insert_1 N/A root N/A
└─HashJoin_28 12.49 root inner join, equal:[eq(test.t2.c, test.t1.a)]
├─HashJoin_30(Build) 9.99 root inner join, equal:[eq(test.t3.e, test.t2.d)]
│ ├─Selection_31(Build) 7.99 root not(isnull(test.t3.e))
│ │ └─CTEFullScan_32 9.99 root CTE:temp data:CTE_0
│ └─TableReader_35(Probe) 9980.01 root data:Selection_34
│ └─Selection_34 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d))
│ └─TableFullScan_33 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_38(Probe) 9990.00 root data:Selection_37
└─Selection_37 9990.00 cop[tikv] not(isnull(test.t1.a))
└─TableFullScan_36 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
CTE_0 9.99 root Non-Recursive CTE
└─Projection_18(Seed Part) 9.99 root test.t3.e
└─TableReader_21 9.99 root data:Selection_20
└─Selection_20 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e))
└─TableFullScan_19 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo
└─HashJoin_25 15.61 root inner join, equal:[eq(test.t2.c, test.t1.a)]
├─HashJoin_27(Build) 12.49 root inner join, equal:[eq(test.t3.e, test.t2.d)]
│ ├─TableReader_30(Build) 9.99 root data:Selection_29
│ │ └─Selection_29 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e))
│ │ └─TableFullScan_28 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo
│ └─TableReader_33(Probe) 9980.01 root data:Selection_32
│ └─Selection_32 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d))
│ └─TableFullScan_31 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_36(Probe) 9990.00 root data:Selection_35
└─Selection_35 9990.00 cop[tikv] not(isnull(test.t1.a))
└─TableFullScan_34 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
select * from t1;
a b
1 4321
1 4321
explain delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
id estRows task access object operator info
Delete_14 N/A root N/A
└─Projection_25 12.49 root test.t1.a, test.t1.b, test.t1._tidb_rowid, test.t2.c
└─HashJoin_27 12.49 root inner join, equal:[eq(test.t2.c, test.t1.a)]
├─HashJoin_29(Build) 9.99 root inner join, equal:[eq(test.t3.e, test.t2.d)]
│ ├─Selection_30(Build) 7.99 root not(isnull(test.t3.e))
│ │ └─CTEFullScan_31 9.99 root CTE:temp data:CTE_0
│ └─TableReader_34(Probe) 9980.01 root data:Selection_33
│ └─Selection_33 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d))
│ └─TableFullScan_32 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_37(Probe) 9990.00 root data:Selection_36
└─Selection_36 9990.00 cop[tikv] not(isnull(test.t1.a))
└─TableFullScan_35 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
CTE_0 9.99 root Non-Recursive CTE
└─Projection_17(Seed Part) 9.99 root test.t3.e
└─TableReader_20 9.99 root data:Selection_19
└─Selection_19 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e))
└─TableFullScan_18 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo
Delete_17 N/A root N/A
└─Projection_22 15.61 root test.t1.a, test.t1.b, test.t1._tidb_rowid, test.t2.c
└─HashJoin_24 15.61 root inner join, equal:[eq(test.t2.c, test.t1.a)]
├─HashJoin_26(Build) 12.49 root inner join, equal:[eq(test.t3.e, test.t2.d)]
│ ├─TableReader_29(Build) 9.99 root data:Selection_28
│ │ └─Selection_28 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e))
│ │ └─TableFullScan_27 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo
│ └─TableReader_32(Probe) 9980.01 root data:Selection_31
│ └─Selection_31 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d))
│ └─TableFullScan_30 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─TableReader_35(Probe) 9990.00 root data:Selection_34
└─Selection_34 9990.00 cop[tikv] not(isnull(test.t1.a))
└─TableFullScan_33 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
select * from t1;
a b
14 changes: 6 additions & 8 deletions cmd/explaintest/r/index_merge.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -798,14 +798,12 @@ create table t1(c1 int, c2 int, c3 int, c4 int, c5 int, key(c1), key(c2), key(c3
insert into t1 values(1, 1, 1, 1, 1), (2, 2, 2, 2, 2), (3, 3, 3, 3, 3), (4, 4, 4, 4, 4), (5, 5, 5, 5, 5);
explain with cte1 as (select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10) select * from cte1 order by 1;
id estRows task access object operator info
Sort_13 2250.55 root test.t1.c1
└─CTEFullScan_16 2250.55 root CTE:cte1 data:CTE_0
CTE_0 2250.55 root Non-Recursive CTE
└─IndexMerge_12(Seed Part) 2250.55 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
Sort_10 4060.74 root test.t1.c1
└─IndexMerge_17 2250.55 root
├─IndexRangeScan_13(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_14(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_16(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
└─TableRowIDScan_15 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
with cte1 as (select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10) select * from cte1 order by 1;
c1 c2 c3 c4 c5
1 1 1 1 1
Expand Down
5 changes: 5 additions & 0 deletions parser/ast/dml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ func (*TableName) resultSet() {}

// Restore implements Node interface.
func (n *TableName) restoreName(ctx *format.RestoreCtx) {
// restore db name
if n.Schema.String() != "" {
ctx.WriteName(n.Schema.String())
ctx.WritePlain(".")
Expand All @@ -298,6 +299,7 @@ func (n *TableName) restoreName(ctx *format.RestoreCtx) {
ctx.WritePlain(".")
}
}
// restore table name
ctx.WriteName(n.Name.String())
}

Expand Down Expand Up @@ -1047,6 +1049,9 @@ type CommonTableExpression struct {
Query *SubqueryExpr
ColNameList []model.CIStr
IsRecursive bool

// Record how many consumers the current cte has
ConsumerCount int
}

// Restore implements Node interface
Expand Down
Loading

0 comments on commit 150813d

Please sign in to comment.