From 470d5546c09474b23279d31ece23cb72c4b2079e Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Thu, 12 Oct 2023 17:26:00 +0800 Subject: [PATCH] planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565) (#45586) ref pingcap/tidb#44409, close pingcap/tidb#45564 --- planner/core/main_test.go | 6 ++ planner/core/rule_join_reorder.go | 7 ++- planner/core/rule_join_reorder_test.go | 39 +++++++----- .../core/testdata/join_reorder_suite_in.json | 12 ++++ .../core/testdata/join_reorder_suite_out.json | 60 +++++++++++++++++++ 5 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 planner/core/testdata/join_reorder_suite_in.json create mode 100644 planner/core/testdata/join_reorder_suite_out.json diff --git a/planner/core/main_test.go b/planner/core/main_test.go index b9cbcdc9e44bd..28390b1dd4227 100644 --- a/planner/core/main_test.go +++ b/planner/core/main_test.go @@ -34,6 +34,8 @@ func TestMain(m *testing.M) { flag.Parse() testDataMap.LoadTestSuiteData("testdata", "plan_suite_unexported") testDataMap.LoadTestSuiteData("testdata", "index_merge_suite") + testDataMap.LoadTestSuiteData("testdata", "join_reorder_suite") + indexMergeSuiteData = testDataMap["index_merge_suite"] planSuiteUnexportedData = testDataMap["plan_suite_unexported"] opts := []goleak.Option{ @@ -56,3 +58,7 @@ func TestMain(m *testing.M) { func GetIndexMergeSuiteData() testdata.TestData { return testDataMap["index_merge_suite"] } + +func GetJoinReorderData() testdata.TestData { + return testDataMap["join_reorder_suite"] +} diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index cae58e21cd7cd..fe5a27e531753 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -533,12 +533,17 @@ func (s *baseSingleGroupJoinOrderSolver) makeJoin(leftPlan, rightPlan LogicalPla remainOtherConds, otherConds = expression.FilterOutInPlace(remainOtherConds, func(expr expression.Expression) bool { return expression.ExprFromSchema(expr, mergedSchema) }) - if (joinType.JoinType == LeftOuterJoin || joinType.JoinType == RightOuterJoin || joinType.JoinType == LeftOuterSemiJoin || joinType.JoinType == AntiLeftOuterSemiJoin) && len(otherConds) > 0 { + + if joinType.JoinType == LeftOuterJoin || joinType.JoinType == RightOuterJoin || joinType.JoinType == LeftOuterSemiJoin || joinType.JoinType == AntiLeftOuterSemiJoin { // the original outer join's other conditions has been bound to the outer join Edge, // these remained other condition here shouldn't be appended to it because on-mismatch // logic will produce more append-null rows which is banned in original semantic. remainOtherConds = append(remainOtherConds, otherConds...) // nozero + remainOtherConds = append(remainOtherConds, leftConds...) // nozero + remainOtherConds = append(remainOtherConds, rightConds...) // nozero otherConds = otherConds[:0] + leftConds = leftConds[:0] + rightConds = rightConds[:0] } if len(joinType.outerBindCondition) > 0 { remainOBOtherConds := make([]expression.Expression, len(joinType.outerBindCondition)) diff --git a/planner/core/rule_join_reorder_test.go b/planner/core/rule_join_reorder_test.go index 5e53b9cc27147..17c14a1a988a3 100644 --- a/planner/core/rule_join_reorder_test.go +++ b/planner/core/rule_join_reorder_test.go @@ -17,7 +17,9 @@ package core_test import ( "testing" + "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/testkit" + "github.com/pingcap/tidb/testkit/testdata" "github.com/stretchr/testify/require" ) @@ -82,21 +84,28 @@ func TestAdditionOtherConditionsRemained4OuterJoin(t *testing.T) { tk.MustExec("INSERT INTO queries_program(`id`, `identifier_id`) values(8, 13), (9, 14);") tk.MustExec("INSERT INTO queries_channel(`id`, `identifier_id`) values(5, 13);") - tk.MustQuery("SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;").Check(testkit.Rows("" + - "13 i1")) - tk.MustQuery("SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` RIGHT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;").Check(testkit.Rows("" + - "13 i1")) - tk.MustQuery("explain format = 'brief' SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;").Check(testkit.Rows(""+ - "Sort 2.50 root test.queries_identifier.id", - "└─Projection 2.50 root test.queries_identifier.id, test.queries_identifier.name", - " └─Selection 2.50 root or(and(eq(test.queries_channel.id, 5), eq(test.queries_program.id, 9)), eq(test.queries_program.id, 8))", - " └─IndexJoin 3.12 root left outer join, inner:IndexReader, outer key:test.queries_identifier.id, inner key:test.queries_channel.identifier_id, equal cond:eq(test.queries_identifier.id, test.queries_channel.identifier_id)", - " ├─IndexHashJoin(Build) 2.50 root inner join, inner:TableReader, outer key:test.queries_program.identifier_id, inner key:test.queries_identifier.id, equal cond:eq(test.queries_program.identifier_id, test.queries_identifier.id)", - " │ ├─Batch_Point_Get(Build) 2.00 root table:queries_program handle:[8 9], keep order:false, desc:false", - " │ └─TableReader(Probe) 2.00 root data:TableRangeScan", - " │ └─TableRangeScan 2.00 cop[tikv] table:queries_identifier range: decided by [test.queries_program.identifier_id], keep order:false, stats:pseudo", - " └─IndexReader(Probe) 2.50 root index:IndexRangeScan", - " └─IndexRangeScan 2.50 cop[tikv] table:queries_channel, index:identifier_id(identifier_id) range: decided by [eq(test.queries_channel.identifier_id, test.queries_identifier.id)], keep order:false, stats:pseudo")) + tk.MustExec("create table t(a int)") + tk.MustExec("create table t1(a int, b int)") + tk.MustExec("create table t2(a int, b int, c int)") + tk.MustExec("create table t3(a int, b int)") + tk.MustExec("create table t4(a int, b int)") + + testData := core.GetJoinReorderData() + var ( + input []string + output []struct { + SQL string + Output []string + } + ) + testData.LoadTestCases(t, &input, &output) + for i, sql := range input { + testdata.OnRecord(func() { + output[i].SQL = sql + output[i].Output = testdata.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Output...)) + } } func TestOuterJoinWIthEqCondCrossInnerJoin(t *testing.T) { diff --git a/planner/core/testdata/join_reorder_suite_in.json b/planner/core/testdata/join_reorder_suite_in.json new file mode 100644 index 0000000000000..053b446000086 --- /dev/null +++ b/planner/core/testdata/join_reorder_suite_in.json @@ -0,0 +1,12 @@ +[ + { + "name": "TestAdditionOtherConditionsRemained4OuterJoin", + "cases": [ + "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` RIGHT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "explain format = 'brief' SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;", + // The where clause should be a Selection out of joins. + "explain format='brief' select * from t left join t1 on t.a=t1.a inner join t2 on t.a=t2.a and t2.c = 100 left join t3 on t2.a=t3.a and t3.b > 1 left join t4 on t2.a = t4.a where (t2.b > 100 or t.a > 10 or t1.b < 10)" + ] + } +] \ No newline at end of file diff --git a/planner/core/testdata/join_reorder_suite_out.json b/planner/core/testdata/join_reorder_suite_out.json new file mode 100644 index 0000000000000..a1cb6c59693bd --- /dev/null +++ b/planner/core/testdata/join_reorder_suite_out.json @@ -0,0 +1,60 @@ +[ + { + "Name": "TestAdditionOtherConditionsRemained4OuterJoin", + "Cases": [ + { + "SQL": "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "Output": [ + "13 i1" + ] + }, + { + "SQL": "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` RIGHT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "Output": [ + "13 i1" + ] + }, + { + "SQL": "explain format = 'brief' SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;", + "Output": [ + "Sort 2.50 root test.queries_identifier.id", + "└─Projection 2.50 root test.queries_identifier.id, test.queries_identifier.name", + " └─Selection 2.50 root or(and(eq(test.queries_channel.id, 5), eq(test.queries_program.id, 9)), eq(test.queries_program.id, 8))", + " └─IndexJoin 3.12 root left outer join, inner:IndexReader, outer key:test.queries_identifier.id, inner key:test.queries_channel.identifier_id, equal cond:eq(test.queries_identifier.id, test.queries_channel.identifier_id)", + " ├─IndexHashJoin(Build) 2.50 root inner join, inner:TableReader, outer key:test.queries_program.identifier_id, inner key:test.queries_identifier.id, equal cond:eq(test.queries_program.identifier_id, test.queries_identifier.id)", + " │ ├─Batch_Point_Get(Build) 2.00 root table:queries_program handle:[8 9], keep order:false, desc:false", + " │ └─TableReader(Probe) 2.00 root data:TableRangeScan", + " │ └─TableRangeScan 2.00 cop[tikv] table:queries_identifier range: decided by [test.queries_program.identifier_id], keep order:false, stats:pseudo", + " └─IndexReader(Probe) 2.50 root index:IndexRangeScan", + " └─IndexRangeScan 2.50 cop[tikv] table:queries_channel, index:identifier_id(identifier_id) range: decided by [eq(test.queries_channel.identifier_id, test.queries_identifier.id)], keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from t left join t1 on t.a=t1.a inner join t2 on t.a=t2.a and t2.c = 100 left join t3 on t2.a=t3.a and t3.b > 1 left join t4 on t2.a = t4.a where (t2.b > 100 or t.a > 10 or t1.b < 10)", + "Output": [ + "Projection 19.51 root test.t.a, test.t1.a, test.t1.b, test.t2.a, test.t2.b, test.t2.c, test.t3.a, test.t3.b, test.t4.a, test.t4.b", + "└─Selection 19.51 root or(gt(test.t2.b, 100), or(gt(test.t.a, 10), lt(test.t1.b, 10))), or(gt(test.t2.b, 100), or(gt(test.t2.a, 10), lt(test.t1.b, 10)))", + " └─HashJoin 24.39 root left outer join, equal:[eq(test.t2.a, test.t4.a)]", + " ├─HashJoin(Build) 19.51 root left outer join, equal:[eq(test.t.a, test.t1.a)]", + " │ ├─HashJoin(Build) 15.61 root inner join, equal:[eq(test.t2.a, test.t.a)]", + " │ │ ├─HashJoin(Build) 12.49 root left outer join, equal:[eq(test.t2.a, test.t3.a)]", + " │ │ │ ├─TableReader(Build) 9.99 root data:Selection", + " │ │ │ │ └─Selection 9.99 cop[tikv] eq(test.t2.c, 100), not(isnull(test.t2.a))", + " │ │ │ │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + " │ │ │ └─TableReader(Probe) 3330.00 root data:Selection", + " │ │ │ └─Selection 3330.00 cop[tikv] gt(test.t3.b, 1), not(isnull(test.t3.a))", + " │ │ │ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo", + " │ │ └─TableReader(Probe) 9990.00 root data:Selection", + " │ │ └─Selection 9990.00 cop[tikv] not(isnull(test.t.a))", + " │ │ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + " │ └─TableReader(Probe) 9990.00 root data:Selection", + " │ └─Selection 9990.00 cop[tikv] not(isnull(test.t1.a))", + " │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", + " └─TableReader(Probe) 9990.00 root data:Selection", + " └─Selection 9990.00 cop[tikv] not(isnull(test.t4.a))", + " └─TableFullScan 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo" + ] + } + ] + } +]