From 3b30d1e1eb8e7a4c9afeb3c406c2e8a085bc778f Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 3 Dec 2021 15:24:40 +0800 Subject: [PATCH 01/11] trace join reorder Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 11 +++ planner/core/rule_join_reorder.go | 91 ++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index a16f0111677d0..9406b56534153 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -86,6 +86,17 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName string assertRuleSteps []assertTraceStep }{ + { + sql: "select * from t t1, t t2, t t3, t t4, t t5 where t1.a = t5.a and t5.a = t4.a and t4.a = t3.a and t3.a = t2.a and t2.a = t1.a and t1.a = t3.a and t2.a = t4.a and t3.b = 1 and t4.a = 1", + flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagJoinReOrder}, + assertRuleName: "join_reorder", + assertRuleSteps: []assertTraceStep{ + { + assertAction: "join order become (((t1*t2)*(t3*t4))*t5) from origin ((((t1*t2)*t3)*t4)*t5)", + assertReason: "new join order is better than origin join order", + }, + }, + }, { sql: "select min(distinct a) from t group by a", flags: []uint64{flagBuildKeyInfo, flagEliminateAgg}, diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 9fb38e572d228..a4358382676c8 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -15,10 +15,13 @@ package core import ( + "bytes" "context" - + "fmt" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/plancodec" + "github.com/pingcap/tidb/util/tracing" ) // extractJoinGroup extracts all the join nodes connected with continuous @@ -56,7 +59,11 @@ type jrNode struct { } func (s *joinReOrderSolver) optimize(ctx context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - return s.optimizeRecursive(p.SCtx(), p) + origin := opt.traceJoinReorder(p) + p, err := s.optimizeRecursive(p.SCtx(), p) + now := opt.traceJoinReorder(p) + appendJoinReorderTraceStep(origin, now, opt) + return p, err } // optimizeRecursive recursively collects join groups and applies join reorder algorithm for each group. @@ -194,3 +201,83 @@ func (s *baseSingleGroupJoinOrderSolver) calcJoinCumCost(join LogicalPlan, lNode func (*joinReOrderSolver) name() string { return "join_reorder" } + +func appendJoinReorderTraceStep(origin, now *tracing.LogicalPlanTrace, opt *logicalOptimizeOp) { + if origin == nil || now == nil { + return + } + foldOriginOrder := joinOrderToString(origin) + foldNowOrder := joinOrderToString(now) + if foldOriginOrder != foldNowOrder { + action := fmt.Sprintf("join order become %v from origin %v", foldNowOrder, foldOriginOrder) + reason := "new join order is better than origin join order" + opt.appendStepToCurrent(now.ID, now.TP, reason, action) + } +} + +func (op *logicalOptimizeOp) traceJoinReorder(p LogicalPlan) *tracing.LogicalPlanTrace { + if op.tracer == nil { + return nil + } + return extractJoinAndDataSource(p.buildLogicalPlanTrace(p)) +} + +// joinOrderToString let Join(DataSource, DataSource) become '(t1*t2)' +func joinOrderToString(t *tracing.LogicalPlanTrace) string { + if t.TP == plancodec.TypeJoin { + buffer := bytes.NewBufferString("(") + for i, child := range t.Children { + if i > 0 { + buffer.WriteString("*") + } + buffer.WriteString(joinOrderToString(child)) + } + buffer.WriteString(")") + return buffer.String() + } else if t.TP == plancodec.TypeDataSource { + return t.ExplainInfo[6:] + } + return "" +} + +// extractJoinAndDataSource will only keep join and dataSource operator and remove other operators. +// For example: Proj->Join->(Proj->DataSource, DataSource) will become Join->(DataSource, DataSource) +func extractJoinAndDataSource(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanTrace { + root := findRootJoin(t) + if root == nil { + return nil + } + simplify(root) + return root +} + +func simplify(node *tracing.LogicalPlanTrace) { + if len(node.Children) < 1 { + return + } + for valid := true; !valid; valid = true { + newChildren := make([]*tracing.LogicalPlanTrace, 0) + for _, child := range node.Children { + if child.TP != plancodec.TypeDataSource && child.TP != plancodec.TypeJoin { + newChildren = append(newChildren, child.Children...) + valid = false + } else { + newChildren = append(newChildren, child) + } + } + node.Children = newChildren + } + for _, child := range node.Children { + simplify(child) + } +} + +func findRootJoin(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanTrace { + if t.TP == plancodec.TypeJoin { + return t + } + if len(t.Children) < 1 { + return nil + } + return findRootJoin(t.Children[0]) +} From 9eb395f27eb8b55b8720ce663d3f8a58f0f7bd8b Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 3 Dec 2021 15:39:35 +0800 Subject: [PATCH 02/11] fmt Signed-off-by: yisaer --- planner/core/rule_join_reorder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index a4358382676c8..05ad649fb82b7 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/plancodec" From c432307f5533b4b09fd8bfb24d4e0aff9064b41d Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 6 Dec 2021 14:19:08 +0800 Subject: [PATCH 03/11] fix lint Signed-off-by: yisaer --- planner/core/rule_join_reorder.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 05ad649fb82b7..6a2567063721e 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -252,11 +252,13 @@ func extractJoinAndDataSource(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanT return root } +// simplify only keeps Proj and DataSource operators, and discard other operators. func simplify(node *tracing.LogicalPlanTrace) { if len(node.Children) < 1 { return } - for valid := true; !valid; valid = true { + for valid := true; !valid; { + valid = true newChildren := make([]*tracing.LogicalPlanTrace, 0) for _, child := range node.Children { if child.TP != plancodec.TypeDataSource && child.TP != plancodec.TypeJoin { From 85e1c61b081af5593a68b917ea28d53074563e87 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 8 Dec 2021 13:54:49 +0800 Subject: [PATCH 04/11] address the comment Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 2 +- planner/core/rule_join_reorder.go | 85 +++++++++++++++-------- planner/core/rule_join_reorder_dp.go | 12 ++-- planner/core/rule_join_reorder_dp_test.go | 4 +- planner/core/rule_join_reorder_greedy.go | 11 +-- 5 files changed, 75 insertions(+), 39 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index 9406b56534153..9ae67f6628566 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -93,7 +93,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleSteps: []assertTraceStep{ { assertAction: "join order become (((t1*t2)*(t3*t4))*t5) from origin ((((t1*t2)*t3)*t4)*t5)", - assertReason: "new join order is better than origin join order", + assertReason: "join cost during reorder[[t5, cost:10000],[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000]]", }, }, }, diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 6a2567063721e..ca93c1aa772e6 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -60,20 +60,21 @@ type jrNode struct { } func (s *joinReOrderSolver) optimize(ctx context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - origin := opt.traceJoinReorder(p) - p, err := s.optimizeRecursive(p.SCtx(), p) - now := opt.traceJoinReorder(p) - appendJoinReorderTraceStep(origin, now, opt) + tracer := &joinReorderTrace{cost: map[string]float64{}, opt: opt} + tracer.traceJoinReorder(p) + p, err := s.optimizeRecursive(p.SCtx(), p, tracer) + tracer.traceJoinReorder(p) + appendJoinReorderTraceStep(tracer, p, opt) return p, err } // optimizeRecursive recursively collects join groups and applies join reorder algorithm for each group. -func (s *joinReOrderSolver) optimizeRecursive(ctx sessionctx.Context, p LogicalPlan) (LogicalPlan, error) { +func (s *joinReOrderSolver) optimizeRecursive(ctx sessionctx.Context, p LogicalPlan, tracer *joinReorderTrace) (LogicalPlan, error) { var err error curJoinGroup, eqEdges, otherConds := extractJoinGroup(p) if len(curJoinGroup) > 1 { for i := range curJoinGroup { - curJoinGroup[i], err = s.optimizeRecursive(ctx, curJoinGroup[i]) + curJoinGroup[i], err = s.optimizeRecursive(ctx, curJoinGroup[i], tracer) if err != nil { return nil, err } @@ -88,13 +89,13 @@ func (s *joinReOrderSolver) optimizeRecursive(ctx sessionctx.Context, p LogicalP baseSingleGroupJoinOrderSolver: baseGroupSolver, eqEdges: eqEdges, } - p, err = groupSolver.solve(curJoinGroup) + p, err = groupSolver.solve(curJoinGroup, tracer) } else { dpSolver := &joinReorderDPSolver{ baseSingleGroupJoinOrderSolver: baseGroupSolver, } dpSolver.newJoin = dpSolver.newJoinWithEdges - p, err = dpSolver.solve(curJoinGroup, expression.ScalarFuncs2Exprs(eqEdges)) + p, err = dpSolver.solve(curJoinGroup, expression.ScalarFuncs2Exprs(eqEdges), tracer) } if err != nil { return nil, err @@ -122,7 +123,7 @@ func (s *joinReOrderSolver) optimizeRecursive(ctx sessionctx.Context, p LogicalP } newChildren := make([]LogicalPlan, 0, len(p.Children())) for _, child := range p.Children() { - newChild, err := s.optimizeRecursive(ctx, child) + newChild, err := s.optimizeRecursive(ctx, child, tracer) if err != nil { return nil, err } @@ -203,24 +204,27 @@ func (*joinReOrderSolver) name() string { return "join_reorder" } -func appendJoinReorderTraceStep(origin, now *tracing.LogicalPlanTrace, opt *logicalOptimizeOp) { - if origin == nil || now == nil { +func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt *logicalOptimizeOp) { + if len(tracer.initial) < 1 || len(tracer.final) < 1 { return } - foldOriginOrder := joinOrderToString(origin) - foldNowOrder := joinOrderToString(now) - if foldOriginOrder != foldNowOrder { - action := fmt.Sprintf("join order become %v from origin %v", foldNowOrder, foldOriginOrder) - reason := "new join order is better than origin join order" - opt.appendStepToCurrent(now.ID, now.TP, reason, action) - } -} - -func (op *logicalOptimizeOp) traceJoinReorder(p LogicalPlan) *tracing.LogicalPlanTrace { - if op.tracer == nil { - return nil + if tracer.final != tracer.initial { + action := fmt.Sprintf("join order become %v from origin %v", tracer.final, tracer.initial) + reason := func() string { + buffer := bytes.NewBufferString("join cost during reorder[") + i := 0 + for join, cost := range tracer.cost { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(fmt.Sprintf("[%s, cost:%v]", join, cost)) + i++ + } + buffer.WriteString("]") + return buffer.String() + }() + opt.appendStepToCurrent(plan.ID(), plan.TP(), reason, action) } - return extractJoinAndDataSource(p.buildLogicalPlanTrace(p)) } // joinOrderToString let Join(DataSource, DataSource) become '(t1*t2)' @@ -244,7 +248,7 @@ func joinOrderToString(t *tracing.LogicalPlanTrace) string { // extractJoinAndDataSource will only keep join and dataSource operator and remove other operators. // For example: Proj->Join->(Proj->DataSource, DataSource) will become Join->(DataSource, DataSource) func extractJoinAndDataSource(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanTrace { - root := findRootJoin(t) + root := findRoot(t) if root == nil { return nil } @@ -275,12 +279,37 @@ func simplify(node *tracing.LogicalPlanTrace) { } } -func findRootJoin(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanTrace { - if t.TP == plancodec.TypeJoin { +func findRoot(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanTrace { + if t.TP == plancodec.TypeJoin || t.TP == plancodec.TypeDataSource { return t } if len(t.Children) < 1 { return nil } - return findRootJoin(t.Children[0]) + return findRoot(t.Children[0]) +} + +type joinReorderTrace struct { + opt *logicalOptimizeOp + initial string + final string + cost map[string]float64 +} + +func (t *joinReorderTrace) traceJoinReorder(p LogicalPlan) { + if t == nil || t.opt == nil || t.opt.tracer == nil { + return + } + if len(t.initial) > 0 { + t.final = joinOrderToString(extractJoinAndDataSource(p.buildLogicalPlanTrace(p))) + return + } + t.initial = joinOrderToString(extractJoinAndDataSource(p.buildLogicalPlanTrace(p))) +} + +func (t *joinReorderTrace) appendLogicalJoinCost(join LogicalPlan, cost float64) { + if t == nil || t.opt == nil || t.opt.tracer == nil { + return + } + t.cost[joinOrderToString(extractJoinAndDataSource(join.buildLogicalPlanTrace(join)))] = cost } diff --git a/planner/core/rule_join_reorder_dp.go b/planner/core/rule_join_reorder_dp.go index 560aa5848b54d..3b18613256e0c 100644 --- a/planner/core/rule_join_reorder_dp.go +++ b/planner/core/rule_join_reorder_dp.go @@ -37,16 +37,18 @@ type joinGroupNonEqEdge struct { expr expression.Expression } -func (s *joinReorderDPSolver) solve(joinGroup []LogicalPlan, eqConds []expression.Expression) (LogicalPlan, error) { +func (s *joinReorderDPSolver) solve(joinGroup []LogicalPlan, eqConds []expression.Expression, tracer *joinReorderTrace) (LogicalPlan, error) { for _, node := range joinGroup { _, err := node.recursiveDeriveStats(nil) if err != nil { return nil, err } + cost := s.baseNodeCumCost(node) s.curJoinGroup = append(s.curJoinGroup, &jrNode{ p: node, - cumCost: s.baseNodeCumCost(node), + cumCost: cost, }) + tracer.appendLogicalJoinCost(node, cost) } adjacents := make([][]int, len(s.curJoinGroup)) totalEqEdges := make([]joinGroupEqEdge, 0, len(eqConds)) @@ -120,7 +122,7 @@ func (s *joinReorderDPSolver) solve(joinGroup []LogicalPlan, eqConds []expressio totalNonEqEdges = append(totalNonEqEdges[:i], totalNonEqEdges[i+1:]...) } // Do DP on each sub graph. - join, err := s.dpGraph(visitID2NodeID, nodeID2VisitID, joinGroup, totalEqEdges, subNonEqEdges) + join, err := s.dpGraph(visitID2NodeID, nodeID2VisitID, joinGroup, totalEqEdges, subNonEqEdges, tracer) if err != nil { return nil, err } @@ -159,7 +161,7 @@ func (s *joinReorderDPSolver) bfsGraph(startNode int, visited []bool, adjacents // It implements the traditional join reorder algorithm: DP by subset using the following formula: // bestPlan[S:set of node] = the best one among Join(bestPlan[S1:subset of S], bestPlan[S2: S/S1]) func (s *joinReorderDPSolver) dpGraph(visitID2NodeID, nodeID2VisitID []int, joinGroup []LogicalPlan, - totalEqEdges []joinGroupEqEdge, totalNonEqEdges []joinGroupNonEqEdge) (LogicalPlan, error) { + totalEqEdges []joinGroupEqEdge, totalNonEqEdges []joinGroupNonEqEdge, tracer *joinReorderTrace) (LogicalPlan, error) { nodeCnt := uint(len(visitID2NodeID)) bestPlan := make([]*jrNode, 1< curCost { bestPlan[nodeBitmap].p = join bestPlan[nodeBitmap].cumCost = curCost + tracer.appendLogicalJoinCost(join, curCost) } } } diff --git a/planner/core/rule_join_reorder_dp_test.go b/planner/core/rule_join_reorder_dp_test.go index 4c6b849d9c0b7..5d674dca9e66a 100644 --- a/planner/core/rule_join_reorder_dp_test.go +++ b/planner/core/rule_join_reorder_dp_test.go @@ -195,7 +195,7 @@ func (s *testJoinReorderDPSuite) TestDPReorderTPCHQ5(c *C) { }, newJoin: s.newMockJoin, } - result, err := solver.solve(joinGroups, eqConds) + result, err := solver.solve(joinGroups, eqConds, nil) c.Assert(err, IsNil) c.Assert(s.planToString(result), Equals, "MockJoin{supplier, MockJoin{lineitem, MockJoin{orders, MockJoin{customer, MockJoin{nation, region}}}}}") } @@ -212,7 +212,7 @@ func (s *testJoinReorderDPSuite) TestDPReorderAllCartesian(c *C) { }, newJoin: s.newMockJoin, } - result, err := solver.solve(joinGroup, nil) + result, err := solver.solve(joinGroup, nil, nil) c.Assert(err, IsNil) c.Assert(s.planToString(result), Equals, "MockJoin{MockJoin{a, b}, MockJoin{c, d}}") } diff --git a/planner/core/rule_join_reorder_greedy.go b/planner/core/rule_join_reorder_greedy.go index 94fcd2e026517..ff3d438c524e7 100644 --- a/planner/core/rule_join_reorder_greedy.go +++ b/planner/core/rule_join_reorder_greedy.go @@ -41,16 +41,18 @@ type joinReorderGreedySolver struct { // // For the nodes and join trees which don't have a join equal condition to // connect them, we make a bushy join tree to do the cartesian joins finally. -func (s *joinReorderGreedySolver) solve(joinNodePlans []LogicalPlan) (LogicalPlan, error) { +func (s *joinReorderGreedySolver) solve(joinNodePlans []LogicalPlan, tracer *joinReorderTrace) (LogicalPlan, error) { for _, node := range joinNodePlans { _, err := node.recursiveDeriveStats(nil) if err != nil { return nil, err } + cost := s.baseNodeCumCost(node) s.curJoinGroup = append(s.curJoinGroup, &jrNode{ p: node, - cumCost: s.baseNodeCumCost(node), + cumCost: cost, }) + tracer.appendLogicalJoinCost(node, cost) } sort.SliceStable(s.curJoinGroup, func(i, j int) bool { return s.curJoinGroup[i].cumCost < s.curJoinGroup[j].cumCost @@ -58,7 +60,7 @@ func (s *joinReorderGreedySolver) solve(joinNodePlans []LogicalPlan) (LogicalPla var cartesianGroup []LogicalPlan for len(s.curJoinGroup) > 0 { - newNode, err := s.constructConnectedJoinTree() + newNode, err := s.constructConnectedJoinTree(tracer) if err != nil { return nil, err } @@ -68,7 +70,7 @@ func (s *joinReorderGreedySolver) solve(joinNodePlans []LogicalPlan) (LogicalPla return s.makeBushyJoin(cartesianGroup), nil } -func (s *joinReorderGreedySolver) constructConnectedJoinTree() (*jrNode, error) { +func (s *joinReorderGreedySolver) constructConnectedJoinTree(tracer *joinReorderTrace) (*jrNode, error) { curJoinTree := s.curJoinGroup[0] s.curJoinGroup = s.curJoinGroup[1:] for { @@ -101,6 +103,7 @@ func (s *joinReorderGreedySolver) constructConnectedJoinTree() (*jrNode, error) p: bestJoin, cumCost: bestCost, } + tracer.appendLogicalJoinCost(bestJoin, bestCost) s.curJoinGroup = append(s.curJoinGroup[:bestIdx], s.curJoinGroup[bestIdx+1:]...) s.otherConds = finalRemainOthers } From 2825e8fce23fba20d07558ce9bdb717b6a26be10 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 8 Dec 2021 14:21:53 +0800 Subject: [PATCH 05/11] address the comment Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 2 +- planner/core/rule_join_reorder.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index 9ae67f6628566..e2c57c36f62a3 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -93,7 +93,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleSteps: []assertTraceStep{ { assertAction: "join order become (((t1*t2)*(t3*t4))*t5) from origin ((((t1*t2)*t3)*t4)*t5)", - assertReason: "join cost during reorder[[t5, cost:10000],[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000]]", + assertReason: "join cost during reorder[[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000],[t5, cost:10000]]", }, }, }, diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index ca93c1aa772e6..4f9246ba96155 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "sort" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" @@ -212,12 +213,16 @@ func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt action := fmt.Sprintf("join order become %v from origin %v", tracer.final, tracer.initial) reason := func() string { buffer := bytes.NewBufferString("join cost during reorder[") - i := 0 - for join, cost := range tracer.cost { + var joins []string + for join := range tracer.cost { + joins = append(joins, join) + } + sort.Strings(joins) + for i, join := range joins { if i > 0 { buffer.WriteString(",") } - buffer.WriteString(fmt.Sprintf("[%s, cost:%v]", join, cost)) + buffer.WriteString(fmt.Sprintf("[%s, cost:%v]", join, tracer.cost[join])) i++ } buffer.WriteString("]") From a5f2cf85cbe899d2f16b4188e55e35731f2945d2 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 8 Dec 2021 14:37:17 +0800 Subject: [PATCH 06/11] address the comment Signed-off-by: yisaer --- planner/core/rule_join_reorder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 4f9246ba96155..9884fee0ee1a1 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -261,12 +261,12 @@ func extractJoinAndDataSource(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanT return root } -// simplify only keeps Proj and DataSource operators, and discard other operators. +// simplify only keeps Join and DataSource operators, and discard other operators. func simplify(node *tracing.LogicalPlanTrace) { if len(node.Children) < 1 { return } - for valid := true; !valid; { + for valid := false; !valid; { valid = true newChildren := make([]*tracing.LogicalPlanTrace, 0) for _, child := range node.Children { From 7295b11a6d5eb508f4b1a5764205e69d0dfbb157 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 8 Dec 2021 15:01:17 +0800 Subject: [PATCH 07/11] address the comment Signed-off-by: yisaer --- planner/core/rule_join_reorder.go | 1 - 1 file changed, 1 deletion(-) diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 9884fee0ee1a1..04f45cc29b2d9 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -223,7 +223,6 @@ func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt buffer.WriteString(",") } buffer.WriteString(fmt.Sprintf("[%s, cost:%v]", join, tracer.cost[join])) - i++ } buffer.WriteString("]") return buffer.String() From d3e3a99bc18e44ecc9f599a0382a590a906e0b1b Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 8 Dec 2021 20:30:02 +0800 Subject: [PATCH 08/11] address the comment Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 17 +++++++-- planner/core/rule_join_reorder.go | 50 +++++++++++++++++-------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index e2c57c36f62a3..c801c7fe96b26 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -86,14 +86,25 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName string assertRuleSteps []assertTraceStep }{ + { + sql: "select * from (t t1, t t2, t t3,t t4) union all select * from (t t5, t t6, t t7,t t8)", + flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagJoinReOrder}, + assertRuleName: "join_reorder", + assertRuleSteps: []assertTraceStep{ + { + assertAction: "join order becomes [((t1*t2)*(t3*t4)),((t5*t6)*(t7*t8))] from original [(((t1*t2)*t3)*t4),(((t5*t6)*t7)*t8)]", + assertReason: "join cost during reorder: [[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000],[t5, cost:10000],[t6, cost:10000],[t7, cost:10000],[t8, cost:10000]]", + }, + }, + }, { sql: "select * from t t1, t t2, t t3, t t4, t t5 where t1.a = t5.a and t5.a = t4.a and t4.a = t3.a and t3.a = t2.a and t2.a = t1.a and t1.a = t3.a and t2.a = t4.a and t3.b = 1 and t4.a = 1", flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagJoinReOrder}, assertRuleName: "join_reorder", assertRuleSteps: []assertTraceStep{ { - assertAction: "join order become (((t1*t2)*(t3*t4))*t5) from origin ((((t1*t2)*t3)*t4)*t5)", - assertReason: "join cost during reorder[[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000],[t5, cost:10000]]", + assertAction: "join order becomes (((t1*t2)*(t3*t4))*t5) from original ((((t1*t2)*t3)*t4)*t5)", + assertReason: "join cost during reorder: [[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000],[t5, cost:10000]]", }, }, }, @@ -176,8 +187,6 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { } p, err = logicalOptimize(ctx, flag, p.(LogicalPlan)) c.Assert(err, IsNil) - _, ok := p.(*LogicalProjection) - c.Assert(ok, IsTrue) otrace := sctx.GetSessionVars().StmtCtx.LogicalOptimizeTrace c.Assert(otrace, NotNil) assert := false diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 04f45cc29b2d9..bea2f5179370a 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -210,9 +210,9 @@ func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt return } if tracer.final != tracer.initial { - action := fmt.Sprintf("join order become %v from origin %v", tracer.final, tracer.initial) + action := fmt.Sprintf("join order becomes %v from original %v", tracer.final, tracer.initial) reason := func() string { - buffer := bytes.NewBufferString("join cost during reorder[") + buffer := bytes.NewBufferString("join cost during reorder: [") var joins []string for join := range tracer.cost { joins = append(joins, join) @@ -231,6 +231,21 @@ func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt } } +func allJoinOrderToString(tt []*tracing.LogicalPlanTrace) string { + if len(tt) == 1 { + return joinOrderToString(tt[0]) + } + buffer := bytes.NewBufferString("[") + for i, t := range tt { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(joinOrderToString(t)) + } + buffer.WriteString("]") + return buffer.String() +} + // joinOrderToString let Join(DataSource, DataSource) become '(t1*t2)' func joinOrderToString(t *tracing.LogicalPlanTrace) string { if t.TP == plancodec.TypeJoin { @@ -251,13 +266,17 @@ func joinOrderToString(t *tracing.LogicalPlanTrace) string { // extractJoinAndDataSource will only keep join and dataSource operator and remove other operators. // For example: Proj->Join->(Proj->DataSource, DataSource) will become Join->(DataSource, DataSource) -func extractJoinAndDataSource(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanTrace { - root := findRoot(t) - if root == nil { +func extractJoinAndDataSource(t *tracing.LogicalPlanTrace) []*tracing.LogicalPlanTrace { + roots := findRoots(t) + if len(roots) < 1 { return nil } - simplify(root) - return root + var rr []*tracing.LogicalPlanTrace + for _, root := range roots { + simplify(root) + rr = append(rr, root) + } + return rr } // simplify only keeps Join and DataSource operators, and discard other operators. @@ -283,14 +302,15 @@ func simplify(node *tracing.LogicalPlanTrace) { } } -func findRoot(t *tracing.LogicalPlanTrace) *tracing.LogicalPlanTrace { +func findRoots(t *tracing.LogicalPlanTrace) []*tracing.LogicalPlanTrace { if t.TP == plancodec.TypeJoin || t.TP == plancodec.TypeDataSource { - return t + return []*tracing.LogicalPlanTrace{t} } - if len(t.Children) < 1 { - return nil + var r []*tracing.LogicalPlanTrace + for _, child := range t.Children { + r = append(r, findRoots(child)...) } - return findRoot(t.Children[0]) + return r } type joinReorderTrace struct { @@ -305,15 +325,15 @@ func (t *joinReorderTrace) traceJoinReorder(p LogicalPlan) { return } if len(t.initial) > 0 { - t.final = joinOrderToString(extractJoinAndDataSource(p.buildLogicalPlanTrace(p))) + t.final = allJoinOrderToString(extractJoinAndDataSource(p.buildLogicalPlanTrace(p))) return } - t.initial = joinOrderToString(extractJoinAndDataSource(p.buildLogicalPlanTrace(p))) + t.initial = allJoinOrderToString(extractJoinAndDataSource(p.buildLogicalPlanTrace(p))) } func (t *joinReorderTrace) appendLogicalJoinCost(join LogicalPlan, cost float64) { if t == nil || t.opt == nil || t.opt.tracer == nil { return } - t.cost[joinOrderToString(extractJoinAndDataSource(join.buildLogicalPlanTrace(join)))] = cost + t.cost[allJoinOrderToString(extractJoinAndDataSource(join.buildLogicalPlanTrace(join)))] = cost } From 3d771c607dbf33df76b4c8a97db1844f9e0d1fea Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 9 Dec 2021 14:07:57 +0800 Subject: [PATCH 09/11] address the comment Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 12 ++++----- planner/core/rule_join_reorder.go | 36 ++++++++++++------------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index c801c7fe96b26..b32237578eb46 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -88,7 +88,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { }{ { sql: "select * from (t t1, t t2, t t3,t t4) union all select * from (t t5, t t6, t t7,t t8)", - flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagJoinReOrder}, + flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagDecorrelate, flagPredicatePushDown, flagEliminateOuterJoin, flagJoinReOrder}, assertRuleName: "join_reorder", assertRuleSteps: []assertTraceStep{ { @@ -98,13 +98,13 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { }, }, { - sql: "select * from t t1, t t2, t t3, t t4, t t5 where t1.a = t5.a and t5.a = t4.a and t4.a = t3.a and t3.a = t2.a and t2.a = t1.a and t1.a = t3.a and t2.a = t4.a and t3.b = 1 and t4.a = 1", - flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagJoinReOrder}, + sql: "select * from t t1, t t2, t t3,t t4 where t1.a=t2.a and t3.a=t2.a", + flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagDecorrelate, flagPredicatePushDown, flagEliminateOuterJoin, flagJoinReOrder}, assertRuleName: "join_reorder", assertRuleSteps: []assertTraceStep{ { - assertAction: "join order becomes (((t1*t2)*(t3*t4))*t5) from original ((((t1*t2)*t3)*t4)*t5)", - assertReason: "join cost during reorder: [[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000],[t5, cost:10000]]", + assertAction: "join order becomes (((t1*t2)*t3)*t4) from original (((t1*t2)*t3)*t4)", + assertReason: "join cost during reorder: [[((t1*t2)*t3), cost:58125],[(t1*t2), cost:32500],[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000]]", }, }, }, @@ -185,7 +185,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { for _, f := range tc.flags { flag = flag | f } - p, err = logicalOptimize(ctx, flag, p.(LogicalPlan)) + _, err = logicalOptimize(ctx, flag, p.(LogicalPlan)) c.Assert(err, IsNil) otrace := sctx.GetSessionVars().StmtCtx.LogicalOptimizeTrace c.Assert(otrace, NotNil) diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index bea2f5179370a..fe38877be85a4 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -209,26 +209,24 @@ func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt if len(tracer.initial) < 1 || len(tracer.final) < 1 { return } - if tracer.final != tracer.initial { - action := fmt.Sprintf("join order becomes %v from original %v", tracer.final, tracer.initial) - reason := func() string { - buffer := bytes.NewBufferString("join cost during reorder: [") - var joins []string - for join := range tracer.cost { - joins = append(joins, join) - } - sort.Strings(joins) - for i, join := range joins { - if i > 0 { - buffer.WriteString(",") - } - buffer.WriteString(fmt.Sprintf("[%s, cost:%v]", join, tracer.cost[join])) + action := fmt.Sprintf("join order becomes %v from original %v", tracer.final, tracer.initial) + reason := func() string { + buffer := bytes.NewBufferString("join cost during reorder: [") + var joins []string + for join := range tracer.cost { + joins = append(joins, join) + } + sort.Strings(joins) + for i, join := range joins { + if i > 0 { + buffer.WriteString(",") } - buffer.WriteString("]") - return buffer.String() - }() - opt.appendStepToCurrent(plan.ID(), plan.TP(), reason, action) - } + buffer.WriteString(fmt.Sprintf("[%s, cost:%v]", join, tracer.cost[join])) + } + buffer.WriteString("]") + return buffer.String() + }() + opt.appendStepToCurrent(plan.ID(), plan.TP(), reason, action) } func allJoinOrderToString(tt []*tracing.LogicalPlanTrace) string { From 1e343e2f6aad2bf7a12dc8652c77cd709f72b189 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 9 Dec 2021 14:33:41 +0800 Subject: [PATCH 10/11] address the comment Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 6 +++--- planner/core/rule_join_reorder.go | 3 ++- planner/core/rule_join_reorder_dp.go | 3 +-- planner/core/rule_join_reorder_greedy.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index b32237578eb46..fa866428feac4 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -98,13 +98,13 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { }, }, { - sql: "select * from t t1, t t2, t t3,t t4 where t1.a=t2.a and t3.a=t2.a", + sql: "select * from t t1, t t2, t t3 where t1.a=t2.a and t3.a=t2.a", flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagDecorrelate, flagPredicatePushDown, flagEliminateOuterJoin, flagJoinReOrder}, assertRuleName: "join_reorder", assertRuleSteps: []assertTraceStep{ { - assertAction: "join order becomes (((t1*t2)*t3)*t4) from original (((t1*t2)*t3)*t4)", - assertReason: "join cost during reorder: [[((t1*t2)*t3), cost:58125],[(t1*t2), cost:32500],[t1, cost:10000],[t2, cost:10000],[t3, cost:10000],[t4, cost:10000]]", + assertAction: "join order becomes ((t1*t2)*t3) from original ((t1*t2)*t3)", + assertReason: "join cost during reorder: [[((t1*t2)*t3), cost:58125],[(t1*t2), cost:32500],[t1, cost:10000],[t2, cost:10000],[t3, cost:10000]]", }, }, }, diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index fe38877be85a4..dd29f7d3f1f30 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -333,5 +333,6 @@ func (t *joinReorderTrace) appendLogicalJoinCost(join LogicalPlan, cost float64) if t == nil || t.opt == nil || t.opt.tracer == nil { return } - t.cost[allJoinOrderToString(extractJoinAndDataSource(join.buildLogicalPlanTrace(join)))] = cost + joinMapKey := allJoinOrderToString(extractJoinAndDataSource(join.buildLogicalPlanTrace(join))) + t.cost[joinMapKey] = cost } diff --git a/planner/core/rule_join_reorder_dp.go b/planner/core/rule_join_reorder_dp.go index 3b18613256e0c..d5db965667258 100644 --- a/planner/core/rule_join_reorder_dp.go +++ b/planner/core/rule_join_reorder_dp.go @@ -194,16 +194,15 @@ func (s *joinReorderDPSolver) dpGraph(visitID2NodeID, nodeID2VisitID []int, join return nil, err } curCost := s.calcJoinCumCost(join, bestPlan[sub], bestPlan[remain]) + tracer.appendLogicalJoinCost(join, curCost) if bestPlan[nodeBitmap] == nil { bestPlan[nodeBitmap] = &jrNode{ p: join, cumCost: curCost, } - tracer.appendLogicalJoinCost(join, curCost) } else if bestPlan[nodeBitmap].cumCost > curCost { bestPlan[nodeBitmap].p = join bestPlan[nodeBitmap].cumCost = curCost - tracer.appendLogicalJoinCost(join, curCost) } } } diff --git a/planner/core/rule_join_reorder_greedy.go b/planner/core/rule_join_reorder_greedy.go index ff3d438c524e7..70b287125e3b9 100644 --- a/planner/core/rule_join_reorder_greedy.go +++ b/planner/core/rule_join_reorder_greedy.go @@ -88,6 +88,7 @@ func (s *joinReorderGreedySolver) constructConnectedJoinTree(tracer *joinReorder return nil, err } curCost := s.calcJoinCumCost(newJoin, curJoinTree, node) + tracer.appendLogicalJoinCost(newJoin, curCost) if bestCost > curCost { bestCost = curCost bestJoin = newJoin @@ -103,7 +104,6 @@ func (s *joinReorderGreedySolver) constructConnectedJoinTree(tracer *joinReorder p: bestJoin, cumCost: bestCost, } - tracer.appendLogicalJoinCost(bestJoin, bestCost) s.curJoinGroup = append(s.curJoinGroup[:bestIdx], s.curJoinGroup[bestIdx+1:]...) s.otherConds = finalRemainOthers } From 553a97b495856e48945109c3bffcb94cec89930f Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 10 Dec 2021 11:44:25 +0800 Subject: [PATCH 11/11] change testcase Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index fa866428feac4..6ea57ede920c8 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -98,13 +98,13 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { }, }, { - sql: "select * from t t1, t t2, t t3 where t1.a=t2.a and t3.a=t2.a", + sql: "select * from t t1, t t2, t t3 where t1.a=t2.a and t3.a=t2.a and t1.a=t3.a", flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagDecorrelate, flagPredicatePushDown, flagEliminateOuterJoin, flagJoinReOrder}, assertRuleName: "join_reorder", assertRuleSteps: []assertTraceStep{ { assertAction: "join order becomes ((t1*t2)*t3) from original ((t1*t2)*t3)", - assertReason: "join cost during reorder: [[((t1*t2)*t3), cost:58125],[(t1*t2), cost:32500],[t1, cost:10000],[t2, cost:10000],[t3, cost:10000]]", + assertReason: "join cost during reorder: [[((t1*t2)*t3), cost:58125],[(t1*t2), cost:32500],[(t1*t3), cost:32500],[t1, cost:10000],[t2, cost:10000],[t3, cost:10000]]", }, }, },